From 02dbd0369be5334bc35e4fa9aa712064c8071e8f Mon Sep 17 00:00:00 2001 From: wushengyeyouya <690574002@qq.com> Date: Mon, 23 Aug 2021 11:06:58 +0800 Subject: [PATCH 1/2] Delete all useless modules to adapt to the new architecture of DSS1.0 --- bin/checkEnv.sh | 45 - bin/checkServices.sh | 91 -- bin/install.sh | 583 ------------ bin/start-all.sh | 189 ---- bin/stop-all.sh | 133 --- datachecker-appjoint/pom.xml | 139 --- .../src/main/assembly/distribution.xml | 136 --- .../jobtype/connector/DataCheckerDao.java | 195 ---- .../jobtype/connector/DataDruidFactory.java | 122 --- .../schedulis/jobtype/entity/DataChecker.java | 90 -- .../src/main/resources/appjoint.properties | 24 - .../src/main/resources/log4j.properties | 37 - .../src/main/resources/log4j2.xml | 39 - .../execution/DataCheckerAppJoint.scala | 51 - .../execution/DataCheckerExecution.scala | 157 ---- .../DataCheckerNodeExecutionAction.scala | 46 - dss-appjoint-auth/pom.xml | 75 -- .../dss/appjoint/auth/AppJointAuth.java | 38 - .../dss/appjoint/auth/RedirectMsg.java | 33 - .../appjoint/auth/impl/AppJointAuthImpl.scala | 99 -- .../appjoint/auth/impl/RedirectMsgImpl.scala | 37 - .../appjoint/auth/impl/UserInfoAction.scala | 28 - .../appjoint/auth/impl/UserInfoResult.scala | 33 - dss-appjoint-core/pom.xml | 93 -- .../wedatasphere/dss/appjoint/AppJoint.java | 73 -- .../exception/AppJointErrorException.java | 35 - .../dss/appjoint/execution/NodeExecution.java | 53 -- .../async/NodeExecutionListener.java | 40 - .../async/NodeExecutionLogListener.java | 26 - .../async/NodeExecutionProgressListener.java | 26 - .../async/NodeExecutionResponseListener.java | 30 - .../async/NodeExecutionStatusListener.java | 26 - .../common/AbstractNodeExecutionAction.java | 35 - .../common/AsyncNodeExecutionResponse.java | 116 --- .../CompletedNodeExecutionResponse.java | 66 -- .../common/LongTermNodeExecutionAction.java | 26 - .../execution/common/NodeExecutionAction.java | 25 - .../common/NodeExecutionResponse.java | 25 - .../execution/common/NodeExecutionState.java | 71 -- .../execution/core/AbstractAppJointNode.java | 144 --- .../appjoint/execution/core/AppJointNode.java | 51 - .../core/CallbackLongTermNodeExecution.java | 59 -- .../execution/core/CommonAppJointNode.java | 71 -- .../core/HttpLongTermNodeExecution.java | 9 - .../dss/appjoint/execution/core/Killable.java | 28 - .../execution/core/LongTermNodeExecution.java | 94 -- .../appjoint/execution/core/NodeContext.java | 112 --- .../execution/core/NodeContextImpl.java | 94 -- .../appjoint/execution/core/ParamsNode.java | 28 - .../appjoint/execution/core/Procedure.java | 33 - .../execution/core/SchedulerManager.java | 38 - .../AsyncNodeExecutionResponseEvent.java | 51 - .../AsyncNodeExecutionSchedulerListener.java | 38 - .../LongTermNodeExecutionScheduler.java | 39 - .../dss/appjoint/service/AppJointUrl.java | 29 - .../dss/appjoint/service/AppJointUrlImpl.java | 36 - .../dss/appjoint/service/NodeService.java | 59 -- .../dss/appjoint/service/ProjectService.java | 43 - .../dss/appjoint/service/SecurityService.java | 32 - .../dss/appjoint/service/session/Session.java | 41 - .../conf/NodeExecutionConfiguration.scala | 36 - .../context/NodeExecutionContext.scala | 81 -- ...stenerEventBusNodeExecutionScheduler.scala | 125 --- dss-appjoint-loader/pom.xml | 71 -- .../clazzloader/AppJointClassLoader.java | 59 -- .../dss/appjoint/loader/AppJointLoader.java | 59 -- .../loader/AppJointLoaderFactory.java | 67 -- .../appjoint/loader/CommonAppJointLoader.java | 150 --- .../dss/appjoint/utils/AppJointUtils.java | 160 ---- .../dss/appjoint/utils/ExceptionHelper.java | 32 - .../exception/NoSuchAppJointException.scala | 26 - dss-application/pom.xml | 86 -- .../dss/application/conf/ApplicationConf.java | 52 -- .../application/dao/ApplicationMapper.java | 36 - .../dao/DSSApplicationUserMapper.java | 31 - .../dss/application/dao/LinkisUserMapper.java | 12 - .../dao/impl/ApplicationMapper.xml | 46 - .../dao/impl/DSSApplicationUserMapper.xml | 41 - .../application/dao/impl/LinkisUserMapper.xml | 34 - .../dss/application/entity/Application.java | 135 --- .../dss/application/entity/DSSUser.java | 68 -- .../dss/application/entity/DSSUserVO.java | 35 - .../dss/application/entity/LinkisUser.java | 144 --- .../application/entity/UserInitResult.java | 69 -- .../dss/application/entity/WorkSpacePath.java | 59 -- .../handler/ApplicationHandlerChain.java | 46 - .../dss/application/handler/Handler.java | 27 - .../handler/LinkisUserFirstLoginHandler.java | 63 -- .../dss/application/handler/Order.java | 27 - .../handler/ProjectUserInitHandler.java | 43 - .../handler/UserFirstLoginHandler.java | 62 -- .../restful/ApplicationRestfulApi.java | 104 --- .../service/ApplicationService.java | 43 - .../service/DSSApplicationUserService.java | 32 - .../service/LinkisUserService.java | 31 - .../service/impl/ApplicationServiceImpl.java | 110 --- .../impl/DSSApplicationUserServiceImpl.java | 49 - .../service/impl/LinkisUserServiceImpl.java | 60 -- .../application/util/ApplicationUtils.java | 85 -- dss-azkaban-scheduler-appjoint/pom.xml | 105 --- .../src/main/assembly/distribution.xml | 136 --- .../azkaban/AzkabanSchedulerAppJoint.java | 85 -- .../scheduler/azkaban/conf/AzkabanConf.java | 14 - .../azkaban/constant/AzkabanConstant.java | 17 - .../azkaban/entity/AzkabanSchedulerFlow.java | 17 - .../azkaban/entity/AzkabanSchedulerNode.java | 22 - .../entity/AzkabanSchedulerProject.java | 21 - .../azkaban/entity/LinkisAzkabanReadNode.java | 37 - .../entity/LinkisAzkabanSchedulerNode.java | 17 - .../entity/LinkisAzkabanShareNode.java | 35 - .../hooks/LinkisAzkabanFlowPublishHook.java | 113 --- .../hooks/LinkisAzkabanNodePublishHook.java | 79 -- .../LinkisAzkabanProjectPublishHook.java | 127 --- .../linkisjob/AzkabanSubFlowJobTuning.java | 23 - .../azkaban/linkisjob/LinkisJob.java | 74 -- .../azkaban/linkisjob/LinkisJobConverter.java | 112 --- .../azkaban/linkisjob/LinkisJobTuning.java | 11 - .../azkaban/parser/AzkabanFlowParser.java | 38 - .../azkaban/parser/AzkabanNodeParser.java | 17 - .../azkaban/parser/AzkabanProjectParser.java | 32 - .../parser/LinkisAzkabanNodeParser.java | 31 - .../LinkisAzkabanSendEmailNodeParser.java | 27 - .../service/AzkabanProjectService.java | 223 ----- .../service/AzkabanSecurityService.java | 160 ---- .../azkaban/service/AzkabanSession.java | 59 -- .../azkaban/tuning/AzkabanProjectTuning.java | 69 -- .../tuning/LinkisAzkabanFlowTuning.java | 98 -- .../tuning/LinkisShareNodeFlowTuning.java | 34 - .../scheduler/azkaban/util/AzkabanUtils.java | 34 - .../azkaban/util/AzkabanUtilsScala.scala | 17 - .../src/main/resources/appjoint.properties | 21 - .../src/main/resources/log4j.properties | 37 - .../src/main/resources/log4j2.xml | 39 - .../dss/common/entity/Resource.java | 62 -- .../dss/common/entity/flow/DSSFlow.java | 194 ---- .../entity/flow/DSSFlowPublishHistory.java | 24 - .../common/entity/flow/DSSFlowVersion.java | 163 ---- .../dss/common/entity/flow/DSSJSONFlow.java | 48 - .../common/entity/flow/FileFlowVersion.java | 27 - .../dss/common/entity/flow/Flow.java | 53 -- .../dss/common/entity/flow/FlowVersion.java | 31 - .../entity/flow/FlowVersionForNodes.java | 35 - .../common/entity/flow/JSONFlowVersion.java | 27 - .../dss/common/entity/node/DSSEdge.java | 40 - .../common/entity/node/DSSEdgeDefault.java | 78 -- .../dss/common/entity/node/DSSNode.java | 49 - .../common/entity/node/DSSNodeDefault.java | 162 ---- .../dss/common/entity/node/Layout.java | 70 -- .../dss/common/entity/node/Node.java | 46 - .../common/entity/project/DSSJSONProject.java | 41 - .../dss/common/entity/project/DSSProject.java | 294 ------ .../project/DSSProjectPublishHistory.java | 116 --- .../entity/project/DSSProjectVersion.java | 126 --- .../dss/common/entity/project/Project.java | 47 - .../common/entity/project/ProjectVersion.java | 31 - .../project/ProjectVersionForFlows.java | 36 - .../common/exception/DSSErrorException.java | 34 - .../common/exception/DSSRuntimeException.java | 30 - .../common/exception/ThrowingConsumer.java | 26 - .../common/exception/ThrowingFunction.java | 26 - .../dss/common/utils/DSSExceptionUtils.java | 59 -- .../dss/common/utils/FileHelper.java | 33 - .../dss/common/utils/ZipHelper.java | 126 --- .../common/protocol/RequestDSSProject.scala | 25 - .../bin/start-dss-flow-execution-entrance.sh | 49 - .../bin/stop-dss-flow-execution-entrance.sh | 47 - dss-flow-execution-entrance/pom.xml | 126 --- .../src/main/assembly/distribution.xml | 303 ------ .../entrance/entity/FlowExecutionCode.java | 70 -- .../entrance/entity/FlowExecutionNode.java | 27 - .../entrance/entity/FlowExecutonReadNode.java | 51 - .../entity/FlowExecutonSharedNode.java | 51 - .../job/FlowExecutionAppJointLinkisJob.java | 48 - .../FlowExecutionAppJointLinkisReadJob.java | 62 -- .../FlowExecutionAppJointLinkisSharedJob.java | 49 - .../FlowExecutionAppJointSignalSharedJob.java | 56 -- .../job/FlowExecutionCommonLinkisJob.java | 54 -- .../job/FlowExecutionCommonLinkisReadJob.java | 62 -- .../FlowExecutionCommonLinkisSharedJob.java | 47 - .../job/FlowExecutionJobSignalKeyCreator.java | 39 - .../parser/FlowExecutionFlowParser.java | 44 - .../parser/FlowExecutionNodeParser.java | 50 - .../parser/FlowExecutionProjectParser.java | 43 - .../parser/FlowExecutionReadNodeParser.java | 55 -- .../restful/FlowExecutionRestfulApi.java | 95 -- .../tuning/FlowExecutionFlowTuning.java | 58 -- .../tuning/FlowExecutionProjectTuning.java | 37 - .../src/main/resources/application.yml | 23 - .../src/main/resources/linkis.properties | 42 - .../src/main/resources/log4j.properties | 37 - .../src/main/resources/log4j2.xml | 39 - .../flow/execution/entrance/FlowContext.scala | 86 -- .../execution/entrance/FlowContextImpl.scala | 65 -- .../FlowExecutionEntranceConfiguration.scala | 95 -- ...ExecutionEntranceSpringConfiguration.scala | 60 -- .../entrance/engine/FlowEntranceEngine.scala | 123 --- .../FlowExecutionExecutorManagerImpl.scala | 78 -- .../entranceparser/FlowExecutionParser.scala | 50 - .../FlowExecutionErrorException.scala | 32 - .../execution/DefaultFlowExecution.scala | 85 -- .../entrance/execution/FlowExecution.scala | 29 - .../execution/NodeExecutionStatusPoller.scala | 44 - .../entrance/job/FlowEntranceJob.scala | 179 ---- .../entrance/job/FlowExecutionRequest.scala | 28 - .../job/parser/FlowEntranceJobParser.scala | 29 - .../job/parser/FlowJobFlowParser.scala | 84 -- .../job/parser/FlowJobNodeParser.scala | 99 -- .../listener/NodeRunnerListener.scala | 29 - .../entrance/log/FlowExecutionLog.scala | 40 - .../entrance/node/AppJointJobBuilder.scala | 180 ---- .../entrance/node/DefaultNodeRunner.scala | 164 ---- .../entrance/node/NodeExecutionState.scala | 44 - .../execution/entrance/node/NodeRunner.scala | 105 --- .../resolver/FlowDependencyResolver.scala | 28 - .../resolver/FlowDependencyResolverImpl.scala | 66 -- .../entrance/utils/FlowExecutionUtils.scala | 92 -- dss-linkis-node-execution/pom.xml | 69 -- .../node/execution/WorkflowContext.java | 44 - .../execution/WorkflowContextFactory.java | 59 -- .../node/execution/WorkflowContextImpl.java | 96 -- .../conf/LinkisJobExecutionConfiguration.java | 101 -- .../node/execution/entity/BMLResource.java | 52 -- .../node/execution/entity/ContextInfo.java | 52 -- .../LinkisJobExecutionErrorException.java | 30 - .../LinkisJobExecutionWarnException.java | 35 - .../execution/LinkisNodeExecution.java | 46 - .../impl/LinkisNodeExecutionImpl.java | 203 ---- .../job/AbstractAppJointLinkisJob.java | 198 ---- .../job/AbstractCommonLinkisJob.java | 240 ----- .../node/execution/job/AppJointLinkisJob.java | 24 - .../linkis/node/execution/job/Builder.java | 150 --- .../node/execution/job/CommonLinkisJob.java | 42 - .../dss/linkis/node/execution/job/Job.java | 68 -- .../execution/job/JobSignalKeyCreator.java | 25 - .../node/execution/job/JobTypeEnum.java | 25 - .../linkis/node/execution/job/LinkisJob.java | 46 - .../linkis/node/execution/job/ReadJob.java | 35 - .../linkis/node/execution/job/RuntimeJob.java | 24 - .../linkis/node/execution/job/SharedJob.java | 29 - .../node/execution/job/SignalSharedJob.java | 42 - .../listener/LinkisExecutionListener.java | 28 - .../execution/log/LinkisJobExecutionLog.java | 41 - .../node/execution/parser/CodeParser.java | 235 ----- .../execution/parser/JobParamsParser.java | 89 -- .../node/execution/parser/JobParser.java | 30 - .../parser/JobRuntimeParamsParser.java | 41 - .../execution/service/BuildJobAction.java | 32 - .../service/impl/BuildJobActionImpl.java | 65 -- .../utils/LinkisJobExecutionUtils.java | 59 -- .../utils/LinkisUjesClientUtils.java | 55 -- dss-scheduler-appjoint-core/pom.xml | 76 -- .../appjoint/scheduler/SchedulerAppJoint.java | 44 - .../constant/SchedulerAppJointConstant.java | 30 - .../entity/AbstractSchedulerNode.java | 90 -- .../entity/AbstractSchedulerProject.java | 112 --- .../scheduler/entity/FileProjectVersion.java | 32 - .../appjoint/scheduler/entity/ReadNode.java | 28 - .../scheduler/entity/SchedulerEdge.java | 29 - .../entity/SchedulerEdgeDefault.java | 38 - .../scheduler/entity/SchedulerFlow.java | 161 ---- .../scheduler/entity/SchedulerNode.java | 31 - .../scheduler/entity/SchedulerProject.java | 29 - .../SchedulerProjectVersionForFlows.java | 73 -- .../appjoint/scheduler/entity/ShareNode.java | 28 - .../hooks/AbstractFlowPublishHook.java | 64 -- .../hooks/AbstractNodePublishHook.java | 36 - .../hooks/AbstractProjectPublishHook.java | 64 -- .../scheduler/hooks/FlowPublishHook.java | 32 - .../scheduler/hooks/NodePublishHook.java | 30 - .../scheduler/hooks/ProjectPublishHook.java | 31 - .../dss/appjoint/scheduler/order/Order.java | 28 - .../scheduler/parser/AbstractFlowParser.java | 117 --- .../scheduler/parser/AbstractNodeParser.java | 40 - .../parser/AbstractProjectParser.java | 105 --- .../parser/AbstractReadNodeParser.java | 77 -- .../scheduler/parser/ContextNodeParser.java | 30 - .../appjoint/scheduler/parser/FlowParser.java | 36 - .../appjoint/scheduler/parser/NodeParser.java | 31 - .../scheduler/parser/ProjectParser.java | 31 - .../scheduler/parser/SendEmailNodeParser.java | 48 - .../service/SchedulerProjectService.java | 31 - .../service/SchedulerSecurityService.java | 27 - .../scheduler/tuning/AbstractFlowTuning.java | 120 --- .../scheduler/tuning/AbstractNodeTuning.java | 30 - .../tuning/AbstractProjectTuning.java | 70 -- .../tuning/AbstractShareNodeFlowTuning.java | 96 -- .../scheduler/tuning/ContextNodeTuning.java | 28 - .../tuning/ContextShareableFlowTuning.java | 34 - .../appjoint/scheduler/tuning/FlowTuning.java | 31 - .../appjoint/scheduler/tuning/NodeTuning.java | 28 - .../scheduler/tuning/ProjectTuning.java | 29 - dss-server/bin/start-dss-server.sh | 39 - dss-server/bin/stop-dss-server.sh | 47 - dss-server/pom.xml | 237 ----- dss-server/src/main/assembly/distribution.xml | 340 ------- .../dss/DSSSpringApplication.java | 222 ----- .../dss/server/conf/DSSServerConf.java | 49 - .../server/constant/DSSServerConstant.java | 33 - .../dss/server/dao/DSSUserMapper.java | 32 - .../dss/server/dao/FlowMapper.java | 71 -- .../dss/server/dao/FlowTaxonomyMapper.java | 51 - .../dss/server/dao/NodeInfoMapper.java | 35 - .../dss/server/dao/ProjectMapper.java | 66 -- .../dss/server/dao/ProjectTaxonomyMapper.java | 49 - .../dss/server/dao/WorkspaceMapper.java | 68 -- .../dss/server/dao/impl/dwsUserMapper.xml | 31 - .../dss/server/dao/impl/flowMapper.xml | 256 ------ .../server/dao/impl/flowTaxonomyMapper.xml | 95 -- .../dss/server/dao/impl/nodeInfoMapper.xml | 52 -- .../dss/server/dao/impl/projectMapper.xml | 177 ---- .../server/dao/impl/projectTaxonomyMapper.xml | 97 -- .../dss/server/dao/impl/workspaceMapper.xml | 200 ---- .../dss/server/db/DSSMybatisConfig.java | 148 --- .../dto/response/HomepageDemoInstanceVo.java | 96 -- .../dto/response/HomepageDemoMenuVo.java | 89 -- .../server/dto/response/HomepageVideoVo.java | 78 -- .../response/OnestopMenuAppInstanceVo.java | 140 --- .../server/dto/response/OnestopMenuVo.java | 62 -- .../dto/response/WorkspaceDepartmentVo.java | 44 - .../dto/response/WorkspaceFavoriteVo.java | 82 -- .../dss/server/entity/ApplicationArea.java | 53 -- .../dss/server/entity/BaseEntity.java | 49 - .../dss/server/entity/DSSFavorite.java | 73 -- .../dss/server/entity/DSSFlowTaxonomy.java | 99 -- .../dss/server/entity/DSSProjectTaxonomy.java | 90 -- .../entity/DSSProjectTaxonomyRelation.java | 49 - .../dss/server/entity/DSSWorkspace.java | 93 -- .../dss/server/entity/NodeInfo.java | 103 --- .../dss/server/function/FunctionInvoker.java | 104 --- .../dss/server/function/FunctionPool.java | 39 - .../server/function/NodeServiceFunction.java | 30 - .../function/ProjectServiceAddFunction.java | 28 - .../function/ProjectServiceFunction.java | 28 - .../wedatasphere/dss/server/lock/Lock.java | 30 - .../dss/server/lock/LockAspect.java | 110 --- .../dss/server/lock/LockEnum.java | 24 - .../server/operate/AppJointNodeOperate.java | 66 -- .../wedatasphere/dss/server/operate/Op.java | 60 -- .../dss/server/operate/Operate.java | 36 - .../dss/server/operate/SubFlowOperate.java | 73 -- .../dss/server/publish/PublishJob.java | 45 - .../dss/server/publish/PublishJobEnum.java | 23 - .../dss/server/publish/PublishJobFactory.java | 47 - .../dss/server/publish/PublishListner.java | 28 - .../dss/server/publish/PublishManager.java | 166 ---- .../server/publish/PublishProjectCache.java | 81 -- .../dss/server/publish/PublishSubmitJob.java | 50 - .../publish/PublishSubmitJobDeamon.java | 48 - .../dss/server/publish/PublishThreadPool.java | 37 - .../dss/server/restful/FlowRestfulApi.java | 180 ---- .../restful/FlowTaxonomyRestfulApi.java | 113 --- .../dss/server/restful/NodeRestfulApi.java | 115 --- .../dss/server/restful/ProjectRestfulApi.java | 201 ---- .../restful/ProjectTaxonomyRestfulApi.java | 98 -- .../dss/server/restful/TreeRestfulApi.java | 59 -- .../dss/server/restful/UserManagerApi.java | 92 -- .../server/restful/WorkspaceRestfulApi.java | 172 ---- .../dss/server/service/DSSFlowService.java | 74 -- .../service/DSSFlowTaxonomyService.java | 41 - .../server/service/DSSNodeInfoService.java | 33 - .../dss/server/service/DSSProjectService.java | 67 -- .../service/DSSProjectTaxonomyService.java | 46 - .../dss/server/service/DSSUserService.java | 23 - .../server/service/DSSWorkspaceService.java | 63 -- .../service/impl/DSSFlowServiceImpl.java | 282 ------ .../impl/DSSFlowTaxonomyServiceImpl.java | 138 --- .../service/impl/DSSNodeInfoServiceImpl.java | 40 - .../service/impl/DSSProjectServiceImpl.java | 609 ------------ .../impl/DSSProjectTaxonomyServiceImpl.java | 132 --- .../service/impl/DSSUserServiceImpl.java | 37 - .../service/impl/DSSWorkspaceServiceImpl.java | 155 ---- .../dss/server/util/ThreadPoolTool.java | 132 --- dss-server/src/main/resources/application.yml | 22 - .../resources/default/AddschedulerUser.sh | 13 - .../resources/default/CreateLdapAccount.sh | 18 - .../main/resources/default/CreateLinuxUser.sh | 35 - .../src/main/resources/default/HdfsPath.sh | 6 - .../src/main/resources/default/LinuxPath.sh | 7 - .../src/main/resources/linkis.properties | 71 -- .../src/main/resources/log4j.properties | 37 - dss-server/src/main/resources/log4j2.xml | 39 - .../src/main/resources/token.properties | 2 - .../dss/server/crumb/CrumbFactory.scala | 144 --- .../dss/server/crumb/QuerParamsParser.scala | 80 -- .../dss/server/entity/Crumb.scala | 28 - .../dss/server/entity/CrumbType.scala | 24 - .../server/receiver/DSSServerReceiver.scala | 49 - .../dss/server/service/BMLService.scala | 111 --- dss-server/src/main/test/HttpTest.java | 185 ---- dss-server/src/main/test/com/webank/Adf.scala | 9 - .../src/main/test/com/webank/HttpTest.java | 141 --- .../src/main/test/com/webank/JobParam.java | 11 - .../src/main/test/com/webank/ReqTest.scala | 9 - .../src/main/test/com/webank/TestUnit.java | 14 - dss-user-manager/pom.xml | 87 -- .../dss/user/conf/DSSUserManagerConfig.java | 60 -- .../user/dto/request/AuthorizationBody.java | 99 -- .../dss/user/dto/request/LinuxServer.java | 53 -- .../dss/user/service/AbsCommand.java | 114 --- .../dss/user/service/Command.java | 58 -- .../dss/user/service/MacroCommand.java | 33 - .../dss/user/service/impl/AzkabanCommand.java | 110 --- .../user/service/impl/KerberosCommand.java | 69 -- .../dss/user/service/impl/LdapCommand.java | 58 -- .../user/service/impl/LinuxUserCommand.java | 60 -- .../user/service/impl/MetastoreCommand.java | 56 -- .../user/service/impl/SchedulisCommand.java | 146 --- .../service/impl/UserAuthorizationClient.java | 57 -- .../user/service/impl/UserMacroCommand.java | 118 --- .../user/service/impl/WorkspaceCommand.java | 70 -- .../dss/user/service/impl/YarnCommand.java | 39 - .../src/main/resources/config/properties.conf | 14 - .../resources/default/AddschedulerUser.sh | 18 - .../resources/default/CreateLdapAccount.sh | 7 - .../main/resources/default/CreateLinuxUser.sh | 34 - .../src/main/resources/default/HdfsPath.sh | 13 - .../src/main/resources/default/Kerberos.sh | 146 --- .../src/main/resources/default/LinuxPath.sh | 12 - .../src/main/resources/default/Metastore.sh | 44 - .../src/main/resources/read_anlexander.txt | 20 - .../service/impl/KerberosCommandTest.java | 43 - .../service/impl/MetastoreCommandTest.java | 43 - .../service/impl/WorkspaceCommandTest.java | 46 - eventchecker-appjoint/pom.xml | 139 --- .../src/main/assembly/distribution.xml | 136 --- .../jobtype/adapter/EventCheckAdapter.java | 37 - .../jobtype/connector/EventDruidFactory.java | 113 --- .../jobtype/entity/EventChecker.java | 277 ------ .../exception/UndefinedPropertyException.java | 31 - .../jobtype/service/AbstractEventCheck.java | 207 ----- .../service/AbstractEventCheckReceiver.java | 225 ----- .../service/DefaultEventcheckReceiver.java | 151 --- .../jobtype/service/EventCheckSender.java | 75 -- .../jobtype/service/EventCheckerService.java | 72 -- .../schedulis/jobtype/utils/Props.java | 869 ------------------ .../schedulis/jobtype/utils/Utils.java | 35 - .../src/main/resources/appjoint.properties | 24 - .../src/main/resources/log4j.properties | 37 - .../src/main/resources/log4j2.xml | 39 - .../execution/EventCheckerAppJoint.scala | 51 - .../execution/EventCheckerNodeExecution.scala | 165 ---- .../EventCheckerNodeExecutionAction.scala | 66 -- .../bin/start-linkis-appjoint-entrance.sh | 37 - .../bin/stop-linkis-appjoint-entrance.sh | 47 - .../conf/application.yml | 22 - .../conf/linkis.properties | 38 - .../linkis-appjoint-entrance/conf/log4j2.xml | 39 - .../linkis/linkis-appjoint-entrance/pom.xml | 141 --- .../src/main/assembly/distribution.xml | 335 ------- .../entrance/AppJointEntranceRestfulApi.java | 96 -- .../entrance/appjoint/AppJointManager.scala | 75 -- .../entrance/conf/AppJointConst.scala | 39 - .../conf/AppJointEntranceConfiguration.scala | 40 - .../AppJointEntranceSpringConfiguration.scala | 53 -- ...ppJointEntranceIllegalParamException.scala | 28 - .../OperateNotSupportedException.scala | 26 - .../exception/RPCFailedException.scala | 27 - .../AppJointEngineExecutorManagerImpl.scala | 70 -- .../execute/AppJointEntranceEngine.scala | 228 ----- .../execute/AppJointNodeContextImpl.scala | 239 ----- .../entrance/job/AppJointEntranceJob.scala | 100 -- .../entrance/job/AppJointExecuteRequest.scala | 33 - .../parser/AppJointEntranceParser.scala | 95 -- pom.xml | 172 +++- qualitis-appjoint/appjoint/pom.xml | 126 --- .../src/main/assembly/distribution.xml | 67 -- .../com.webank/wedatasphere/Md5Utils.java | 27 - .../appjoint/QualitisAppJoint.java | 56 -- .../appjoint/QualitisNodeExecution.java | 326 ------- .../appjoint/QualitisNodeExecutionAction.java | 19 - .../appjoint/QualitisSubmitRequest.java | 55 -- .../appjoint/QualitisTaskStatusEnum.java | 43 - .../project/QualitisAddProjectRequest.java | 41 - .../project/QualitisDeleteProjectRequest.java | 32 - .../project/QualitisProjectServiceImpl.java | 261 ------ .../project/QualitisUpdateProjectRequest.java | 51 - .../src/main/resources/appjoint.properties | 19 - .../src/main/resources/log4j.properties | 37 - .../appjoint/src/main/resources/log4j2.xml | 39 - sendemail-appjoint/sendemail-core/pom.xml | 129 --- .../dss/appjoint/sendemail/Email.java | 45 - .../dss/appjoint/sendemail/EmailContent.java | 29 - .../SendEmailAppJointSpringConfiguration.java | 80 -- .../exception/EmailSendFailedException.java | 29 - .../sendemail/sender/AbstractEmailSender.java | 40 - .../sender/SpringJavaEmailSender.java | 99 -- .../dss/appjoint/sendemail/Attachment.scala | 32 - .../sendemail/EmailContentGenerator.scala | 27 - .../sendemail/EmailContentParser.scala | 27 - .../appjoint/sendemail/EmailGenerator.scala | 29 - .../dss/appjoint/sendemail/EmailSender.scala | 35 - .../sendemail/SendEmailAppJoint.scala | 51 - .../sendemail/SendEmailNodeExecution.scala | 85 -- .../SendEmailNodeExecutionHook.scala | 31 - .../conf/SendEmailAppJointConfiguration.scala | 43 - .../conf/SendEmailAppJointUtils.scala | 33 - .../sendemail/email/AbstractEmail.scala | 60 -- .../sendemail/email/MultiContentEmail.scala | 38 - .../sendemail/email/PngAttachment.scala | 38 - .../generate/AbstractEmailGenerator.scala | 61 -- .../generate/MultiContentEmailGenerator.scala | 63 -- .../emailcontent/ArrayEmailContent.scala | 34 - .../FsPathStoreEmailContent.scala | 30 - .../emailcontent/StringEmailContent.scala | 32 - .../AbstractEmailContentGenerator.scala | 64 -- .../MultiEmailContentGenerator.scala | 56 -- .../sendemail/emailcontent/package.scala | 86 -- .../parser/AbstractEmailContentParser.scala | 71 -- .../parser/FileEmailContentParser.scala | 44 - .../parser/HtmlEmailContentParser.scala | 33 - .../parser/PictureEmailContentParser.scala | 74 -- .../parser/TableEmailContentParser.scala | 69 -- visualis-appjoint/appjoint/pom.xml | 130 --- .../src/main/assembly/distribution.xml | 226 ----- .../appjoint/visualis/VisualisAppJoint.java | 71 -- .../visualis/service/VisualisNodeService.java | 70 -- .../service/VisualisSecurityService.java | 42 - .../visualis/service/VisualisSession.java | 74 -- .../nodeservice/DashboardNodeService.java | 151 --- .../nodeservice/DisplayNodeService.java | 125 --- .../service/nodeservice/HttpUtils.java | 153 --- .../execution/VisualisNodeExecution.scala | 142 --- .../VisualisNodeExecutionConfiguration.scala | 35 - .../linkis/server/JSONUtils.scala | 27 - 524 files changed, 121 insertions(+), 38246 deletions(-) delete mode 100644 bin/checkEnv.sh delete mode 100644 bin/checkServices.sh delete mode 100644 bin/install.sh delete mode 100644 bin/start-all.sh delete mode 100644 bin/stop-all.sh delete mode 100644 datachecker-appjoint/pom.xml delete mode 100644 datachecker-appjoint/src/main/assembly/distribution.xml delete mode 100644 datachecker-appjoint/src/main/java/com/webank/wedatasphere/dss/appjoint/schedulis/jobtype/connector/DataCheckerDao.java delete mode 100644 datachecker-appjoint/src/main/java/com/webank/wedatasphere/dss/appjoint/schedulis/jobtype/connector/DataDruidFactory.java delete mode 100644 datachecker-appjoint/src/main/java/com/webank/wedatasphere/dss/appjoint/schedulis/jobtype/entity/DataChecker.java delete mode 100644 datachecker-appjoint/src/main/resources/appjoint.properties delete mode 100644 datachecker-appjoint/src/main/resources/log4j.properties delete mode 100644 datachecker-appjoint/src/main/resources/log4j2.xml delete mode 100644 datachecker-appjoint/src/main/scala/com/webank/wedatasphere/dss/appjoint/schedulis/jobtype/execution/DataCheckerAppJoint.scala delete mode 100644 datachecker-appjoint/src/main/scala/com/webank/wedatasphere/dss/appjoint/schedulis/jobtype/execution/DataCheckerExecution.scala delete mode 100644 datachecker-appjoint/src/main/scala/com/webank/wedatasphere/dss/appjoint/schedulis/jobtype/execution/DataCheckerNodeExecutionAction.scala delete mode 100644 dss-appjoint-auth/pom.xml delete mode 100644 dss-appjoint-auth/src/main/java/com/webank/wedatasphere/dss/appjoint/auth/AppJointAuth.java delete mode 100644 dss-appjoint-auth/src/main/java/com/webank/wedatasphere/dss/appjoint/auth/RedirectMsg.java delete mode 100644 dss-appjoint-auth/src/main/scala/com/webank/wedatasphere/dss/appjoint/auth/impl/AppJointAuthImpl.scala delete mode 100644 dss-appjoint-auth/src/main/scala/com/webank/wedatasphere/dss/appjoint/auth/impl/RedirectMsgImpl.scala delete mode 100644 dss-appjoint-auth/src/main/scala/com/webank/wedatasphere/dss/appjoint/auth/impl/UserInfoAction.scala delete mode 100644 dss-appjoint-auth/src/main/scala/com/webank/wedatasphere/dss/appjoint/auth/impl/UserInfoResult.scala delete mode 100644 dss-appjoint-core/pom.xml delete mode 100644 dss-appjoint-core/src/main/java/com/webank/wedatasphere/dss/appjoint/AppJoint.java delete mode 100644 dss-appjoint-core/src/main/java/com/webank/wedatasphere/dss/appjoint/exception/AppJointErrorException.java delete mode 100644 dss-appjoint-core/src/main/java/com/webank/wedatasphere/dss/appjoint/execution/NodeExecution.java delete mode 100644 dss-appjoint-core/src/main/java/com/webank/wedatasphere/dss/appjoint/execution/async/NodeExecutionListener.java delete mode 100644 dss-appjoint-core/src/main/java/com/webank/wedatasphere/dss/appjoint/execution/async/NodeExecutionLogListener.java delete mode 100644 dss-appjoint-core/src/main/java/com/webank/wedatasphere/dss/appjoint/execution/async/NodeExecutionProgressListener.java delete mode 100644 dss-appjoint-core/src/main/java/com/webank/wedatasphere/dss/appjoint/execution/async/NodeExecutionResponseListener.java delete mode 100644 dss-appjoint-core/src/main/java/com/webank/wedatasphere/dss/appjoint/execution/async/NodeExecutionStatusListener.java delete mode 100644 dss-appjoint-core/src/main/java/com/webank/wedatasphere/dss/appjoint/execution/common/AbstractNodeExecutionAction.java delete mode 100644 dss-appjoint-core/src/main/java/com/webank/wedatasphere/dss/appjoint/execution/common/AsyncNodeExecutionResponse.java delete mode 100644 dss-appjoint-core/src/main/java/com/webank/wedatasphere/dss/appjoint/execution/common/CompletedNodeExecutionResponse.java delete mode 100644 dss-appjoint-core/src/main/java/com/webank/wedatasphere/dss/appjoint/execution/common/LongTermNodeExecutionAction.java delete mode 100644 dss-appjoint-core/src/main/java/com/webank/wedatasphere/dss/appjoint/execution/common/NodeExecutionAction.java delete mode 100644 dss-appjoint-core/src/main/java/com/webank/wedatasphere/dss/appjoint/execution/common/NodeExecutionResponse.java delete mode 100644 dss-appjoint-core/src/main/java/com/webank/wedatasphere/dss/appjoint/execution/common/NodeExecutionState.java delete mode 100644 dss-appjoint-core/src/main/java/com/webank/wedatasphere/dss/appjoint/execution/core/AbstractAppJointNode.java delete mode 100644 dss-appjoint-core/src/main/java/com/webank/wedatasphere/dss/appjoint/execution/core/AppJointNode.java delete mode 100644 dss-appjoint-core/src/main/java/com/webank/wedatasphere/dss/appjoint/execution/core/CallbackLongTermNodeExecution.java delete mode 100644 dss-appjoint-core/src/main/java/com/webank/wedatasphere/dss/appjoint/execution/core/CommonAppJointNode.java delete mode 100644 dss-appjoint-core/src/main/java/com/webank/wedatasphere/dss/appjoint/execution/core/HttpLongTermNodeExecution.java delete mode 100644 dss-appjoint-core/src/main/java/com/webank/wedatasphere/dss/appjoint/execution/core/Killable.java delete mode 100644 dss-appjoint-core/src/main/java/com/webank/wedatasphere/dss/appjoint/execution/core/LongTermNodeExecution.java delete mode 100644 dss-appjoint-core/src/main/java/com/webank/wedatasphere/dss/appjoint/execution/core/NodeContext.java delete mode 100644 dss-appjoint-core/src/main/java/com/webank/wedatasphere/dss/appjoint/execution/core/NodeContextImpl.java delete mode 100644 dss-appjoint-core/src/main/java/com/webank/wedatasphere/dss/appjoint/execution/core/ParamsNode.java delete mode 100644 dss-appjoint-core/src/main/java/com/webank/wedatasphere/dss/appjoint/execution/core/Procedure.java delete mode 100644 dss-appjoint-core/src/main/java/com/webank/wedatasphere/dss/appjoint/execution/core/SchedulerManager.java delete mode 100644 dss-appjoint-core/src/main/java/com/webank/wedatasphere/dss/appjoint/execution/scheduler/AsyncNodeExecutionResponseEvent.java delete mode 100644 dss-appjoint-core/src/main/java/com/webank/wedatasphere/dss/appjoint/execution/scheduler/AsyncNodeExecutionSchedulerListener.java delete mode 100644 dss-appjoint-core/src/main/java/com/webank/wedatasphere/dss/appjoint/execution/scheduler/LongTermNodeExecutionScheduler.java delete mode 100644 dss-appjoint-core/src/main/java/com/webank/wedatasphere/dss/appjoint/service/AppJointUrl.java delete mode 100644 dss-appjoint-core/src/main/java/com/webank/wedatasphere/dss/appjoint/service/AppJointUrlImpl.java delete mode 100644 dss-appjoint-core/src/main/java/com/webank/wedatasphere/dss/appjoint/service/NodeService.java delete mode 100644 dss-appjoint-core/src/main/java/com/webank/wedatasphere/dss/appjoint/service/ProjectService.java delete mode 100644 dss-appjoint-core/src/main/java/com/webank/wedatasphere/dss/appjoint/service/SecurityService.java delete mode 100644 dss-appjoint-core/src/main/java/com/webank/wedatasphere/dss/appjoint/service/session/Session.java delete mode 100644 dss-appjoint-core/src/main/scala/com/webank/wedatasphere/dss/appjoint/execution/conf/NodeExecutionConfiguration.scala delete mode 100644 dss-appjoint-core/src/main/scala/com/webank/wedatasphere/dss/appjoint/execution/context/NodeExecutionContext.scala delete mode 100644 dss-appjoint-core/src/main/scala/com/webank/wedatasphere/dss/appjoint/execution/scheduler/ListenerEventBusNodeExecutionScheduler.scala delete mode 100644 dss-appjoint-loader/pom.xml delete mode 100644 dss-appjoint-loader/src/main/java/com/webank/wedatasphere/dss/appjoint/clazzloader/AppJointClassLoader.java delete mode 100644 dss-appjoint-loader/src/main/java/com/webank/wedatasphere/dss/appjoint/loader/AppJointLoader.java delete mode 100644 dss-appjoint-loader/src/main/java/com/webank/wedatasphere/dss/appjoint/loader/AppJointLoaderFactory.java delete mode 100644 dss-appjoint-loader/src/main/java/com/webank/wedatasphere/dss/appjoint/loader/CommonAppJointLoader.java delete mode 100644 dss-appjoint-loader/src/main/java/com/webank/wedatasphere/dss/appjoint/utils/AppJointUtils.java delete mode 100644 dss-appjoint-loader/src/main/java/com/webank/wedatasphere/dss/appjoint/utils/ExceptionHelper.java delete mode 100644 dss-appjoint-loader/src/main/scala/com/webank/wedatasphere/dss/appjoint/exception/NoSuchAppJointException.scala delete mode 100644 dss-application/pom.xml delete mode 100644 dss-application/src/main/java/com/webank/wedatasphere/dss/application/conf/ApplicationConf.java delete mode 100644 dss-application/src/main/java/com/webank/wedatasphere/dss/application/dao/ApplicationMapper.java delete mode 100644 dss-application/src/main/java/com/webank/wedatasphere/dss/application/dao/DSSApplicationUserMapper.java delete mode 100644 dss-application/src/main/java/com/webank/wedatasphere/dss/application/dao/LinkisUserMapper.java delete mode 100644 dss-application/src/main/java/com/webank/wedatasphere/dss/application/dao/impl/ApplicationMapper.xml delete mode 100644 dss-application/src/main/java/com/webank/wedatasphere/dss/application/dao/impl/DSSApplicationUserMapper.xml delete mode 100644 dss-application/src/main/java/com/webank/wedatasphere/dss/application/dao/impl/LinkisUserMapper.xml delete mode 100644 dss-application/src/main/java/com/webank/wedatasphere/dss/application/entity/Application.java delete mode 100644 dss-application/src/main/java/com/webank/wedatasphere/dss/application/entity/DSSUser.java delete mode 100644 dss-application/src/main/java/com/webank/wedatasphere/dss/application/entity/DSSUserVO.java delete mode 100644 dss-application/src/main/java/com/webank/wedatasphere/dss/application/entity/LinkisUser.java delete mode 100644 dss-application/src/main/java/com/webank/wedatasphere/dss/application/entity/UserInitResult.java delete mode 100644 dss-application/src/main/java/com/webank/wedatasphere/dss/application/entity/WorkSpacePath.java delete mode 100644 dss-application/src/main/java/com/webank/wedatasphere/dss/application/handler/ApplicationHandlerChain.java delete mode 100644 dss-application/src/main/java/com/webank/wedatasphere/dss/application/handler/Handler.java delete mode 100644 dss-application/src/main/java/com/webank/wedatasphere/dss/application/handler/LinkisUserFirstLoginHandler.java delete mode 100644 dss-application/src/main/java/com/webank/wedatasphere/dss/application/handler/Order.java delete mode 100644 dss-application/src/main/java/com/webank/wedatasphere/dss/application/handler/ProjectUserInitHandler.java delete mode 100644 dss-application/src/main/java/com/webank/wedatasphere/dss/application/handler/UserFirstLoginHandler.java delete mode 100644 dss-application/src/main/java/com/webank/wedatasphere/dss/application/restful/ApplicationRestfulApi.java delete mode 100644 dss-application/src/main/java/com/webank/wedatasphere/dss/application/service/ApplicationService.java delete mode 100644 dss-application/src/main/java/com/webank/wedatasphere/dss/application/service/DSSApplicationUserService.java delete mode 100644 dss-application/src/main/java/com/webank/wedatasphere/dss/application/service/LinkisUserService.java delete mode 100644 dss-application/src/main/java/com/webank/wedatasphere/dss/application/service/impl/ApplicationServiceImpl.java delete mode 100644 dss-application/src/main/java/com/webank/wedatasphere/dss/application/service/impl/DSSApplicationUserServiceImpl.java delete mode 100644 dss-application/src/main/java/com/webank/wedatasphere/dss/application/service/impl/LinkisUserServiceImpl.java delete mode 100644 dss-application/src/main/java/com/webank/wedatasphere/dss/application/util/ApplicationUtils.java delete mode 100644 dss-azkaban-scheduler-appjoint/pom.xml delete mode 100644 dss-azkaban-scheduler-appjoint/src/main/assembly/distribution.xml delete mode 100644 dss-azkaban-scheduler-appjoint/src/main/java/com/webank/wedatasphere/dss/appjoint/scheduler/azkaban/AzkabanSchedulerAppJoint.java delete mode 100644 dss-azkaban-scheduler-appjoint/src/main/java/com/webank/wedatasphere/dss/appjoint/scheduler/azkaban/conf/AzkabanConf.java delete mode 100644 dss-azkaban-scheduler-appjoint/src/main/java/com/webank/wedatasphere/dss/appjoint/scheduler/azkaban/constant/AzkabanConstant.java delete mode 100644 dss-azkaban-scheduler-appjoint/src/main/java/com/webank/wedatasphere/dss/appjoint/scheduler/azkaban/entity/AzkabanSchedulerFlow.java delete mode 100644 dss-azkaban-scheduler-appjoint/src/main/java/com/webank/wedatasphere/dss/appjoint/scheduler/azkaban/entity/AzkabanSchedulerNode.java delete mode 100644 dss-azkaban-scheduler-appjoint/src/main/java/com/webank/wedatasphere/dss/appjoint/scheduler/azkaban/entity/AzkabanSchedulerProject.java delete mode 100644 dss-azkaban-scheduler-appjoint/src/main/java/com/webank/wedatasphere/dss/appjoint/scheduler/azkaban/entity/LinkisAzkabanReadNode.java delete mode 100644 dss-azkaban-scheduler-appjoint/src/main/java/com/webank/wedatasphere/dss/appjoint/scheduler/azkaban/entity/LinkisAzkabanSchedulerNode.java delete mode 100644 dss-azkaban-scheduler-appjoint/src/main/java/com/webank/wedatasphere/dss/appjoint/scheduler/azkaban/entity/LinkisAzkabanShareNode.java delete mode 100644 dss-azkaban-scheduler-appjoint/src/main/java/com/webank/wedatasphere/dss/appjoint/scheduler/azkaban/hooks/LinkisAzkabanFlowPublishHook.java delete mode 100644 dss-azkaban-scheduler-appjoint/src/main/java/com/webank/wedatasphere/dss/appjoint/scheduler/azkaban/hooks/LinkisAzkabanNodePublishHook.java delete mode 100644 dss-azkaban-scheduler-appjoint/src/main/java/com/webank/wedatasphere/dss/appjoint/scheduler/azkaban/hooks/LinkisAzkabanProjectPublishHook.java delete mode 100644 dss-azkaban-scheduler-appjoint/src/main/java/com/webank/wedatasphere/dss/appjoint/scheduler/azkaban/linkisjob/AzkabanSubFlowJobTuning.java delete mode 100644 dss-azkaban-scheduler-appjoint/src/main/java/com/webank/wedatasphere/dss/appjoint/scheduler/azkaban/linkisjob/LinkisJob.java delete mode 100644 dss-azkaban-scheduler-appjoint/src/main/java/com/webank/wedatasphere/dss/appjoint/scheduler/azkaban/linkisjob/LinkisJobConverter.java delete mode 100644 dss-azkaban-scheduler-appjoint/src/main/java/com/webank/wedatasphere/dss/appjoint/scheduler/azkaban/linkisjob/LinkisJobTuning.java delete mode 100644 dss-azkaban-scheduler-appjoint/src/main/java/com/webank/wedatasphere/dss/appjoint/scheduler/azkaban/parser/AzkabanFlowParser.java delete mode 100644 dss-azkaban-scheduler-appjoint/src/main/java/com/webank/wedatasphere/dss/appjoint/scheduler/azkaban/parser/AzkabanNodeParser.java delete mode 100644 dss-azkaban-scheduler-appjoint/src/main/java/com/webank/wedatasphere/dss/appjoint/scheduler/azkaban/parser/AzkabanProjectParser.java delete mode 100644 dss-azkaban-scheduler-appjoint/src/main/java/com/webank/wedatasphere/dss/appjoint/scheduler/azkaban/parser/LinkisAzkabanNodeParser.java delete mode 100644 dss-azkaban-scheduler-appjoint/src/main/java/com/webank/wedatasphere/dss/appjoint/scheduler/azkaban/parser/LinkisAzkabanSendEmailNodeParser.java delete mode 100644 dss-azkaban-scheduler-appjoint/src/main/java/com/webank/wedatasphere/dss/appjoint/scheduler/azkaban/service/AzkabanProjectService.java delete mode 100644 dss-azkaban-scheduler-appjoint/src/main/java/com/webank/wedatasphere/dss/appjoint/scheduler/azkaban/service/AzkabanSecurityService.java delete mode 100644 dss-azkaban-scheduler-appjoint/src/main/java/com/webank/wedatasphere/dss/appjoint/scheduler/azkaban/service/AzkabanSession.java delete mode 100644 dss-azkaban-scheduler-appjoint/src/main/java/com/webank/wedatasphere/dss/appjoint/scheduler/azkaban/tuning/AzkabanProjectTuning.java delete mode 100644 dss-azkaban-scheduler-appjoint/src/main/java/com/webank/wedatasphere/dss/appjoint/scheduler/azkaban/tuning/LinkisAzkabanFlowTuning.java delete mode 100644 dss-azkaban-scheduler-appjoint/src/main/java/com/webank/wedatasphere/dss/appjoint/scheduler/azkaban/tuning/LinkisShareNodeFlowTuning.java delete mode 100644 dss-azkaban-scheduler-appjoint/src/main/java/com/webank/wedatasphere/dss/appjoint/scheduler/azkaban/util/AzkabanUtils.java delete mode 100644 dss-azkaban-scheduler-appjoint/src/main/java/com/webank/wedatasphere/dss/appjoint/scheduler/azkaban/util/AzkabanUtilsScala.scala delete mode 100644 dss-azkaban-scheduler-appjoint/src/main/resources/appjoint.properties delete mode 100644 dss-azkaban-scheduler-appjoint/src/main/resources/log4j.properties delete mode 100644 dss-azkaban-scheduler-appjoint/src/main/resources/log4j2.xml delete mode 100644 dss-common/src/main/java/com/webank/wedatasphere/dss/common/entity/Resource.java delete mode 100644 dss-common/src/main/java/com/webank/wedatasphere/dss/common/entity/flow/DSSFlow.java delete mode 100644 dss-common/src/main/java/com/webank/wedatasphere/dss/common/entity/flow/DSSFlowPublishHistory.java delete mode 100644 dss-common/src/main/java/com/webank/wedatasphere/dss/common/entity/flow/DSSFlowVersion.java delete mode 100644 dss-common/src/main/java/com/webank/wedatasphere/dss/common/entity/flow/DSSJSONFlow.java delete mode 100644 dss-common/src/main/java/com/webank/wedatasphere/dss/common/entity/flow/FileFlowVersion.java delete mode 100644 dss-common/src/main/java/com/webank/wedatasphere/dss/common/entity/flow/Flow.java delete mode 100644 dss-common/src/main/java/com/webank/wedatasphere/dss/common/entity/flow/FlowVersion.java delete mode 100644 dss-common/src/main/java/com/webank/wedatasphere/dss/common/entity/flow/FlowVersionForNodes.java delete mode 100644 dss-common/src/main/java/com/webank/wedatasphere/dss/common/entity/flow/JSONFlowVersion.java delete mode 100644 dss-common/src/main/java/com/webank/wedatasphere/dss/common/entity/node/DSSEdge.java delete mode 100644 dss-common/src/main/java/com/webank/wedatasphere/dss/common/entity/node/DSSEdgeDefault.java delete mode 100644 dss-common/src/main/java/com/webank/wedatasphere/dss/common/entity/node/DSSNode.java delete mode 100644 dss-common/src/main/java/com/webank/wedatasphere/dss/common/entity/node/DSSNodeDefault.java delete mode 100644 dss-common/src/main/java/com/webank/wedatasphere/dss/common/entity/node/Layout.java delete mode 100644 dss-common/src/main/java/com/webank/wedatasphere/dss/common/entity/node/Node.java delete mode 100644 dss-common/src/main/java/com/webank/wedatasphere/dss/common/entity/project/DSSJSONProject.java delete mode 100644 dss-common/src/main/java/com/webank/wedatasphere/dss/common/entity/project/DSSProject.java delete mode 100644 dss-common/src/main/java/com/webank/wedatasphere/dss/common/entity/project/DSSProjectPublishHistory.java delete mode 100644 dss-common/src/main/java/com/webank/wedatasphere/dss/common/entity/project/DSSProjectVersion.java delete mode 100644 dss-common/src/main/java/com/webank/wedatasphere/dss/common/entity/project/Project.java delete mode 100644 dss-common/src/main/java/com/webank/wedatasphere/dss/common/entity/project/ProjectVersion.java delete mode 100644 dss-common/src/main/java/com/webank/wedatasphere/dss/common/entity/project/ProjectVersionForFlows.java delete mode 100644 dss-common/src/main/java/com/webank/wedatasphere/dss/common/exception/DSSErrorException.java delete mode 100644 dss-common/src/main/java/com/webank/wedatasphere/dss/common/exception/DSSRuntimeException.java delete mode 100644 dss-common/src/main/java/com/webank/wedatasphere/dss/common/exception/ThrowingConsumer.java delete mode 100644 dss-common/src/main/java/com/webank/wedatasphere/dss/common/exception/ThrowingFunction.java delete mode 100644 dss-common/src/main/java/com/webank/wedatasphere/dss/common/utils/DSSExceptionUtils.java delete mode 100644 dss-common/src/main/java/com/webank/wedatasphere/dss/common/utils/FileHelper.java delete mode 100644 dss-common/src/main/java/com/webank/wedatasphere/dss/common/utils/ZipHelper.java delete mode 100644 dss-common/src/main/scala/com/webank/wedatasphere/dss/common/protocol/RequestDSSProject.scala delete mode 100644 dss-flow-execution-entrance/bin/start-dss-flow-execution-entrance.sh delete mode 100644 dss-flow-execution-entrance/bin/stop-dss-flow-execution-entrance.sh delete mode 100644 dss-flow-execution-entrance/pom.xml delete mode 100644 dss-flow-execution-entrance/src/main/assembly/distribution.xml delete mode 100644 dss-flow-execution-entrance/src/main/java/com/webank/wedatasphere/dss/flow/execution/entrance/entity/FlowExecutionCode.java delete mode 100644 dss-flow-execution-entrance/src/main/java/com/webank/wedatasphere/dss/flow/execution/entrance/entity/FlowExecutionNode.java delete mode 100644 dss-flow-execution-entrance/src/main/java/com/webank/wedatasphere/dss/flow/execution/entrance/entity/FlowExecutonReadNode.java delete mode 100644 dss-flow-execution-entrance/src/main/java/com/webank/wedatasphere/dss/flow/execution/entrance/entity/FlowExecutonSharedNode.java delete mode 100644 dss-flow-execution-entrance/src/main/java/com/webank/wedatasphere/dss/flow/execution/entrance/job/FlowExecutionAppJointLinkisJob.java delete mode 100644 dss-flow-execution-entrance/src/main/java/com/webank/wedatasphere/dss/flow/execution/entrance/job/FlowExecutionAppJointLinkisReadJob.java delete mode 100644 dss-flow-execution-entrance/src/main/java/com/webank/wedatasphere/dss/flow/execution/entrance/job/FlowExecutionAppJointLinkisSharedJob.java delete mode 100644 dss-flow-execution-entrance/src/main/java/com/webank/wedatasphere/dss/flow/execution/entrance/job/FlowExecutionAppJointSignalSharedJob.java delete mode 100644 dss-flow-execution-entrance/src/main/java/com/webank/wedatasphere/dss/flow/execution/entrance/job/FlowExecutionCommonLinkisJob.java delete mode 100644 dss-flow-execution-entrance/src/main/java/com/webank/wedatasphere/dss/flow/execution/entrance/job/FlowExecutionCommonLinkisReadJob.java delete mode 100644 dss-flow-execution-entrance/src/main/java/com/webank/wedatasphere/dss/flow/execution/entrance/job/FlowExecutionCommonLinkisSharedJob.java delete mode 100644 dss-flow-execution-entrance/src/main/java/com/webank/wedatasphere/dss/flow/execution/entrance/job/FlowExecutionJobSignalKeyCreator.java delete mode 100644 dss-flow-execution-entrance/src/main/java/com/webank/wedatasphere/dss/flow/execution/entrance/parser/FlowExecutionFlowParser.java delete mode 100644 dss-flow-execution-entrance/src/main/java/com/webank/wedatasphere/dss/flow/execution/entrance/parser/FlowExecutionNodeParser.java delete mode 100644 dss-flow-execution-entrance/src/main/java/com/webank/wedatasphere/dss/flow/execution/entrance/parser/FlowExecutionProjectParser.java delete mode 100644 dss-flow-execution-entrance/src/main/java/com/webank/wedatasphere/dss/flow/execution/entrance/parser/FlowExecutionReadNodeParser.java delete mode 100644 dss-flow-execution-entrance/src/main/java/com/webank/wedatasphere/dss/flow/execution/entrance/restful/FlowExecutionRestfulApi.java delete mode 100644 dss-flow-execution-entrance/src/main/java/com/webank/wedatasphere/dss/flow/execution/entrance/tuning/FlowExecutionFlowTuning.java delete mode 100644 dss-flow-execution-entrance/src/main/java/com/webank/wedatasphere/dss/flow/execution/entrance/tuning/FlowExecutionProjectTuning.java delete mode 100644 dss-flow-execution-entrance/src/main/resources/application.yml delete mode 100644 dss-flow-execution-entrance/src/main/resources/linkis.properties delete mode 100644 dss-flow-execution-entrance/src/main/resources/log4j.properties delete mode 100644 dss-flow-execution-entrance/src/main/resources/log4j2.xml delete mode 100644 dss-flow-execution-entrance/src/main/scala/com/webank/wedatasphere/dss/flow/execution/entrance/FlowContext.scala delete mode 100644 dss-flow-execution-entrance/src/main/scala/com/webank/wedatasphere/dss/flow/execution/entrance/FlowContextImpl.scala delete mode 100644 dss-flow-execution-entrance/src/main/scala/com/webank/wedatasphere/dss/flow/execution/entrance/conf/FlowExecutionEntranceConfiguration.scala delete mode 100644 dss-flow-execution-entrance/src/main/scala/com/webank/wedatasphere/dss/flow/execution/entrance/conf/FlowExecutionEntranceSpringConfiguration.scala delete mode 100644 dss-flow-execution-entrance/src/main/scala/com/webank/wedatasphere/dss/flow/execution/entrance/engine/FlowEntranceEngine.scala delete mode 100644 dss-flow-execution-entrance/src/main/scala/com/webank/wedatasphere/dss/flow/execution/entrance/engine/FlowExecutionExecutorManagerImpl.scala delete mode 100644 dss-flow-execution-entrance/src/main/scala/com/webank/wedatasphere/dss/flow/execution/entrance/entranceparser/FlowExecutionParser.scala delete mode 100644 dss-flow-execution-entrance/src/main/scala/com/webank/wedatasphere/dss/flow/execution/entrance/exception/FlowExecutionErrorException.scala delete mode 100644 dss-flow-execution-entrance/src/main/scala/com/webank/wedatasphere/dss/flow/execution/entrance/execution/DefaultFlowExecution.scala delete mode 100644 dss-flow-execution-entrance/src/main/scala/com/webank/wedatasphere/dss/flow/execution/entrance/execution/FlowExecution.scala delete mode 100644 dss-flow-execution-entrance/src/main/scala/com/webank/wedatasphere/dss/flow/execution/entrance/execution/NodeExecutionStatusPoller.scala delete mode 100644 dss-flow-execution-entrance/src/main/scala/com/webank/wedatasphere/dss/flow/execution/entrance/job/FlowEntranceJob.scala delete mode 100644 dss-flow-execution-entrance/src/main/scala/com/webank/wedatasphere/dss/flow/execution/entrance/job/FlowExecutionRequest.scala delete mode 100644 dss-flow-execution-entrance/src/main/scala/com/webank/wedatasphere/dss/flow/execution/entrance/job/parser/FlowEntranceJobParser.scala delete mode 100644 dss-flow-execution-entrance/src/main/scala/com/webank/wedatasphere/dss/flow/execution/entrance/job/parser/FlowJobFlowParser.scala delete mode 100644 dss-flow-execution-entrance/src/main/scala/com/webank/wedatasphere/dss/flow/execution/entrance/job/parser/FlowJobNodeParser.scala delete mode 100644 dss-flow-execution-entrance/src/main/scala/com/webank/wedatasphere/dss/flow/execution/entrance/listener/NodeRunnerListener.scala delete mode 100644 dss-flow-execution-entrance/src/main/scala/com/webank/wedatasphere/dss/flow/execution/entrance/log/FlowExecutionLog.scala delete mode 100644 dss-flow-execution-entrance/src/main/scala/com/webank/wedatasphere/dss/flow/execution/entrance/node/AppJointJobBuilder.scala delete mode 100644 dss-flow-execution-entrance/src/main/scala/com/webank/wedatasphere/dss/flow/execution/entrance/node/DefaultNodeRunner.scala delete mode 100644 dss-flow-execution-entrance/src/main/scala/com/webank/wedatasphere/dss/flow/execution/entrance/node/NodeExecutionState.scala delete mode 100644 dss-flow-execution-entrance/src/main/scala/com/webank/wedatasphere/dss/flow/execution/entrance/node/NodeRunner.scala delete mode 100644 dss-flow-execution-entrance/src/main/scala/com/webank/wedatasphere/dss/flow/execution/entrance/resolver/FlowDependencyResolver.scala delete mode 100644 dss-flow-execution-entrance/src/main/scala/com/webank/wedatasphere/dss/flow/execution/entrance/resolver/FlowDependencyResolverImpl.scala delete mode 100644 dss-flow-execution-entrance/src/main/scala/com/webank/wedatasphere/dss/flow/execution/entrance/utils/FlowExecutionUtils.scala delete mode 100644 dss-linkis-node-execution/pom.xml delete mode 100644 dss-linkis-node-execution/src/main/java/com/webank/wedatasphere/dss/linkis/node/execution/WorkflowContext.java delete mode 100644 dss-linkis-node-execution/src/main/java/com/webank/wedatasphere/dss/linkis/node/execution/WorkflowContextFactory.java delete mode 100644 dss-linkis-node-execution/src/main/java/com/webank/wedatasphere/dss/linkis/node/execution/WorkflowContextImpl.java delete mode 100644 dss-linkis-node-execution/src/main/java/com/webank/wedatasphere/dss/linkis/node/execution/conf/LinkisJobExecutionConfiguration.java delete mode 100644 dss-linkis-node-execution/src/main/java/com/webank/wedatasphere/dss/linkis/node/execution/entity/BMLResource.java delete mode 100644 dss-linkis-node-execution/src/main/java/com/webank/wedatasphere/dss/linkis/node/execution/entity/ContextInfo.java delete mode 100644 dss-linkis-node-execution/src/main/java/com/webank/wedatasphere/dss/linkis/node/execution/exception/LinkisJobExecutionErrorException.java delete mode 100644 dss-linkis-node-execution/src/main/java/com/webank/wedatasphere/dss/linkis/node/execution/exception/LinkisJobExecutionWarnException.java delete mode 100644 dss-linkis-node-execution/src/main/java/com/webank/wedatasphere/dss/linkis/node/execution/execution/LinkisNodeExecution.java delete mode 100644 dss-linkis-node-execution/src/main/java/com/webank/wedatasphere/dss/linkis/node/execution/execution/impl/LinkisNodeExecutionImpl.java delete mode 100644 dss-linkis-node-execution/src/main/java/com/webank/wedatasphere/dss/linkis/node/execution/job/AbstractAppJointLinkisJob.java delete mode 100644 dss-linkis-node-execution/src/main/java/com/webank/wedatasphere/dss/linkis/node/execution/job/AbstractCommonLinkisJob.java delete mode 100644 dss-linkis-node-execution/src/main/java/com/webank/wedatasphere/dss/linkis/node/execution/job/AppJointLinkisJob.java delete mode 100644 dss-linkis-node-execution/src/main/java/com/webank/wedatasphere/dss/linkis/node/execution/job/Builder.java delete mode 100644 dss-linkis-node-execution/src/main/java/com/webank/wedatasphere/dss/linkis/node/execution/job/CommonLinkisJob.java delete mode 100644 dss-linkis-node-execution/src/main/java/com/webank/wedatasphere/dss/linkis/node/execution/job/Job.java delete mode 100644 dss-linkis-node-execution/src/main/java/com/webank/wedatasphere/dss/linkis/node/execution/job/JobSignalKeyCreator.java delete mode 100644 dss-linkis-node-execution/src/main/java/com/webank/wedatasphere/dss/linkis/node/execution/job/JobTypeEnum.java delete mode 100644 dss-linkis-node-execution/src/main/java/com/webank/wedatasphere/dss/linkis/node/execution/job/LinkisJob.java delete mode 100644 dss-linkis-node-execution/src/main/java/com/webank/wedatasphere/dss/linkis/node/execution/job/ReadJob.java delete mode 100644 dss-linkis-node-execution/src/main/java/com/webank/wedatasphere/dss/linkis/node/execution/job/RuntimeJob.java delete mode 100644 dss-linkis-node-execution/src/main/java/com/webank/wedatasphere/dss/linkis/node/execution/job/SharedJob.java delete mode 100644 dss-linkis-node-execution/src/main/java/com/webank/wedatasphere/dss/linkis/node/execution/job/SignalSharedJob.java delete mode 100644 dss-linkis-node-execution/src/main/java/com/webank/wedatasphere/dss/linkis/node/execution/listener/LinkisExecutionListener.java delete mode 100644 dss-linkis-node-execution/src/main/java/com/webank/wedatasphere/dss/linkis/node/execution/log/LinkisJobExecutionLog.java delete mode 100644 dss-linkis-node-execution/src/main/java/com/webank/wedatasphere/dss/linkis/node/execution/parser/CodeParser.java delete mode 100644 dss-linkis-node-execution/src/main/java/com/webank/wedatasphere/dss/linkis/node/execution/parser/JobParamsParser.java delete mode 100644 dss-linkis-node-execution/src/main/java/com/webank/wedatasphere/dss/linkis/node/execution/parser/JobParser.java delete mode 100644 dss-linkis-node-execution/src/main/java/com/webank/wedatasphere/dss/linkis/node/execution/parser/JobRuntimeParamsParser.java delete mode 100644 dss-linkis-node-execution/src/main/java/com/webank/wedatasphere/dss/linkis/node/execution/service/BuildJobAction.java delete mode 100644 dss-linkis-node-execution/src/main/java/com/webank/wedatasphere/dss/linkis/node/execution/service/impl/BuildJobActionImpl.java delete mode 100644 dss-linkis-node-execution/src/main/java/com/webank/wedatasphere/dss/linkis/node/execution/utils/LinkisJobExecutionUtils.java delete mode 100644 dss-linkis-node-execution/src/main/java/com/webank/wedatasphere/dss/linkis/node/execution/utils/LinkisUjesClientUtils.java delete mode 100644 dss-scheduler-appjoint-core/pom.xml delete mode 100644 dss-scheduler-appjoint-core/src/main/java/com/webank/wedatasphere/dss/appjoint/scheduler/SchedulerAppJoint.java delete mode 100644 dss-scheduler-appjoint-core/src/main/java/com/webank/wedatasphere/dss/appjoint/scheduler/constant/SchedulerAppJointConstant.java delete mode 100644 dss-scheduler-appjoint-core/src/main/java/com/webank/wedatasphere/dss/appjoint/scheduler/entity/AbstractSchedulerNode.java delete mode 100644 dss-scheduler-appjoint-core/src/main/java/com/webank/wedatasphere/dss/appjoint/scheduler/entity/AbstractSchedulerProject.java delete mode 100644 dss-scheduler-appjoint-core/src/main/java/com/webank/wedatasphere/dss/appjoint/scheduler/entity/FileProjectVersion.java delete mode 100644 dss-scheduler-appjoint-core/src/main/java/com/webank/wedatasphere/dss/appjoint/scheduler/entity/ReadNode.java delete mode 100644 dss-scheduler-appjoint-core/src/main/java/com/webank/wedatasphere/dss/appjoint/scheduler/entity/SchedulerEdge.java delete mode 100644 dss-scheduler-appjoint-core/src/main/java/com/webank/wedatasphere/dss/appjoint/scheduler/entity/SchedulerEdgeDefault.java delete mode 100644 dss-scheduler-appjoint-core/src/main/java/com/webank/wedatasphere/dss/appjoint/scheduler/entity/SchedulerFlow.java delete mode 100644 dss-scheduler-appjoint-core/src/main/java/com/webank/wedatasphere/dss/appjoint/scheduler/entity/SchedulerNode.java delete mode 100644 dss-scheduler-appjoint-core/src/main/java/com/webank/wedatasphere/dss/appjoint/scheduler/entity/SchedulerProject.java delete mode 100644 dss-scheduler-appjoint-core/src/main/java/com/webank/wedatasphere/dss/appjoint/scheduler/entity/SchedulerProjectVersionForFlows.java delete mode 100644 dss-scheduler-appjoint-core/src/main/java/com/webank/wedatasphere/dss/appjoint/scheduler/entity/ShareNode.java delete mode 100644 dss-scheduler-appjoint-core/src/main/java/com/webank/wedatasphere/dss/appjoint/scheduler/hooks/AbstractFlowPublishHook.java delete mode 100644 dss-scheduler-appjoint-core/src/main/java/com/webank/wedatasphere/dss/appjoint/scheduler/hooks/AbstractNodePublishHook.java delete mode 100644 dss-scheduler-appjoint-core/src/main/java/com/webank/wedatasphere/dss/appjoint/scheduler/hooks/AbstractProjectPublishHook.java delete mode 100644 dss-scheduler-appjoint-core/src/main/java/com/webank/wedatasphere/dss/appjoint/scheduler/hooks/FlowPublishHook.java delete mode 100644 dss-scheduler-appjoint-core/src/main/java/com/webank/wedatasphere/dss/appjoint/scheduler/hooks/NodePublishHook.java delete mode 100644 dss-scheduler-appjoint-core/src/main/java/com/webank/wedatasphere/dss/appjoint/scheduler/hooks/ProjectPublishHook.java delete mode 100644 dss-scheduler-appjoint-core/src/main/java/com/webank/wedatasphere/dss/appjoint/scheduler/order/Order.java delete mode 100644 dss-scheduler-appjoint-core/src/main/java/com/webank/wedatasphere/dss/appjoint/scheduler/parser/AbstractFlowParser.java delete mode 100644 dss-scheduler-appjoint-core/src/main/java/com/webank/wedatasphere/dss/appjoint/scheduler/parser/AbstractNodeParser.java delete mode 100644 dss-scheduler-appjoint-core/src/main/java/com/webank/wedatasphere/dss/appjoint/scheduler/parser/AbstractProjectParser.java delete mode 100644 dss-scheduler-appjoint-core/src/main/java/com/webank/wedatasphere/dss/appjoint/scheduler/parser/AbstractReadNodeParser.java delete mode 100644 dss-scheduler-appjoint-core/src/main/java/com/webank/wedatasphere/dss/appjoint/scheduler/parser/ContextNodeParser.java delete mode 100644 dss-scheduler-appjoint-core/src/main/java/com/webank/wedatasphere/dss/appjoint/scheduler/parser/FlowParser.java delete mode 100644 dss-scheduler-appjoint-core/src/main/java/com/webank/wedatasphere/dss/appjoint/scheduler/parser/NodeParser.java delete mode 100644 dss-scheduler-appjoint-core/src/main/java/com/webank/wedatasphere/dss/appjoint/scheduler/parser/ProjectParser.java delete mode 100644 dss-scheduler-appjoint-core/src/main/java/com/webank/wedatasphere/dss/appjoint/scheduler/parser/SendEmailNodeParser.java delete mode 100644 dss-scheduler-appjoint-core/src/main/java/com/webank/wedatasphere/dss/appjoint/scheduler/service/SchedulerProjectService.java delete mode 100644 dss-scheduler-appjoint-core/src/main/java/com/webank/wedatasphere/dss/appjoint/scheduler/service/SchedulerSecurityService.java delete mode 100644 dss-scheduler-appjoint-core/src/main/java/com/webank/wedatasphere/dss/appjoint/scheduler/tuning/AbstractFlowTuning.java delete mode 100644 dss-scheduler-appjoint-core/src/main/java/com/webank/wedatasphere/dss/appjoint/scheduler/tuning/AbstractNodeTuning.java delete mode 100644 dss-scheduler-appjoint-core/src/main/java/com/webank/wedatasphere/dss/appjoint/scheduler/tuning/AbstractProjectTuning.java delete mode 100644 dss-scheduler-appjoint-core/src/main/java/com/webank/wedatasphere/dss/appjoint/scheduler/tuning/AbstractShareNodeFlowTuning.java delete mode 100644 dss-scheduler-appjoint-core/src/main/java/com/webank/wedatasphere/dss/appjoint/scheduler/tuning/ContextNodeTuning.java delete mode 100644 dss-scheduler-appjoint-core/src/main/java/com/webank/wedatasphere/dss/appjoint/scheduler/tuning/ContextShareableFlowTuning.java delete mode 100644 dss-scheduler-appjoint-core/src/main/java/com/webank/wedatasphere/dss/appjoint/scheduler/tuning/FlowTuning.java delete mode 100644 dss-scheduler-appjoint-core/src/main/java/com/webank/wedatasphere/dss/appjoint/scheduler/tuning/NodeTuning.java delete mode 100644 dss-scheduler-appjoint-core/src/main/java/com/webank/wedatasphere/dss/appjoint/scheduler/tuning/ProjectTuning.java delete mode 100644 dss-server/bin/start-dss-server.sh delete mode 100644 dss-server/bin/stop-dss-server.sh delete mode 100644 dss-server/pom.xml delete mode 100644 dss-server/src/main/assembly/distribution.xml delete mode 100644 dss-server/src/main/java/com/webank/wedatasphere/dss/DSSSpringApplication.java delete mode 100644 dss-server/src/main/java/com/webank/wedatasphere/dss/server/conf/DSSServerConf.java delete mode 100644 dss-server/src/main/java/com/webank/wedatasphere/dss/server/constant/DSSServerConstant.java delete mode 100644 dss-server/src/main/java/com/webank/wedatasphere/dss/server/dao/DSSUserMapper.java delete mode 100644 dss-server/src/main/java/com/webank/wedatasphere/dss/server/dao/FlowMapper.java delete mode 100644 dss-server/src/main/java/com/webank/wedatasphere/dss/server/dao/FlowTaxonomyMapper.java delete mode 100644 dss-server/src/main/java/com/webank/wedatasphere/dss/server/dao/NodeInfoMapper.java delete mode 100644 dss-server/src/main/java/com/webank/wedatasphere/dss/server/dao/ProjectMapper.java delete mode 100644 dss-server/src/main/java/com/webank/wedatasphere/dss/server/dao/ProjectTaxonomyMapper.java delete mode 100644 dss-server/src/main/java/com/webank/wedatasphere/dss/server/dao/WorkspaceMapper.java delete mode 100644 dss-server/src/main/java/com/webank/wedatasphere/dss/server/dao/impl/dwsUserMapper.xml delete mode 100644 dss-server/src/main/java/com/webank/wedatasphere/dss/server/dao/impl/flowMapper.xml delete mode 100644 dss-server/src/main/java/com/webank/wedatasphere/dss/server/dao/impl/flowTaxonomyMapper.xml delete mode 100644 dss-server/src/main/java/com/webank/wedatasphere/dss/server/dao/impl/nodeInfoMapper.xml delete mode 100644 dss-server/src/main/java/com/webank/wedatasphere/dss/server/dao/impl/projectMapper.xml delete mode 100644 dss-server/src/main/java/com/webank/wedatasphere/dss/server/dao/impl/projectTaxonomyMapper.xml delete mode 100644 dss-server/src/main/java/com/webank/wedatasphere/dss/server/dao/impl/workspaceMapper.xml delete mode 100644 dss-server/src/main/java/com/webank/wedatasphere/dss/server/db/DSSMybatisConfig.java delete mode 100644 dss-server/src/main/java/com/webank/wedatasphere/dss/server/dto/response/HomepageDemoInstanceVo.java delete mode 100644 dss-server/src/main/java/com/webank/wedatasphere/dss/server/dto/response/HomepageDemoMenuVo.java delete mode 100644 dss-server/src/main/java/com/webank/wedatasphere/dss/server/dto/response/HomepageVideoVo.java delete mode 100644 dss-server/src/main/java/com/webank/wedatasphere/dss/server/dto/response/OnestopMenuAppInstanceVo.java delete mode 100644 dss-server/src/main/java/com/webank/wedatasphere/dss/server/dto/response/OnestopMenuVo.java delete mode 100644 dss-server/src/main/java/com/webank/wedatasphere/dss/server/dto/response/WorkspaceDepartmentVo.java delete mode 100644 dss-server/src/main/java/com/webank/wedatasphere/dss/server/dto/response/WorkspaceFavoriteVo.java delete mode 100644 dss-server/src/main/java/com/webank/wedatasphere/dss/server/entity/ApplicationArea.java delete mode 100644 dss-server/src/main/java/com/webank/wedatasphere/dss/server/entity/BaseEntity.java delete mode 100644 dss-server/src/main/java/com/webank/wedatasphere/dss/server/entity/DSSFavorite.java delete mode 100644 dss-server/src/main/java/com/webank/wedatasphere/dss/server/entity/DSSFlowTaxonomy.java delete mode 100644 dss-server/src/main/java/com/webank/wedatasphere/dss/server/entity/DSSProjectTaxonomy.java delete mode 100644 dss-server/src/main/java/com/webank/wedatasphere/dss/server/entity/DSSProjectTaxonomyRelation.java delete mode 100644 dss-server/src/main/java/com/webank/wedatasphere/dss/server/entity/DSSWorkspace.java delete mode 100644 dss-server/src/main/java/com/webank/wedatasphere/dss/server/entity/NodeInfo.java delete mode 100644 dss-server/src/main/java/com/webank/wedatasphere/dss/server/function/FunctionInvoker.java delete mode 100644 dss-server/src/main/java/com/webank/wedatasphere/dss/server/function/FunctionPool.java delete mode 100644 dss-server/src/main/java/com/webank/wedatasphere/dss/server/function/NodeServiceFunction.java delete mode 100644 dss-server/src/main/java/com/webank/wedatasphere/dss/server/function/ProjectServiceAddFunction.java delete mode 100644 dss-server/src/main/java/com/webank/wedatasphere/dss/server/function/ProjectServiceFunction.java delete mode 100644 dss-server/src/main/java/com/webank/wedatasphere/dss/server/lock/Lock.java delete mode 100644 dss-server/src/main/java/com/webank/wedatasphere/dss/server/lock/LockAspect.java delete mode 100644 dss-server/src/main/java/com/webank/wedatasphere/dss/server/lock/LockEnum.java delete mode 100644 dss-server/src/main/java/com/webank/wedatasphere/dss/server/operate/AppJointNodeOperate.java delete mode 100644 dss-server/src/main/java/com/webank/wedatasphere/dss/server/operate/Op.java delete mode 100644 dss-server/src/main/java/com/webank/wedatasphere/dss/server/operate/Operate.java delete mode 100644 dss-server/src/main/java/com/webank/wedatasphere/dss/server/operate/SubFlowOperate.java delete mode 100644 dss-server/src/main/java/com/webank/wedatasphere/dss/server/publish/PublishJob.java delete mode 100644 dss-server/src/main/java/com/webank/wedatasphere/dss/server/publish/PublishJobEnum.java delete mode 100644 dss-server/src/main/java/com/webank/wedatasphere/dss/server/publish/PublishJobFactory.java delete mode 100644 dss-server/src/main/java/com/webank/wedatasphere/dss/server/publish/PublishListner.java delete mode 100644 dss-server/src/main/java/com/webank/wedatasphere/dss/server/publish/PublishManager.java delete mode 100644 dss-server/src/main/java/com/webank/wedatasphere/dss/server/publish/PublishProjectCache.java delete mode 100644 dss-server/src/main/java/com/webank/wedatasphere/dss/server/publish/PublishSubmitJob.java delete mode 100644 dss-server/src/main/java/com/webank/wedatasphere/dss/server/publish/PublishSubmitJobDeamon.java delete mode 100644 dss-server/src/main/java/com/webank/wedatasphere/dss/server/publish/PublishThreadPool.java delete mode 100644 dss-server/src/main/java/com/webank/wedatasphere/dss/server/restful/FlowRestfulApi.java delete mode 100644 dss-server/src/main/java/com/webank/wedatasphere/dss/server/restful/FlowTaxonomyRestfulApi.java delete mode 100644 dss-server/src/main/java/com/webank/wedatasphere/dss/server/restful/NodeRestfulApi.java delete mode 100644 dss-server/src/main/java/com/webank/wedatasphere/dss/server/restful/ProjectRestfulApi.java delete mode 100644 dss-server/src/main/java/com/webank/wedatasphere/dss/server/restful/ProjectTaxonomyRestfulApi.java delete mode 100644 dss-server/src/main/java/com/webank/wedatasphere/dss/server/restful/TreeRestfulApi.java delete mode 100644 dss-server/src/main/java/com/webank/wedatasphere/dss/server/restful/UserManagerApi.java delete mode 100644 dss-server/src/main/java/com/webank/wedatasphere/dss/server/restful/WorkspaceRestfulApi.java delete mode 100644 dss-server/src/main/java/com/webank/wedatasphere/dss/server/service/DSSFlowService.java delete mode 100644 dss-server/src/main/java/com/webank/wedatasphere/dss/server/service/DSSFlowTaxonomyService.java delete mode 100644 dss-server/src/main/java/com/webank/wedatasphere/dss/server/service/DSSNodeInfoService.java delete mode 100644 dss-server/src/main/java/com/webank/wedatasphere/dss/server/service/DSSProjectService.java delete mode 100644 dss-server/src/main/java/com/webank/wedatasphere/dss/server/service/DSSProjectTaxonomyService.java delete mode 100644 dss-server/src/main/java/com/webank/wedatasphere/dss/server/service/DSSUserService.java delete mode 100644 dss-server/src/main/java/com/webank/wedatasphere/dss/server/service/DSSWorkspaceService.java delete mode 100644 dss-server/src/main/java/com/webank/wedatasphere/dss/server/service/impl/DSSFlowServiceImpl.java delete mode 100644 dss-server/src/main/java/com/webank/wedatasphere/dss/server/service/impl/DSSFlowTaxonomyServiceImpl.java delete mode 100644 dss-server/src/main/java/com/webank/wedatasphere/dss/server/service/impl/DSSNodeInfoServiceImpl.java delete mode 100644 dss-server/src/main/java/com/webank/wedatasphere/dss/server/service/impl/DSSProjectServiceImpl.java delete mode 100644 dss-server/src/main/java/com/webank/wedatasphere/dss/server/service/impl/DSSProjectTaxonomyServiceImpl.java delete mode 100644 dss-server/src/main/java/com/webank/wedatasphere/dss/server/service/impl/DSSUserServiceImpl.java delete mode 100644 dss-server/src/main/java/com/webank/wedatasphere/dss/server/service/impl/DSSWorkspaceServiceImpl.java delete mode 100644 dss-server/src/main/java/com/webank/wedatasphere/dss/server/util/ThreadPoolTool.java delete mode 100644 dss-server/src/main/resources/application.yml delete mode 100644 dss-server/src/main/resources/default/AddschedulerUser.sh delete mode 100644 dss-server/src/main/resources/default/CreateLdapAccount.sh delete mode 100644 dss-server/src/main/resources/default/CreateLinuxUser.sh delete mode 100644 dss-server/src/main/resources/default/HdfsPath.sh delete mode 100644 dss-server/src/main/resources/default/LinuxPath.sh delete mode 100644 dss-server/src/main/resources/linkis.properties delete mode 100644 dss-server/src/main/resources/log4j.properties delete mode 100644 dss-server/src/main/resources/log4j2.xml delete mode 100644 dss-server/src/main/resources/token.properties delete mode 100644 dss-server/src/main/scala/com/webank/wedatasphere/dss/server/crumb/CrumbFactory.scala delete mode 100644 dss-server/src/main/scala/com/webank/wedatasphere/dss/server/crumb/QuerParamsParser.scala delete mode 100644 dss-server/src/main/scala/com/webank/wedatasphere/dss/server/entity/Crumb.scala delete mode 100644 dss-server/src/main/scala/com/webank/wedatasphere/dss/server/entity/CrumbType.scala delete mode 100644 dss-server/src/main/scala/com/webank/wedatasphere/dss/server/receiver/DSSServerReceiver.scala delete mode 100644 dss-server/src/main/scala/com/webank/wedatasphere/dss/server/service/BMLService.scala delete mode 100644 dss-server/src/main/test/HttpTest.java delete mode 100644 dss-server/src/main/test/com/webank/Adf.scala delete mode 100644 dss-server/src/main/test/com/webank/HttpTest.java delete mode 100644 dss-server/src/main/test/com/webank/JobParam.java delete mode 100644 dss-server/src/main/test/com/webank/ReqTest.scala delete mode 100644 dss-server/src/main/test/com/webank/TestUnit.java delete mode 100644 dss-user-manager/pom.xml delete mode 100644 dss-user-manager/src/main/java/com/webank/wedatasphpere/dss/user/conf/DSSUserManagerConfig.java delete mode 100644 dss-user-manager/src/main/java/com/webank/wedatasphpere/dss/user/dto/request/AuthorizationBody.java delete mode 100644 dss-user-manager/src/main/java/com/webank/wedatasphpere/dss/user/dto/request/LinuxServer.java delete mode 100644 dss-user-manager/src/main/java/com/webank/wedatasphpere/dss/user/service/AbsCommand.java delete mode 100644 dss-user-manager/src/main/java/com/webank/wedatasphpere/dss/user/service/Command.java delete mode 100644 dss-user-manager/src/main/java/com/webank/wedatasphpere/dss/user/service/MacroCommand.java delete mode 100644 dss-user-manager/src/main/java/com/webank/wedatasphpere/dss/user/service/impl/AzkabanCommand.java delete mode 100644 dss-user-manager/src/main/java/com/webank/wedatasphpere/dss/user/service/impl/KerberosCommand.java delete mode 100644 dss-user-manager/src/main/java/com/webank/wedatasphpere/dss/user/service/impl/LdapCommand.java delete mode 100644 dss-user-manager/src/main/java/com/webank/wedatasphpere/dss/user/service/impl/LinuxUserCommand.java delete mode 100644 dss-user-manager/src/main/java/com/webank/wedatasphpere/dss/user/service/impl/MetastoreCommand.java delete mode 100644 dss-user-manager/src/main/java/com/webank/wedatasphpere/dss/user/service/impl/SchedulisCommand.java delete mode 100644 dss-user-manager/src/main/java/com/webank/wedatasphpere/dss/user/service/impl/UserAuthorizationClient.java delete mode 100644 dss-user-manager/src/main/java/com/webank/wedatasphpere/dss/user/service/impl/UserMacroCommand.java delete mode 100644 dss-user-manager/src/main/java/com/webank/wedatasphpere/dss/user/service/impl/WorkspaceCommand.java delete mode 100644 dss-user-manager/src/main/java/com/webank/wedatasphpere/dss/user/service/impl/YarnCommand.java delete mode 100644 dss-user-manager/src/main/resources/config/properties.conf delete mode 100644 dss-user-manager/src/main/resources/default/AddschedulerUser.sh delete mode 100644 dss-user-manager/src/main/resources/default/CreateLdapAccount.sh delete mode 100644 dss-user-manager/src/main/resources/default/CreateLinuxUser.sh delete mode 100644 dss-user-manager/src/main/resources/default/HdfsPath.sh delete mode 100644 dss-user-manager/src/main/resources/default/Kerberos.sh delete mode 100644 dss-user-manager/src/main/resources/default/LinuxPath.sh delete mode 100644 dss-user-manager/src/main/resources/default/Metastore.sh delete mode 100644 dss-user-manager/src/main/resources/read_anlexander.txt delete mode 100644 dss-user-manager/src/test/java/com/webank/wedatasphpere/dss/user/service/impl/KerberosCommandTest.java delete mode 100644 dss-user-manager/src/test/java/com/webank/wedatasphpere/dss/user/service/impl/MetastoreCommandTest.java delete mode 100644 dss-user-manager/src/test/java/com/webank/wedatasphpere/dss/user/service/impl/WorkspaceCommandTest.java delete mode 100644 eventchecker-appjoint/pom.xml delete mode 100644 eventchecker-appjoint/src/main/assembly/distribution.xml delete mode 100644 eventchecker-appjoint/src/main/java/com/webank/wedatasphere/dss/appjoint/schedulis/jobtype/adapter/EventCheckAdapter.java delete mode 100644 eventchecker-appjoint/src/main/java/com/webank/wedatasphere/dss/appjoint/schedulis/jobtype/connector/EventDruidFactory.java delete mode 100644 eventchecker-appjoint/src/main/java/com/webank/wedatasphere/dss/appjoint/schedulis/jobtype/entity/EventChecker.java delete mode 100644 eventchecker-appjoint/src/main/java/com/webank/wedatasphere/dss/appjoint/schedulis/jobtype/exception/UndefinedPropertyException.java delete mode 100644 eventchecker-appjoint/src/main/java/com/webank/wedatasphere/dss/appjoint/schedulis/jobtype/service/AbstractEventCheck.java delete mode 100644 eventchecker-appjoint/src/main/java/com/webank/wedatasphere/dss/appjoint/schedulis/jobtype/service/AbstractEventCheckReceiver.java delete mode 100644 eventchecker-appjoint/src/main/java/com/webank/wedatasphere/dss/appjoint/schedulis/jobtype/service/DefaultEventcheckReceiver.java delete mode 100644 eventchecker-appjoint/src/main/java/com/webank/wedatasphere/dss/appjoint/schedulis/jobtype/service/EventCheckSender.java delete mode 100644 eventchecker-appjoint/src/main/java/com/webank/wedatasphere/dss/appjoint/schedulis/jobtype/service/EventCheckerService.java delete mode 100644 eventchecker-appjoint/src/main/java/com/webank/wedatasphere/dss/appjoint/schedulis/jobtype/utils/Props.java delete mode 100644 eventchecker-appjoint/src/main/java/com/webank/wedatasphere/dss/appjoint/schedulis/jobtype/utils/Utils.java delete mode 100644 eventchecker-appjoint/src/main/resources/appjoint.properties delete mode 100644 eventchecker-appjoint/src/main/resources/log4j.properties delete mode 100644 eventchecker-appjoint/src/main/resources/log4j2.xml delete mode 100644 eventchecker-appjoint/src/main/scala/com/webank/wedatasphere/dss/appjoint/schedulis/jobtype/execution/EventCheckerAppJoint.scala delete mode 100644 eventchecker-appjoint/src/main/scala/com/webank/wedatasphere/dss/appjoint/schedulis/jobtype/execution/EventCheckerNodeExecution.scala delete mode 100644 eventchecker-appjoint/src/main/scala/com/webank/wedatasphere/dss/appjoint/schedulis/jobtype/execution/EventCheckerNodeExecutionAction.scala delete mode 100644 plugins/linkis/linkis-appjoint-entrance/bin/start-linkis-appjoint-entrance.sh delete mode 100644 plugins/linkis/linkis-appjoint-entrance/bin/stop-linkis-appjoint-entrance.sh delete mode 100644 plugins/linkis/linkis-appjoint-entrance/conf/application.yml delete mode 100644 plugins/linkis/linkis-appjoint-entrance/conf/linkis.properties delete mode 100644 plugins/linkis/linkis-appjoint-entrance/conf/log4j2.xml delete mode 100644 plugins/linkis/linkis-appjoint-entrance/pom.xml delete mode 100644 plugins/linkis/linkis-appjoint-entrance/src/main/assembly/distribution.xml delete mode 100644 plugins/linkis/linkis-appjoint-entrance/src/main/java/com/webank/wedatasphere/dss/linkis/appjoint/entrance/AppJointEntranceRestfulApi.java delete mode 100644 plugins/linkis/linkis-appjoint-entrance/src/main/scala/com/webank/wedatasphere/dss/linkis/appjoint/entrance/appjoint/AppJointManager.scala delete mode 100644 plugins/linkis/linkis-appjoint-entrance/src/main/scala/com/webank/wedatasphere/dss/linkis/appjoint/entrance/conf/AppJointConst.scala delete mode 100644 plugins/linkis/linkis-appjoint-entrance/src/main/scala/com/webank/wedatasphere/dss/linkis/appjoint/entrance/conf/AppJointEntranceConfiguration.scala delete mode 100644 plugins/linkis/linkis-appjoint-entrance/src/main/scala/com/webank/wedatasphere/dss/linkis/appjoint/entrance/conf/AppJointEntranceSpringConfiguration.scala delete mode 100644 plugins/linkis/linkis-appjoint-entrance/src/main/scala/com/webank/wedatasphere/dss/linkis/appjoint/entrance/exception/AppJointEntranceIllegalParamException.scala delete mode 100644 plugins/linkis/linkis-appjoint-entrance/src/main/scala/com/webank/wedatasphere/dss/linkis/appjoint/entrance/exception/OperateNotSupportedException.scala delete mode 100644 plugins/linkis/linkis-appjoint-entrance/src/main/scala/com/webank/wedatasphere/dss/linkis/appjoint/entrance/exception/RPCFailedException.scala delete mode 100644 plugins/linkis/linkis-appjoint-entrance/src/main/scala/com/webank/wedatasphere/dss/linkis/appjoint/entrance/execute/AppJointEngineExecutorManagerImpl.scala delete mode 100644 plugins/linkis/linkis-appjoint-entrance/src/main/scala/com/webank/wedatasphere/dss/linkis/appjoint/entrance/execute/AppJointEntranceEngine.scala delete mode 100644 plugins/linkis/linkis-appjoint-entrance/src/main/scala/com/webank/wedatasphere/dss/linkis/appjoint/entrance/execute/AppJointNodeContextImpl.scala delete mode 100644 plugins/linkis/linkis-appjoint-entrance/src/main/scala/com/webank/wedatasphere/dss/linkis/appjoint/entrance/job/AppJointEntranceJob.scala delete mode 100644 plugins/linkis/linkis-appjoint-entrance/src/main/scala/com/webank/wedatasphere/dss/linkis/appjoint/entrance/job/AppJointExecuteRequest.scala delete mode 100644 plugins/linkis/linkis-appjoint-entrance/src/main/scala/com/webank/wedatasphere/dss/linkis/appjoint/entrance/parser/AppJointEntranceParser.scala delete mode 100644 qualitis-appjoint/appjoint/pom.xml delete mode 100644 qualitis-appjoint/appjoint/src/main/assembly/distribution.xml delete mode 100644 qualitis-appjoint/appjoint/src/main/java/com.webank/wedatasphere/Md5Utils.java delete mode 100644 qualitis-appjoint/appjoint/src/main/java/com.webank/wedatasphere/appjoint/QualitisAppJoint.java delete mode 100644 qualitis-appjoint/appjoint/src/main/java/com.webank/wedatasphere/appjoint/QualitisNodeExecution.java delete mode 100644 qualitis-appjoint/appjoint/src/main/java/com.webank/wedatasphere/appjoint/QualitisNodeExecutionAction.java delete mode 100644 qualitis-appjoint/appjoint/src/main/java/com.webank/wedatasphere/appjoint/QualitisSubmitRequest.java delete mode 100644 qualitis-appjoint/appjoint/src/main/java/com.webank/wedatasphere/appjoint/QualitisTaskStatusEnum.java delete mode 100644 qualitis-appjoint/appjoint/src/main/java/com.webank/wedatasphere/project/QualitisAddProjectRequest.java delete mode 100644 qualitis-appjoint/appjoint/src/main/java/com.webank/wedatasphere/project/QualitisDeleteProjectRequest.java delete mode 100644 qualitis-appjoint/appjoint/src/main/java/com.webank/wedatasphere/project/QualitisProjectServiceImpl.java delete mode 100644 qualitis-appjoint/appjoint/src/main/java/com.webank/wedatasphere/project/QualitisUpdateProjectRequest.java delete mode 100644 qualitis-appjoint/appjoint/src/main/resources/appjoint.properties delete mode 100644 qualitis-appjoint/appjoint/src/main/resources/log4j.properties delete mode 100644 qualitis-appjoint/appjoint/src/main/resources/log4j2.xml delete mode 100644 sendemail-appjoint/sendemail-core/pom.xml delete mode 100644 sendemail-appjoint/sendemail-core/src/main/java/com/webank/wedatasphere/dss/appjoint/sendemail/Email.java delete mode 100644 sendemail-appjoint/sendemail-core/src/main/java/com/webank/wedatasphere/dss/appjoint/sendemail/EmailContent.java delete mode 100644 sendemail-appjoint/sendemail-core/src/main/java/com/webank/wedatasphere/dss/appjoint/sendemail/conf/SendEmailAppJointSpringConfiguration.java delete mode 100644 sendemail-appjoint/sendemail-core/src/main/java/com/webank/wedatasphere/dss/appjoint/sendemail/exception/EmailSendFailedException.java delete mode 100644 sendemail-appjoint/sendemail-core/src/main/java/com/webank/wedatasphere/dss/appjoint/sendemail/sender/AbstractEmailSender.java delete mode 100644 sendemail-appjoint/sendemail-core/src/main/java/com/webank/wedatasphere/dss/appjoint/sendemail/sender/SpringJavaEmailSender.java delete mode 100644 sendemail-appjoint/sendemail-core/src/main/scala/com/webank/wedatasphere/dss/appjoint/sendemail/Attachment.scala delete mode 100644 sendemail-appjoint/sendemail-core/src/main/scala/com/webank/wedatasphere/dss/appjoint/sendemail/EmailContentGenerator.scala delete mode 100644 sendemail-appjoint/sendemail-core/src/main/scala/com/webank/wedatasphere/dss/appjoint/sendemail/EmailContentParser.scala delete mode 100644 sendemail-appjoint/sendemail-core/src/main/scala/com/webank/wedatasphere/dss/appjoint/sendemail/EmailGenerator.scala delete mode 100644 sendemail-appjoint/sendemail-core/src/main/scala/com/webank/wedatasphere/dss/appjoint/sendemail/EmailSender.scala delete mode 100644 sendemail-appjoint/sendemail-core/src/main/scala/com/webank/wedatasphere/dss/appjoint/sendemail/SendEmailAppJoint.scala delete mode 100644 sendemail-appjoint/sendemail-core/src/main/scala/com/webank/wedatasphere/dss/appjoint/sendemail/SendEmailNodeExecution.scala delete mode 100644 sendemail-appjoint/sendemail-core/src/main/scala/com/webank/wedatasphere/dss/appjoint/sendemail/SendEmailNodeExecutionHook.scala delete mode 100644 sendemail-appjoint/sendemail-core/src/main/scala/com/webank/wedatasphere/dss/appjoint/sendemail/conf/SendEmailAppJointConfiguration.scala delete mode 100644 sendemail-appjoint/sendemail-core/src/main/scala/com/webank/wedatasphere/dss/appjoint/sendemail/conf/SendEmailAppJointUtils.scala delete mode 100644 sendemail-appjoint/sendemail-core/src/main/scala/com/webank/wedatasphere/dss/appjoint/sendemail/email/AbstractEmail.scala delete mode 100644 sendemail-appjoint/sendemail-core/src/main/scala/com/webank/wedatasphere/dss/appjoint/sendemail/email/MultiContentEmail.scala delete mode 100644 sendemail-appjoint/sendemail-core/src/main/scala/com/webank/wedatasphere/dss/appjoint/sendemail/email/PngAttachment.scala delete mode 100644 sendemail-appjoint/sendemail-core/src/main/scala/com/webank/wedatasphere/dss/appjoint/sendemail/email/generate/AbstractEmailGenerator.scala delete mode 100644 sendemail-appjoint/sendemail-core/src/main/scala/com/webank/wedatasphere/dss/appjoint/sendemail/email/generate/MultiContentEmailGenerator.scala delete mode 100644 sendemail-appjoint/sendemail-core/src/main/scala/com/webank/wedatasphere/dss/appjoint/sendemail/emailcontent/ArrayEmailContent.scala delete mode 100644 sendemail-appjoint/sendemail-core/src/main/scala/com/webank/wedatasphere/dss/appjoint/sendemail/emailcontent/FsPathStoreEmailContent.scala delete mode 100644 sendemail-appjoint/sendemail-core/src/main/scala/com/webank/wedatasphere/dss/appjoint/sendemail/emailcontent/StringEmailContent.scala delete mode 100644 sendemail-appjoint/sendemail-core/src/main/scala/com/webank/wedatasphere/dss/appjoint/sendemail/emailcontent/generator/AbstractEmailContentGenerator.scala delete mode 100644 sendemail-appjoint/sendemail-core/src/main/scala/com/webank/wedatasphere/dss/appjoint/sendemail/emailcontent/generator/MultiEmailContentGenerator.scala delete mode 100644 sendemail-appjoint/sendemail-core/src/main/scala/com/webank/wedatasphere/dss/appjoint/sendemail/emailcontent/package.scala delete mode 100644 sendemail-appjoint/sendemail-core/src/main/scala/com/webank/wedatasphere/dss/appjoint/sendemail/emailcontent/parser/AbstractEmailContentParser.scala delete mode 100644 sendemail-appjoint/sendemail-core/src/main/scala/com/webank/wedatasphere/dss/appjoint/sendemail/emailcontent/parser/FileEmailContentParser.scala delete mode 100644 sendemail-appjoint/sendemail-core/src/main/scala/com/webank/wedatasphere/dss/appjoint/sendemail/emailcontent/parser/HtmlEmailContentParser.scala delete mode 100644 sendemail-appjoint/sendemail-core/src/main/scala/com/webank/wedatasphere/dss/appjoint/sendemail/emailcontent/parser/PictureEmailContentParser.scala delete mode 100644 sendemail-appjoint/sendemail-core/src/main/scala/com/webank/wedatasphere/dss/appjoint/sendemail/emailcontent/parser/TableEmailContentParser.scala delete mode 100644 visualis-appjoint/appjoint/pom.xml delete mode 100644 visualis-appjoint/appjoint/src/main/assembly/distribution.xml delete mode 100644 visualis-appjoint/appjoint/src/main/java/com/webank/wedatasphere/dss/appjoint/visualis/VisualisAppJoint.java delete mode 100644 visualis-appjoint/appjoint/src/main/java/com/webank/wedatasphere/dss/appjoint/visualis/service/VisualisNodeService.java delete mode 100644 visualis-appjoint/appjoint/src/main/java/com/webank/wedatasphere/dss/appjoint/visualis/service/VisualisSecurityService.java delete mode 100644 visualis-appjoint/appjoint/src/main/java/com/webank/wedatasphere/dss/appjoint/visualis/service/VisualisSession.java delete mode 100644 visualis-appjoint/appjoint/src/main/java/com/webank/wedatasphere/dss/appjoint/visualis/service/nodeservice/DashboardNodeService.java delete mode 100644 visualis-appjoint/appjoint/src/main/java/com/webank/wedatasphere/dss/appjoint/visualis/service/nodeservice/DisplayNodeService.java delete mode 100644 visualis-appjoint/appjoint/src/main/java/com/webank/wedatasphere/dss/appjoint/visualis/service/nodeservice/HttpUtils.java delete mode 100644 visualis-appjoint/appjoint/src/main/scala/com/webank/wedatasphere/dss/appjoint/visualis/execution/VisualisNodeExecution.scala delete mode 100644 visualis-appjoint/appjoint/src/main/scala/com/webank/wedatasphere/dss/appjoint/visualis/execution/VisualisNodeExecutionConfiguration.scala delete mode 100644 visualis-appjoint/appjoint/src/main/scala/com/webank/wedatasphere/linkis/server/JSONUtils.scala diff --git a/bin/checkEnv.sh b/bin/checkEnv.sh deleted file mode 100644 index d51bd5ca21..0000000000 --- a/bin/checkEnv.sh +++ /dev/null @@ -1,45 +0,0 @@ -#!/bin/sh -# -# Copyright 2019 WeBank -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -say() { - printf 'check command fail \n %s\n' "$1" -} - -err() { - say "$1" >&2 - exit 1 -} - -check_cmd() { - command -v "$1" > /dev/null 2>&1 -} - -need_cmd() { - if ! check_cmd "$1"; then - err "need '$1' (your linux command not found)" - fi -} -echo "<-----start to check used cmd---->" -need_cmd yum -need_cmd java -need_cmd mysql -need_cmd unzip -need_cmd expect -need_cmd telnet -need_cmd tar -need_cmd sed -need_cmd dos2unix -echo "<-----end to check used cmd---->" diff --git a/bin/checkServices.sh b/bin/checkServices.sh deleted file mode 100644 index 72df04be43..0000000000 --- a/bin/checkServices.sh +++ /dev/null @@ -1,91 +0,0 @@ -# -# Copyright 2019 WeBank -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -#!/bin/sh -source ~/.bash_profile - -shellDir=`dirname $0` -workDir=`cd ${shellDir}/..;pwd` - -##load config -export LINKIS_DSS_CONF_FILE=${LINKIS_DSS_CONF_FILE:-"${workDir}/conf/config.sh"} -export DISTRIBUTION=${DISTRIBUTION:-"${workDir}/conf/config.sh"} -source ${LINKIS_DSS_CONF_FILE} -source ${DISTRIBUTION} - -MICRO_SERVICE_NAME=$1 -MICRO_SERVICE_IP=$2 -MICRO_SERVICE_PORT=$3 - -local_host="`hostname --fqdn`" - -ipaddr=$(ip addr | awk '/^[0-9]+: / {}; /inet.*global/ {print gensub(/(.*)\/(.*)/, "\\1", "g", $2)}'|awk 'NR==1') - -function isLocal(){ - if [ "$1" == "127.0.0.1" ];then - return 0 - elif [ $1 == "localhost" ]; then - return 0 - elif [ $1 == $local_host ]; then - return 0 - elif [ $1 == $ipaddr ]; then - return 0 - fi - return 1 -} - -function executeCMD(){ - isLocal $1 - flag=$? - echo "Is local "$flag - if [ $flag == "0" ];then - eval $2 - else - ssh -p $SSH_PORT $1 $2 - fi - -} - -#echo "<--------------------------------------------------------------------------->" -#echo "Start to Check if your microservice:$MICRO_SERVICE_NAME is normal via telnet" -#echo "" -#if ! executeCMD $SERVER_IP "test -e $DSS_INSTALL_HOME/$MICRO_SERVICE_NAME"; then -# echo "$MICRO_SERVICE_NAME is not installed,the check steps will be skipped" -# exit 0 -#fi -echo "===========================================================" -echo $MICRO_SERVICE_NAME -echo $MICRO_SERVICE_IP -echo $MICRO_SERVICE_PORT -echo "===========================================================" - -if [ $MICRO_SERVICE_NAME == "visualis-server" ] && [ $MICRO_SERVICE_IP == "127.0.0.1" ]; then - MICRO_SERVICE_IP=$ipaddr -fi - -result=`echo -e "\n" | telnet $MICRO_SERVICE_IP $MICRO_SERVICE_PORT 2>/dev/null | grep Connected | wc -l` -if [ $result -eq 1 ]; then - echo "$MICRO_SERVICE_NAME is ok." -else - echo "<--------------------------------------------------------------------------->" - echo "ERROR your $MICRO_SERVICE_NAME microservice is not start successful !!! ERROR logs as follows :" - echo "PLEAESE CHECK DETAIL LOG,LOCATION:$DSS_INSTALL_HOME/$MICRO_SERVICE_NAME/logs/linkis.out" - echo '<------------------------------------------------------------->' - executeCMD $MICRO_SERVICE_IP "tail -n 50 $DSS_INSTALL_HOME/$MICRO_SERVICE_NAME/logs/*.out" - echo '<-------------------------------------------------------------->' - echo "PLEAESE CHECK DETAIL LOG,LOCATION:$DSS_INSTALL_HOME/$MICRO_SERVICE_NAME/logs/linkis.out" - exit 1 -fi - diff --git a/bin/install.sh b/bin/install.sh deleted file mode 100644 index 7a0b46db6d..0000000000 --- a/bin/install.sh +++ /dev/null @@ -1,583 +0,0 @@ -# -# Copyright 2019 WeBank -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -#!/bin/sh -#Actively load user env - -source ~/.bash_profile - -shellDir=`dirname $0` -workDir=`cd ${shellDir}/..;pwd` - -#To be compatible with MacOS and Linux -txt="" -if [[ "$OSTYPE" == "darwin"* ]]; then - txt="''" -elif [[ "$OSTYPE" == "linux-gnu" ]]; then - # linux - txt="" -elif [[ "$OSTYPE" == "cygwin" ]]; then - echo "dss not support Windows operating system" - exit 1 -elif [[ "$OSTYPE" == "msys" ]]; then - echo "dss not support Windows operating system" - exit 1 -elif [[ "$OSTYPE" == "win32" ]]; then - echo "dss not support Windows operating system" - exit 1 -elif [[ "$OSTYPE" == "freebsd"* ]]; then - txt="" -else - echo "Operating system unknown, please tell us(submit issue) for better service" - exit 1 -fi - -function isSuccess(){ -if [ $? -ne 0 ]; then - echo "Failed to " + $1 - exit 1 -else - echo "Succeed to" + $1 -fi -} - -function checkJava(){ - java -version - isSuccess "execute java --version" -} - -function checkExternalServer(){ -echo "telnet check for your $SERVER_NAME, if you wait for a long time,may be your $SERVER_NAME does not prepared" -result=`echo -e "\n" | telnet $EXTERNAL_SERVER_IP $EXTERNAL_SERVER_PORT 2>/dev/null | grep Connected | wc -l` -if [ $result -eq 1 ]; then - echo "$SERVER_NAME is OK." -else - echo "$SERVER_NAME is Bad. You need to prepare the' $SERVER_NAME ' environment in advance" - exit 1 -fi -} - - -say() { - printf 'check command fail \n %s\n' "$1" -} - -err() { - say "$1" >&2 - exit 1 -} - -check_cmd() { - command -v "$1" > /dev/null 2>&1 -} - -need_cmd() { - if ! check_cmd "$1"; then - err "need '$1' (command not found)" - fi -} - -#check env -sh ${workDir}/bin/checkEnv.sh -isSuccess "check env" - -##load config -echo "step1:load config" -export DSS_CONFIG_PATH=${DSS_CONFIG_PATH:-"${workDir}/conf/config.sh"} -export DSS_DB_CONFIG_PATH=${DSS_DB_CONFIG_PATH:-"${workDir}/conf/db.sh"} -export DISTRIBUTION=${DISTRIBUTION:-"${workDir}/conf/config.sh"} -source ${DSS_CONFIG_PATH} -source ${DSS_DB_CONFIG_PATH} -source ${DISTRIBUTION} -isSuccess "load config" - -local_host="`hostname --fqdn`" -ipaddr=$(ip addr | awk '/^[0-9]+: / {}; /inet.*global/ {print gensub(/(.*)\/(.*)/, "\\1", "g", $2)}'|awk 'NR==1') - -function isLocal(){ - if [ "$1" == "127.0.0.1" ];then - return 0 - elif [ $1 == "localhost" ]; then - return 0 - elif [ $1 == $local_host ]; then - return 0 - elif [ $1 == $ipaddr ]; then - return 0 - fi - return 1 -} - -function executeCMD(){ - isLocal $1 - flag=$? - if [ $flag == "0" ];then - echo "Is local execution:$2" - eval $2 - else - echo "Is remote execution:$2" - ssh -p $SSH_PORT $1 $2 - fi -} - -function copyFile(){ - isLocal $1 - flag=$? - src=$2 - dest=$3 - if [ $flag == "0" ];then - echo "Is local cp " - eval "cp -r $src $dest" - else - echo "Is remote cp " - scp -r -P $SSH_PORT $src $1:$dest - fi -} - -##install mode choice -if [ "$INSTALL_MODE" == "" ];then - echo "Please enter the mode selection such as: 1" - echo " 1: Lite" - echo " 2: Simple" - echo " 3: Standard" - echo "" - read -p "Please input the choice:" idx - INSTALL_MODE=$idx -fi - -if [[ '1' = "$INSTALL_MODE" ]];then - echo "You chose lite installation mode" - checkJava - SERVER_NAME=MYSQL - EXTERNAL_SERVER_IP=$MYSQL_HOST - EXTERNAL_SERVER_PORT=$MYSQL_PORT - checkExternalServer -elif [[ '2' = "$INSTALL_MODE" ]];then - echo "You chose sample installation mode" - checkJava - SERVER_NAME=MYSQL - EXTERNAL_SERVER_IP=$MYSQL_HOST - EXTERNAL_SERVER_PORT=$MYSQL_PORT - checkExternalServer -elif [[ '3' = "$INSTALL_MODE" ]];then - echo "You chose Standard installation mode" - #check for Java - checkJava - #check for mysql - SERVER_NAME=MYSQL - EXTERNAL_SERVER_IP=$MYSQL_HOST - EXTERNAL_SERVER_PORT=$MYSQL_PORT - checkExternalServer - #check qualitis serivice - SERVER_NAME=Qualitis - EXTERNAL_SERVER_IP=$QUALITIS_ADRESS_IP - EXTERNAL_SERVER_PORT=$QUALITIS_ADRESS_PORT - if [[ $IGNORECHECK = "" ]];then - checkExternalServer - fi - #check azkaban serivice - SERVER_NAME=AZKABAN - EXTERNAL_SERVER_IP=$AZKABAN_ADRESS_IP - EXTERNAL_SERVER_PORT=$AZKABAN_ADRESS_PORT - if [[ $IGNORECHECK = "" ]];then - checkExternalServer - fi -else - echo "no choice,exit!" - exit 1 -fi - -##init db -echo "Do you want to clear DSS table information in the database?" -echo " 1: Do not execute table-building statements" -echo " 2: Dangerous! Clear all data and rebuild the tables." -echo "" - -MYSQL_INSTALL_MODE=1 - -read -p "Please input the choice:" idx -if [[ '2' = "$idx" ]];then - MYSQL_INSTALL_MODE=2 - echo "You chose Rebuild the table" -elif [[ '1' = "$idx" ]];then - MYSQL_INSTALL_MODE=1 - echo "You chose not execute table-building statements" -else - echo "no choice,exit!" - exit 1 -fi - -echo "create hdfs directory and local directory" -if [ "$WORKSPACE_USER_ROOT_PATH" != "" ] -then - localRootDir=$WORKSPACE_USER_ROOT_PATH - if [[ $WORKSPACE_USER_ROOT_PATH == file://* ]];then - localRootDir=${WORKSPACE_USER_ROOT_PATH#file://} - mkdir -p $localRootDir/$deployUser - sudo chmod -R 775 $localRootDir/$deployUser - elif [[ $WORKSPACE_USER_ROOT_PATH == hdfs://* ]];then - localRootDir=${WORKSPACE_USER_ROOT_PATH#hdfs://} - hdfs dfs -mkdir -p $localRootDir/$deployUser - hdfs dfs -chmod -R 775 $localRootDir/$deployUser - else - echo "does not support $WORKSPACE_USER_ROOT_PATH filesystem types" - fi -isSuccess "create $WORKSPACE_USER_ROOT_PATH directory" -fi - - -if [ "$RESULT_SET_ROOT_PATH" != "" ] -then - localRootDir=$RESULT_SET_ROOT_PATH - if [[ $RESULT_SET_ROOT_PATH == file://* ]];then - localRootDir=${RESULT_SET_ROOT_PATH#file://} - mkdir -p $localRootDir/$deployUser - sudo chmod -R 775 $localRootDir/$deployUser - elif [[ $RESULT_SET_ROOT_PATH == hdfs://* ]];then - localRootDir=${RESULT_SET_ROOT_PATH#hdfs://} - hdfs dfs -mkdir -p $localRootDir/$deployUser - hdfs dfs -chmod -R 775 $localRootDir/$deployUser - else - echo "does not support $RESULT_SET_ROOT_PATH filesystem types" - fi -isSuccess "create $RESULT_SET_ROOT_PATH directory" -fi - - -if [ "$WDS_SCHEDULER_PATH" != "" ] -then - localRootDir=$WDS_SCHEDULER_PATH - if [[ $WDS_SCHEDULER_PATH == file://* ]];then - localRootDir=${WDS_SCHEDULER_PATH#file://} - mkdir -p $localRootDir - sudo chmod -R 775 $localRootDir - elif [[ $WDS_SCHEDULER_PATH == hdfs://* ]];then - localRootDir=${WDS_SCHEDULER_PATH#hdfs://} - hdfs dfs -mkdir -p $localRootDir - hdfs dfs -chmod -R 775 $localRootDir - else - echo "does not support $WDS_SCHEDULER_PATH filesystem types" - fi -isSuccess "create $WDS_SCHEDULER_PATH directory" -fi - - -##init db -if [[ '2' = "$MYSQL_INSTALL_MODE" ]];then - mysql -h$MYSQL_HOST -P$MYSQL_PORT -u$MYSQL_USER -p$MYSQL_PASSWORD -D$MYSQL_DB --default-character-set=utf8 -e "source ${workDir}/db/dss_ddl.sql" - isSuccess "source dss_ddl.sql" - LOCAL_IP=$ipaddr - if [ $GATEWAY_INSTALL_IP == "127.0.0.1" ];then - echo "GATEWAY_INSTALL_IP is equals 127.0.0.1 ,we will change it to ip address" - GATEWAY_INSTALL_IP_2=$LOCAL_IP - else - GATEWAY_INSTALL_IP_2=$GATEWAY_INSTALL_IP - fi - #echo $GATEWAY_INSTALL_IP_2 - sed -i "s/GATEWAY_INSTALL_IP_2/$GATEWAY_INSTALL_IP_2/g" ${workDir}/db/dss_dml.sql - sed -i "s/GATEWAY_PORT/$GATEWAY_PORT/g" ${workDir}/db/dss_dml.sql - mysql -h$MYSQL_HOST -P$MYSQL_PORT -u$MYSQL_USER -p$MYSQL_PASSWORD -D$MYSQL_DB --default-character-set=utf8 -e "source ${workDir}/db/dss_dml.sql" - isSuccess "source dss_dml.sql" - - if [[ '2' = "$INSTALL_MODE" ]] || [[ '3' = "$INSTALL_MODE" ]];then - echo "visualis support,visualis database will be initialized !" - if [ $VISUALIS_NGINX_IP == "127.0.0.1" ]||[ $VISUALIS_NGINX_IP == "0.0.0.0" ];then - echo "VISUALIS_NGINX_IP is equals $VISUALIS_NGINX_IP ,we will change it to ip address" - VISUALIS_NGINX_IP_2=$LOCAL_IP - else - VISUALIS_NGINX_IP_2=$VISUALIS_NGINX_IP - fi - #echo $VISUALIS_NGINX_IP_2 - sed -i "s/VISUALIS_NGINX_IP_2/$VISUALIS_NGINX_IP_2/g" ${workDir}/db/visualis.sql - sed -i "s/VISUALIS_NGINX_PORT/$VISUALIS_NGINX_PORT/g" ${workDir}/db/visualis.sql - mysql -h$MYSQL_HOST -P$MYSQL_PORT -u$MYSQL_USER -p$MYSQL_PASSWORD -D$MYSQL_DB --default-character-set=utf8 -e "source ${workDir}/db/visualis.sql" - isSuccess "source visualis.sql" - mysql -h$MYSQL_HOST -P$MYSQL_PORT -u$MYSQL_USER -p$MYSQL_PASSWORD -D$MYSQL_DB --default-character-set=utf8 -e "source ${workDir}/db/davinci.sql" - isSuccess "source davinci.sql" - fi - - if [[ '3' = "$INSTALL_MODE" ]];then - echo "azkaban and qualitis support, azkaban and qualitis database will be initialized !" - #azkaban - if [ $AZKABAN_ADRESS_IP == "127.0.0.1" ];then - echo "AZKABAN_ADRESS_IP is equals 127.0.0.1 ,we will change it to ip address" - AZKABAN_ADRESS_IP_2=$LOCAL_IP - else - AZKABAN_ADRESS_IP_2=$AZKABAN_ADRESS_IP - fi - echo $AZKABAN_ADRESS_IP_2 - sed -i "s/AZKABAN_ADRESS_IP_2/$AZKABAN_ADRESS_IP_2/g" ${workDir}/db/azkaban.sql - sed -i "s/AZKABAN_ADRESS_PORT/$AZKABAN_ADRESS_PORT/g" ${workDir}/db/azkaban.sql - mysql -h$MYSQL_HOST -P$MYSQL_PORT -u$MYSQL_USER -p$MYSQL_PASSWORD -D$MYSQL_DB --default-character-set=utf8 -e "source ${workDir}/db/azkaban.sql" - isSuccess "source azkaban.sql" - #qualitis - if [ $QUALITIS_ADRESS_IP == "127.0.0.1" ];then - echo "QUALITIS_ADRESS_IP is equals 127.0.0.1 ,we will change it to ip address" - QUALITIS_ADRESS_IP_2=$LOCAL_IP - else - QUALITIS_ADRESS_IP_2=$QUALITIS_ADRESS_IP - fi - echo $QUALITIS_ADRESS_IP_2 - sed -i "s/QUALITIS_ADRESS_IP_2/$QUALITIS_ADRESS_IP_2/g" ${workDir}/db/qualitis.sql - sed -i "s/QUALITIS_ADRESS_PORT/$QUALITIS_ADRESS_PORT/g" ${workDir}/db/qualitis.sql - mysql -h$MYSQL_HOST -P$MYSQL_PORT -u$MYSQL_USER -p$MYSQL_PASSWORD -D$MYSQL_DB --default-character-set=utf8 -e "source ${workDir}/db/qualitis.sql" - isSuccess "source qualitis.sql" - fi -fi - -##Deal special symbol '#' -HIVE_META_PASSWORD=$(echo ${HIVE_META_PASSWORD//'#'/'\#'}) -MYSQL_PASSWORD=$(echo ${MYSQL_PASSWORD//'#'/'\#'}) - -###linkis Eurkea info -SERVER_IP=$EUREKA_INSTALL_IP -SERVER_PORT=$EUREKA_PORT -SERVER_HOME=$DSS_INSTALL_HOME - -if test -z "$SERVER_IP" -then - SERVER_IP=$local_host -fi -EUREKA_URL=http://$SERVER_IP:$EUREKA_PORT/eureka/ - -##function start -function installPackage(){ -echo "start to install $SERVERNAME" -echo "$SERVERNAME-step1: create dir" -if test -z "$SERVER_IP" -then - SERVER_IP=$local_host -fi - -if ! executeCMD $SERVER_IP "test -e $SERVER_HOME"; then - executeCMD $SERVER_IP "sudo mkdir -p $SERVER_HOME;sudo chown -R $deployUser:$deployUser $SERVER_HOME" - isSuccess "create the dir of $SERVERNAME" -fi - -echo "$SERVERNAME-step2:copy install package" -copyFile $SERVER_IP ${workDir}/share/$PACKAGE_DIR/$SERVERNAME.zip $SERVER_HOME - -if ! executeCMD $SERVER_IP "test -e $SERVER_HOME/lib"; then - copyFile $SERVER_IP ${workDir}/lib $SERVER_HOME -fi - -#copyFile $SERVER_IP ${workDir}/lib $SERVER_HOME -isSuccess "copy ${SERVERNAME}.zip" -executeCMD $SERVER_IP "cd $SERVER_HOME/;rm -rf $SERVERNAME-bak; mv -f $SERVERNAME $SERVERNAME-bak" -executeCMD $SERVER_IP "cd $SERVER_HOME/;unzip $SERVERNAME.zip > /dev/null" -executeCMD $SERVER_IP "cd $SERVER_HOME/;scp -r lib/* $SERVER_HOME/$SERVERNAME/lib" -isSuccess "unzip ${SERVERNAME}.zip" - -echo "$SERVERNAME-step3:subsitution conf" -SERVER_CONF_PATH=$SERVER_HOME/$SERVERNAME/conf/application.yml -executeCMD $SERVER_IP "sed -i \"s#port:.*#port: $SERVER_PORT#g\" $SERVER_CONF_PATH" -executeCMD $SERVER_IP "sed -i \"s#defaultZone:.*#defaultZone: $EUREKA_URL#g\" $SERVER_CONF_PATH" -executeCMD $SERVER_IP "sed -i \"s#hostname:.*#hostname: $SERVER_IP#g\" $SERVER_CONF_PATH" -isSuccess "subsitution conf of $SERVERNAME" -} -##function end - -##function start -function installVisualis(){ -echo "start to install $SERVERNAME" -echo "$SERVERNAME-step1: create dir" -if test -z "$SERVER_IP" -then - SERVER_IP=$local_host -fi - -if ! executeCMD $SERVER_IP "test -e $SERVER_HOME"; then - executeCMD $SERVER_IP "sudo mkdir -p $SERVER_HOME;sudo chown -R $deployUser:$deployUser $SERVER_HOME" - isSuccess "create the dir of $SERVERNAME" -fi - -echo "$SERVERNAME-step2:copy install package" -copyFile $SERVER_IP ${workDir}/share/$PACKAGE_DIR/$SERVERNAME.zip $SERVER_HOME -isSuccess "copy ${SERVERNAME}.zip" -executeCMD $SERVER_IP "cd $SERVER_HOME/;rm -rf $SERVERNAME-bak; mv -f $SERVERNAME $SERVERNAME-bak" -executeCMD $SERVER_IP "cd $SERVER_HOME/;unzip $SERVERNAME.zip > /dev/null" -isSuccess "unzip ${SERVERNAME}.zip" -} -##function end - - -##function start -function installAppjoints(){ -echo "start to install $APPJOINTNAME" -echo "$APPJOINTNAME Install-step1: create dir" -if test -z "$SERVER_IP" -then - SERVER_IP=$local_host -fi - -if ! executeCMD $SERVER_IP "test -e $SERVER_HOME/$APPJOINTPARENT"; then - executeCMD $SERVER_IP "sudo mkdir -p $SERVER_HOME/$APPJOINTPARENT;sudo chown -R $deployUser:$deployUser $SERVER_HOME/$APPJOINTPARENT" - isSuccess "create the dir of $SERVER_HOME/$APPJOINTPARENT;" -fi - -echo "$APPJOINTNAME-step2:copy install package" -copyFile $SERVER_IP $workDir/share/appjoints/$APPJOINTNAME/*.zip $SERVER_HOME/$APPJOINTPARENT -isSuccess "copy ${APPJOINTNAME}.zip" -executeCMD $SERVER_IP "cd $SERVER_HOME/$APPJOINTPARENT/;unzip -o dss-*-appjoint.zip > /dev/null;rm -rf dss-*-appjoint.zip" -isSuccess "install ${APPJOINTNAME}.zip" -} -##function end - -##dss-Server install -PACKAGE_DIR=dss/dss-server -SERVERNAME=dss-server -SERVER_IP=$DSS_SERVER_INSTALL_IP -SERVER_PORT=$DSS_SERVER_PORT -SERVER_HOME=$DSS_INSTALL_HOME -###install Dss-Server -installPackage -###update Dss-Server linkis.properties -echo "$SERVERNAME-step4:update linkis.properties" -SERVER_CONF_PATH=$SERVER_HOME/$SERVERNAME/conf/linkis.properties -executeCMD $SERVER_IP "sed -i \"s#wds.linkis.server.mybatis.datasource.url.*#wds.linkis.server.mybatis.datasource.url=jdbc:mysql://${MYSQL_HOST}:${MYSQL_PORT}/${MYSQL_DB}?characterEncoding=UTF-8#g\" $SERVER_CONF_PATH" -executeCMD $SERVER_IP "sed -i \"s#wds.linkis.server.mybatis.datasource.username.*#wds.linkis.server.mybatis.datasource.username=$MYSQL_USER#g\" $SERVER_CONF_PATH" -executeCMD $SERVER_IP "sed -i \"s#wds.linkis.server.mybatis.datasource.password.*#wds.linkis.server.mybatis.datasource.password=$MYSQL_PASSWORD#g\" $SERVER_CONF_PATH" -executeCMD $SERVER_IP "sed -i \"s#wds.dss.appjoint.scheduler.azkaban.address.*#wds.dss.appjoint.scheduler.azkaban.address=http://${AZKABAN_ADRESS_IP}:${AZKABAN_ADRESS_PORT}#g\" $SERVER_CONF_PATH" -executeCMD $SERVER_IP "sed -i \"s#wds.linkis.gateway.ip.*#wds.linkis.gateway.ip=$GATEWAY_INSTALL_IP#g\" $SERVER_CONF_PATH" -executeCMD $SERVER_IP "sed -i \"s#wds.linkis.gateway.port.*#wds.linkis.gateway.port=$GATEWAY_PORT#g\" $SERVER_CONF_PATH" -executeCMD $SERVER_IP "sed -i \"s#wds.dss.appjoint.scheduler.project.store.dir.*#wds.dss.appjoint.scheduler.project.store.dir=$WDS_SCHEDULER_PATH#g\" $SERVER_CONF_PATH" -executeCMD $SERVER_IP "echo "$deployUser=$deployUser" >> $SERVER_HOME/$SERVERNAME/conf/token.properties" -isSuccess "subsitution linkis.properties of $SERVERNAME" -echo "<----------------$SERVERNAME:end------------------->" -echo "" - -if [[ '2' = "$INSTALL_MODE" ]]||[[ '3' = "$INSTALL_MODE" ]];then -##Flow execution Install -PACKAGE_DIR=dss/dss-flow-execution-entrance -SERVERNAME=dss-flow-execution-entrance -SERVER_IP=$FLOW_EXECUTION_INSTALL_IP -SERVER_PORT=$FLOW_EXECUTION_PORT -SERVER_HOME=$DSS_INSTALL_HOME -###Install flow execution -installPackage -###Update flow execution linkis.properties -echo "$SERVERNAME-step4:update linkis.properties" -SERVER_CONF_PATH=$SERVER_HOME/$SERVERNAME/conf/linkis.properties -executeCMD $SERVER_IP "sed -i \"s#wds.linkis.entrance.config.logPath.*#wds.linkis.entrance.config.logPath=$WORKSPACE_USER_ROOT_PATH#g\" $SERVER_CONF_PATH" -executeCMD $SERVER_IP "sed -i \"s#wds.linkis.resultSet.store.path.*#wds.linkis.resultSet.store.path=$RESULT_SET_ROOT_PATH#g\" $SERVER_CONF_PATH" -executeCMD $SERVER_IP "sed -i \"s#wds.linkis.gateway.url.*#wds.linkis.gateway.url=http://${GATEWAY_INSTALL_IP}:${GATEWAY_PORT}#g\" $SERVER_CONF_PATH" -isSuccess "subsitution linkis.properties of $SERVERNAME" -echo "<----------------$SERVERNAME:end------------------->" -echo "" -##Appjoint entrance Install -PACKAGE_DIR=plugins/linkis/linkis-appjoint-entrance -SERVERNAME=linkis-appjoint-entrance -SERVER_IP=$APPJOINT_ENTRANCE_INSTALL_IP -SERVER_PORT=$APPJOINT_ENTRANCE_PORT -SERVER_HOME=$DSS_INSTALL_HOME -###Install appjoint entrance -installPackage -###Update appjoint entrance linkis.properties -echo "$SERVERNAME-step4:update linkis.properties" -SERVER_CONF_PATH=$SERVER_HOME/$SERVERNAME/conf/linkis.properties -executeCMD $SERVER_IP "sed -i \"s#wds.linkis.entrance.config.logPath.*#wds.linkis.entrance.config.logPath=$WORKSPACE_USER_ROOT_PATH#g\" $SERVER_CONF_PATH" -executeCMD $SERVER_IP "sed -i \"s#wds.linkis.resultSet.store.path.*#wds.linkis.resultSet.store.path=$RESULT_SET_ROOT_PATH#g\" $SERVER_CONF_PATH" -isSuccess "subsitution linkis.properties of $SERVERNAME" -echo "<----------------$SERVERNAME:end------------------->" -echo "" -##visualis-server Install -PACKAGE_DIR=visualis-server -SERVERNAME=visualis-server -SERVER_IP=$VISUALIS_SERVER_INSTALL_IP -SERVER_PORT=$VISUALIS_SERVER_PORT -SERVER_HOME=$DSS_INSTALL_HOME -###install visualis-server -installVisualis -###update visualis-server linkis.properties -echo "$SERVERNAME-step4:update linkis.properties" -SERVER_CONF_PATH=$SERVER_HOME/$SERVERNAME/conf/linkis.properties -if [ $VISUALIS_NGINX_IP == "127.0.0.1" ]||[ $VISUALIS_NGINX_IP == "0.0.0.0" ]; then - VISUALIS_NGINX_IP=$ipaddr -fi -if [ $VISUALIS_SERVER_INSTALL_IP == "127.0.0.1" ]||[ $VISUALIS_SERVER_INSTALL_IP == "0.0.0.0" ]; then - VISUALIS_SERVER_INSTALL_IP=$ipaddr -fi -executeCMD $SERVER_IP "sed -i \"s#wds.linkis.entrance.config.logPath.*#wds.linkis.entrance.config.logPath=$WORKSPACE_USER_ROOT_PATH#g\" $SERVER_CONF_PATH" -executeCMD $SERVER_IP "sed -i \"s#wds.linkis.resultSet.store.path.*#wds.linkis.resultSet.store.path=$RESULT_SET_ROOT_PATH#g\" $SERVER_CONF_PATH" -executeCMD $SERVER_IP "sed -i \"s#wds.dss.visualis.gateway.ip.*#wds.dss.visualis.gateway.ip=$GATEWAY_INSTALL_IP#g\" $SERVER_CONF_PATH" -executeCMD $SERVER_IP "sed -i \"s#wds.dss.visualis.gateway.port.*#wds.dss.visualis.gateway.port=$GATEWAY_PORT#g\" $SERVER_CONF_PATH" -SERVER_CONF_PATH=$SERVER_HOME/$SERVERNAME/conf/application.yml -executeCMD $SERVER_IP "sed -i \"s#address: 127.0.0.1#address: $VISUALIS_SERVER_INSTALL_IP#g\" $SERVER_CONF_PATH" -executeCMD $SERVER_IP "sed -i \"s#port: 9007#port: $VISUALIS_SERVER_PORT#g\" $SERVER_CONF_PATH" -executeCMD $SERVER_IP "sed -i \"s#url: http://0.0.0.0:0000/dss/visualis#url: http://$VISUALIS_NGINX_IP:$VISUALIS_NGINX_PORT/dss/visualis#g\" $SERVER_CONF_PATH" -executeCMD $SERVER_IP "sed -i \"s#address: 0.0.0.0#address: $VISUALIS_NGINX_IP#g\" $SERVER_CONF_PATH" -executeCMD $SERVER_IP "sed -i \"s#port: 0000#port: $VISUALIS_NGINX_PORT#g\" $SERVER_CONF_PATH" -executeCMD $SERVER_IP "sed -i \"s#defaultZone: http://127.0.0.1:20303/eureka/#defaultZone: http://$EUREKA_INSTALL_IP:$EUREKA_PORT/eureka/#g\" $SERVER_CONF_PATH" -executeCMD $SERVER_IP "sed -i \"s#url: jdbc:mysql://127.0.0.1:3306/xxx?characterEncoding=UTF-8#url: jdbc:mysql://$MYSQL_HOST:$MYSQL_PORT/$MYSQL_DB?characterEncoding=UTF-8#g\" $SERVER_CONF_PATH" -executeCMD $SERVER_IP "sed -i \"s#username: xxx#username: $MYSQL_USER#g\" $SERVER_CONF_PATH" -executeCMD $SERVER_IP "sed -i \"s#password: xxx#password: $MYSQL_PASSWORD#g\" $SERVER_CONF_PATH" -isSuccess "subsitution linkis.properties of $SERVERNAME" -echo "<----------------$SERVERNAME:end------------------->" -echo "" -#APPJOINTS INSTALL -echo "<----------------datachecker appjoint install start------------------->" -APPJOINTPARENT=dss-appjoints -APPJOINTNAME=datachecker -#datachecker appjoint install -installAppjoints -echo "$APPJOINTNAME:subsitution conf" -APPJOINTNAME_CONF_PATH_PATENT=$SERVER_HOME/$APPJOINTPARENT/$APPJOINTNAME/appjoint.properties -executeCMD $SERVER_IP "sed -i \"s#job.datachecker.jdo.option.url.*#job.datachecker.jdo.option.url=$HIVE_META_URL#g\" $APPJOINTNAME_CONF_PATH_PATENT" -executeCMD $SERVER_IP "sed -i \"s#job.datachecker.jdo.option.username.*#job.datachecker.jdo.option.username=$HIVE_META_USER#g\" $APPJOINTNAME_CONF_PATH_PATENT" -executeCMD $SERVER_IP "sed -i \"s#job.datachecker.jdo.option.password.*#job.datachecker.jdo.option.password=$HIVE_META_PASSWORD#g\" $APPJOINTNAME_CONF_PATH_PATENT" -isSuccess "subsitution conf of datachecker" -echo "<----------------datachecker appjoint install end------------------->" -echo "" -echo "<----------------eventchecker appjoint install start------------------->" -APPJOINTPARENT=dss-appjoints -APPJOINTNAME=eventchecker -#eventchecker appjoint install -installAppjoints -echo "$APPJOINTNAME:subsitution conf" -APPJOINTNAME_CONF_PATH_PATENT=$SERVER_HOME/$APPJOINTPARENT/$APPJOINTNAME/appjoint.properties -executeCMD $SERVER_IP "sed -i \"s#msg.eventchecker.jdo.option.url.*#msg.eventchecker.jdo.option.url=jdbc:mysql://${MYSQL_HOST}:${MYSQL_PORT}/${MYSQL_DB}?characterEncoding=UTF-8#g\" $APPJOINTNAME_CONF_PATH_PATENT" -executeCMD $SERVER_IP "sed -i \"s#msg.eventchecker.jdo.option.username.*#msg.eventchecker.jdo.option.username=$MYSQL_USER#g\" $APPJOINTNAME_CONF_PATH_PATENT" -executeCMD $SERVER_IP "sed -i \"s#msg.eventchecker.jdo.option.password.*#msg.eventchecker.jdo.option.password=$MYSQL_PASSWORD#g\" $APPJOINTNAME_CONF_PATH_PATENT" -isSuccess "subsitution conf of eventchecker" -echo "<----------------$APPJOINTNAME:end------------------->" -echo "" -echo "<----------------visualis appjoint install start------------------->" -APPJOINTPARENT=dss-appjoints -APPJOINTNAME=visualis -#visualis appjoint install -installAppjoints -echo "<----------------$APPJOINTNAME:end------------------->" -fi - -##lite and sample version does not install qualitis APPJoint and scheduis APPJoint -if [[ '3' = "$INSTALL_MODE" ]];then -echo "" -echo "<----------------qualitis appjoint install start------------------->" -APPJOINTPARENT=dss-appjoints -APPJOINTNAME=qualitis -#qualitis appjoint install -installAppjoints -APPJOINTNAME_CONF_PATH_PATENT=$SERVER_HOME/$APPJOINTPARENT/$APPJOINTNAME/appjoint.properties -executeCMD $SERVER_IP "sed -i \"s#baseUrl=http://127.0.0.1:8090#baseUrl=http://$QUALITIS_ADRESS_IP:$QUALITIS_ADRESS_PORT#g\" $APPJOINTNAME_CONF_PATH_PATENT" -isSuccess "subsitution conf of qualitis" -echo "<----------------$APPJOINTNAME:end------------------->" -echo "" -echo "<----------------schedulis appjoint install start------------------->" -APPJOINTPARENT=dss-appjoints -APPJOINTNAME=schedulis -#schedulis appjoint install -installAppjoints -isSuccess "subsitution conf of schedulis" -echo "<----------------$APPJOINTNAME:end------------------->" -fi diff --git a/bin/start-all.sh b/bin/start-all.sh deleted file mode 100644 index 98a07f6bdc..0000000000 --- a/bin/start-all.sh +++ /dev/null @@ -1,189 +0,0 @@ -#!/usr/bin/env bash -# -# Copyright 2019 WeBank -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -# Start all dss applications -info="We will start all dss applications, it will take some time, please wait" -echo ${info} - -#Actively load user env -source /etc/profile -source ~/.bash_profile - -shellDir=`dirname $0` - -workDir=`cd ${shellDir}/..;pwd` - -CONF_DIR="${workDir}"/conf - -export LINKIS_DSS_CONF_FILE=${LINKIS_DSS_CONF_FILE:-"${CONF_DIR}/config.sh"} -export DISTRIBUTION=${DISTRIBUTION:-"${CONF_DIR}/config.sh"} -source $LINKIS_DSS_CONF_FILE -source ${DISTRIBUTION} -function isSuccess(){ -if [ $? -ne 0 ]; then - echo "ERROR: " + $1 - exit 1 -else - echo "INFO:" + $1 -fi -} -local_host="`hostname --fqdn`" - -ipaddr=$(ip addr | awk '/^[0-9]+: / {}; /inet.*global/ {print gensub(/(.*)\/(.*)/, "\\1", "g", $2)}'|awk 'NR==1') - -function isLocal(){ - if [ "$1" == "127.0.0.1" ];then - return 0 - elif [ $1 == "localhost" ]; then - return 0 - elif [ $1 == $local_host ]; then - return 0 - elif [ $1 == $ipaddr ]; then - return 0 - fi - return 1 -} - -function executeCMD(){ - isLocal $1 - flag=$? - echo "Is local "$flag - if [ $flag == "0" ];then - eval $2 - else - ssh -p $SSH_PORT $1 $2 - fi - -} - -#if there is no LINKIS_INSTALL_HOME，we need to source config again -if [ -z ${DSS_INSTALL_HOME} ];then - echo "Warning: DSS_INSTALL_HOME does not exist, we will source config" - if [ ! -f "${LINKIS_DSS_CONF_FILE}" ];then - echo "Error: can not find config file, start applications failed" - exit 1 - else - source ${LINKIS_DSS_CONF_FILE} - fi -fi - -function startApp(){ -echo "<-------------------------------->" -echo "Begin to start $SERVER_NAME" -SERVER_BIN=${DSS_INSTALL_HOME}/${SERVER_NAME}/bin -#echo $SERVER_BIN -SERVER_LOCAL_START_CMD="dos2unix ${SERVER_BIN}/* > /dev/null 2>&1; dos2unix ${SERVER_BIN}/../conf/* > /dev/null 2>&1;sh ${SERVER_BIN}/start-${SERVER_NAME}.sh > /dev/null 2>&1 &" -SERVER_REMOTE_START_CMD="source /etc/profile;source ~/.bash_profile;cd ${SERVER_BIN}; dos2unix ./* > /dev/null 2>&1; dos2unix ../conf/* > /dev/null 2>&1; sh start-${SERVER_NAME}.sh > /dev/null 2>&1" - -if test -z "$SERVER_IP" -then - SERVER_IP=$local_host -fi - -if ! executeCMD $SERVER_IP "test -e $SERVER_BIN"; then - echo "<-------------------------------->" - echo "$SERVER_NAME is not installed,the start steps will be skipped" - echo "<-------------------------------->" - return -fi - -isLocal $SERVER_IP -flag=$? -echo "Is local "$flag -if [ $flag == "0" ];then - eval $SERVER_LOCAL_START_CMD -else - ssh -p $SSH_PORT $SERVER_IP $SERVER_REMOTE_START_CMD -fi -isSuccess "End to start $SERVER_NAME" -echo "<-------------------------------->" -sleep 15 #for Eureka register -} - -#dss-server -SERVER_NAME=dss-server -SERVER_IP=$DSS_SERVER_INSTALL_IP -startApp - -#dss-flow-execution-entrance -SERVER_NAME=dss-flow-execution-entrance -SERVER_IP=$FLOW_EXECUTION_INSTALL_IP -startApp - -#dss-flow-execution-entrance -SERVER_NAME=linkis-appjoint-entrance -SERVER_IP=$APPJOINT_ENTRANCE_INSTALL_IP -startApp - -#visualis-server -SERVER_NAME=visualis-server -SERVER_IP=$VISUALIS_SERVER_INSTALL_IP -startApp - -echo "" -echo "Start to check all dss microservice" -echo "" - -function checkServer(){ -echo "<-------------------------------->" -echo "Begin to check $SERVER_NAME" -if test -z "$SERVER_IP" -then - SERVER_IP=$local_host -fi - -SERVER_BIN=${SERVER_HOME}/${SERVER_NAME}/bin - -if ! executeCMD $SERVER_IP "test -e ${DSS_INSTALL_HOME}/${SERVER_NAME}"; then - echo "$SERVER_NAME is not installed,the checkServer steps will be skipped" - return -fi - -sh $workDir/bin/checkServices.sh $SERVER_NAME $SERVER_IP $SERVER_PORT -isSuccess "start $SERVER_NAME " -sleep 3 -echo "<-------------------------------->" -} - -#check dss-server -SERVER_NAME=dss-server -SERVER_IP=$DSS_SERVER_INSTALL_IP -SERVER_PORT=$DSS_SERVER_PORT -checkServer - - -#check dss-flow-execution-entrance -SERVER_NAME=dss-flow-execution-entrance -SERVER_IP=$FLOW_EXECUTION_INSTALL_IP -SERVER_PORT=$FLOW_EXECUTION_PORT -checkServer - -#check linkis-appjoint-entrance -SERVER_NAME=linkis-appjoint-entrance -SERVER_IP=$APPJOINT_ENTRANCE_INSTALL_IP -SERVER_PORT=$APPJOINT_ENTRANCE_PORT -checkServer - - -#check visualis-server -sleep 10 #visualis service need more time to register -SERVER_NAME=visualis-server -SERVER_IP=$VISUALIS_SERVER_INSTALL_IP -SERVER_PORT=$VISUALIS_SERVER_PORT -checkServer - -echo "DSS started successfully" diff --git a/bin/stop-all.sh b/bin/stop-all.sh deleted file mode 100644 index 838b9babc9..0000000000 --- a/bin/stop-all.sh +++ /dev/null @@ -1,133 +0,0 @@ -#!/usr/bin/env bash -# -# Copyright 2019 WeBank -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - - - -# Start all dss applications -info="We will stop all dss applications, it will take some time, please wait" -echo ${info} - -#Actively load user env -source ~/.bash_profile - -workDir=`dirname "${BASH_SOURCE-$0}"` -workDir=`cd "$workDir"; pwd` - - -CONF_DIR="${workDir}"/../conf -export LINKIS_DSS_CONF_FILE=${LINKIS_DSS_CONF_FILE:-"${CONF_DIR}/config.sh"} -export DISTRIBUTION=${DISTRIBUTION:-"${CONF_DIR}/config.sh"} -source ${DISTRIBUTION} - -local_host="`hostname --fqdn`" -ipaddr=$(ip addr | awk '/^[0-9]+: / {}; /inet.*global/ {print gensub(/(.*)\/(.*)/, "\\1", "g", $2)}'|awk 'NR==1') - -function isSuccess(){ -if [ $? -ne 0 ]; then - echo "ERROR: " + $1 - exit 1 -else - echo "INFO:" + $1 -fi -} - -function isLocal(){ - if [ "$1" == "127.0.0.1" ];then - return 0 - elif [ $1 == "localhost" ]; then - return 0 - elif [ $1 == $local_host ]; then - return 0 - elif [ $1 == $ipaddr ]; then - return 0 - fi - return 1 -} - -function executeCMD(){ - isLocal $1 - flag=$? - echo "Is local "$flag - if [ $flag == "0" ];then - eval $2 - else - ssh -p $SSH_PORT $1 $2 - fi - -} - - -#if there is no LINKIS_INSTALL_HOME，we need to source config again -if [ -z ${DSS_INSTALL_HOME} ];then - echo "Warning: DSS_INSTALL_HOME does not exist, we will source config" - if [ ! -f "${LINKIS_DSS_CONF_FILE}" ];then - echo "Error: can not find config file, stop applications failed" - exit 1 - else - source ${LINKIS_DSS_CONF_FILE} - fi -fi - -function stopAPP(){ -echo "<-------------------------------->" -echo "Begin to stop $SERVER_NAME" -SERVER_BIN=${DSS_INSTALL_HOME}/${SERVER_NAME}/bin -SERVER_LOCAL_STOP_CMD="sh ${SERVER_BIN}/stop-${SERVER_NAME}.sh" -SERVER_REMOTE_STOP_CMD="source /etc/profile;source ~/.bash_profile;cd ${SERVER_BIN}; sh stop-${SERVER_NAME}.sh " -if test -z "$SERVER_IP" -then - SERVER_IP=$local_host -fi - -if ! executeCMD $SERVER_IP "test -e ${DSS_INSTALL_HOME}/${SERVER_NAME}"; then - echo "$SERVER_NAME is not installed,the stop steps will be skipped" - return -fi - -isLocal $SERVER_IP -flag=$? -echo "Is local "$flag -if [ $flag == "0" ];then - eval $SERVER_LOCAL_STOP_CMD -else - ssh -p $SSH_PORT $SERVER_IP $SERVER_REMOTE_STOP_CMD -fi -echo "<-------------------------------->" -sleep 3 -} - -#dss-server -SERVER_NAME=dss-server -SERVER_IP=$DSS_SERVER_INSTALL_IP -stopAPP - -#dss-flow-execution-entrance -SERVER_NAME=dss-flow-execution-entrance -SERVER_IP=$FLOW_EXECUTION_INSTALL_IP -stopAPP - -#dss-flow-execution-entrance -SERVER_NAME=linkis-appjoint-entrance -SERVER_IP=$APPJOINT_ENTRANCE_INSTALL_IP -stopAPP - -#visualis-server -SERVER_NAME=visualis-server -SERVER_IP=$VISUALIS_SERVER_INSTALL_IP -stopAPP - -echo "stop-all shell script executed completely" diff --git a/datachecker-appjoint/pom.xml b/datachecker-appjoint/pom.xml deleted file mode 100644 index d609c3ab32..0000000000 --- a/datachecker-appjoint/pom.xml +++ /dev/null @@ -1,139 +0,0 @@ - - - - - - dss - com.webank.wedatasphere.dss - 0.9.1 - - 4.0.0 - - dss-datachecker-appjoint - - - - org.apache.commons - commons-lang3 - 3.4 - - - - com.alibaba - druid - 1.0.28 - - - - com.webank.wedatasphere.dss - dss-appjoint-core - ${dss.version} - provided - true - - - - - - - - - - - - - - - - log4j - log4j - 1.2.17 - - - - - - - - - - org.apache.maven.plugins - maven-deploy-plugin - - - - net.alchim31.maven - scala-maven-plugin - - - org.apache.maven.plugins - maven-jar-plugin - - - org.apache.maven.plugins - maven-assembly-plugin - 2.3 - false - - - make-assembly - package - - single - - - - src/main/assembly/distribution.xml - - - - - - false - dss-datachecker-appjoint - false - false - - src/main/assembly/distribution.xml - - - - - - - src/main/java - - **/*.xml - - - - src/main/resources - - **/*.properties - **/application.yml - **/bootstrap.yml - **/log4j2.xml - - - - - - - \ No newline at end of file diff --git a/datachecker-appjoint/src/main/assembly/distribution.xml b/datachecker-appjoint/src/main/assembly/distribution.xml deleted file mode 100644 index 503ab0f381..0000000000 --- a/datachecker-appjoint/src/main/assembly/distribution.xml +++ /dev/null @@ -1,136 +0,0 @@ - - - - dss-datachecker-appjoint - - zip - - true - datachecker - - - - - - lib - true - true - false - true - true - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - ${basedir}/src/main/resources - - appjoint.properties - - 0777 - / - unix - - - - ${basedir}/src/main/resources - - log4j.properties - log4j2.xml - - 0777 - conf - unix - - - - - - diff --git a/datachecker-appjoint/src/main/java/com/webank/wedatasphere/dss/appjoint/schedulis/jobtype/connector/DataCheckerDao.java b/datachecker-appjoint/src/main/java/com/webank/wedatasphere/dss/appjoint/schedulis/jobtype/connector/DataCheckerDao.java deleted file mode 100644 index d84a0d4bc6..0000000000 --- a/datachecker-appjoint/src/main/java/com/webank/wedatasphere/dss/appjoint/schedulis/jobtype/connector/DataCheckerDao.java +++ /dev/null @@ -1,195 +0,0 @@ -/* - * Copyright 2019 WeBank - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package com.webank.wedatasphere.dss.appjoint.schedulis.jobtype.connector; - -import com.alibaba.druid.pool.DruidDataSource; -import com.webank.wedatasphere.dss.appjoint.schedulis.jobtype.entity.DataChecker; -import org.apache.log4j.Logger; - -import javax.sql.DataSource; -import java.sql.Connection; -import java.sql.PreparedStatement; -import java.sql.ResultSet; -import java.sql.SQLException; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Properties; -import java.util.regex.Matcher; -import java.util.regex.Pattern; -import java.util.stream.Collectors; - -public class DataCheckerDao { - - private static final String SQL_SOURCE_TYPE_JOB_TABLE = - "SELECT * FROM DBS d JOIN TBLS t ON t.DB_ID = d.DB_ID WHERE d.NAME=? AND t.TBL_NAME=?"; - - private static final String SQL_SOURCE_TYPE_JOB_PARTITION = - "SELECT * FROM DBS d JOIN TBLS t ON t.DB_ID = d.DB_ID JOIN PARTITIONS p ON p.TBL_ID = t.TBL_ID WHERE d.NAME=? AND t.TBL_NAME=? AND p.PART_NAME=?"; - - private static DataSource jobDS; - private static DataCheckerDao instance; - - public static DataCheckerDao getInstance() { - if (instance == null) { - synchronized (DataCheckerDao.class) { - if (instance == null) { - instance = new DataCheckerDao(); - } - } - } - return instance; - } - - public boolean validateTableStatusFunction(Properties props, Logger log) { - if (jobDS == null) { - jobDS = DataDruidFactory.getJobInstance(props, log); - if (jobDS == null) { - log.error("Error getting Druid DataSource instance"); - return false; - } - } - removeBlankSpace(props); - log.info("=============================Data Check Start=========================================="); - String dataCheckerInfo = props.getProperty(DataChecker.DATA_OBJECT); - log.info("(DataChecker info) database table partition info : " + dataCheckerInfo); - long waitTime = Long.valueOf(props.getProperty(DataChecker.WAIT_TIME, "1")) * 3600 * 1000; - int queryFrequency = Integer.valueOf(props.getProperty(DataChecker.QUERY_FREQUENCY, "30000")); -// String timeScape = props.getProperty(DataChecker.TIME_SCAPE, "NULL"); - log.info("(DataChecker info) wait time : " + waitTime); - log.info("(DataChecker info) query frequency : " + queryFrequency); -// log.info("(DataChecker info) time scape : " + timeScape); - List> dataObjectList = extractProperties(props); - try (Connection conn = jobDS.getConnection()) { - boolean flag = dataObjectList - .stream() - .allMatch(proObjectMap -> { - long count = getTotalCount(proObjectMap, conn, log); - return count > 0; - }); - if (flag){ - log.info("=============================Data Check End=========================================="); - return true; - } - - } catch (SQLException e) { - throw new RuntimeException("get DataChecker result failed", e); - } - - log.info("=============================Data Check End=========================================="); - return false; - } - - private void sleep(long sleepTime) { - try { - Thread.sleep(sleepTime); - } catch (InterruptedException e) { - e.printStackTrace(); - } - } - - private void removeBlankSpace(Properties props) { - try { - props.entrySet().forEach(entry -> { - String value = entry.getValue().toString().replaceAll(" ", "").trim(); - entry.setValue(value); - }); - }catch (Exception e){ - throw new RuntimeException("remove job space char failed",e); - } - } - - private List> extractProperties(Properties p) { - return p.keySet().stream() - .map(key -> key2Map(key, p)).filter(x -> x.size() >0) - .collect(Collectors.toList()); - } - - private Map key2Map(Object key, Properties p) { - Map proMap = new HashMap<>(); - String skey = String.valueOf(key); - if(skey.contains(DataChecker.DATA_OBJECT)){ - String[] keyArr = skey.split("\\."); - if(keyArr.length == 3){ - String keyNum = keyArr[2]; - String doKey = DataChecker.DATA_OBJECT + "." + keyNum; - proMap.put(DataChecker.DATA_OBJECT, String.valueOf(p.get(doKey))); - }else{ - String doKey = DataChecker.DATA_OBJECT; - proMap.put(DataChecker.DATA_OBJECT, String.valueOf(p.get(doKey))); - } - } - return proMap; - } - - private long getTotalCount(Map proObjectMap, Connection conn, Logger log) { - String dataObject = proObjectMap.get(DataChecker.DATA_OBJECT); - if(dataObject != null) { - dataObject = dataObject.replace(" ", "").trim(); - }else{ - log.error("DataObject is null"); - return 0; - } - log.info("-------------------------------------- search hive/spark/mr data "); - log.info("-------------------------------------- : " + dataObject); - try (PreparedStatement pstmt = getStatement(conn, dataObject)) { - ResultSet rs = pstmt.executeQuery(); - return rs.last() ? rs.getRow() : 0; - } catch (SQLException e) { - log.error("fetch data from Hive MetaStore error", e); - return 0; - } - } - - private PreparedStatement getStatement(Connection conn, String dataObject) throws SQLException { - String dataScape = dataObject.contains("{") ? "Partition" : "Table"; - String[] dataObjectArray = dataObject.split("\\."); - String dbName = dataObjectArray[0]; - String tableName = dataObjectArray[1]; - if(dataScape.equals("Partition")) { - Pattern pattern = Pattern.compile("\\{([^\\}]+)\\}"); - Matcher matcher = pattern.matcher(dataObject); - String partitionName = null; - if(matcher.find()){ - partitionName = matcher.group(1); - } - partitionName = partitionName.replace("\'", "").replace("\"", ""); - tableName = tableName.split("\\{")[0]; - PreparedStatement pstmt = conn.prepareCall(SQL_SOURCE_TYPE_JOB_PARTITION); - pstmt.setString(1, dbName); - pstmt.setString(2, tableName); - pstmt.setString(3, partitionName); - return pstmt; - } else if(dataObjectArray.length == 2){ - PreparedStatement pstmt = conn.prepareCall(SQL_SOURCE_TYPE_JOB_TABLE); - pstmt.setString(1, dbName); - pstmt.setString(2, tableName); - return pstmt; - }else { - throw new SQLException("Incorrect input format for dataObject "+ dataObject); - } - } - - public static void closeDruidDataSource(){ - DruidDataSource jobDSObject = (DruidDataSource)jobDS; - if(jobDSObject != null){ - jobDSObject.close(); - } - } - -} diff --git a/datachecker-appjoint/src/main/java/com/webank/wedatasphere/dss/appjoint/schedulis/jobtype/connector/DataDruidFactory.java b/datachecker-appjoint/src/main/java/com/webank/wedatasphere/dss/appjoint/schedulis/jobtype/connector/DataDruidFactory.java deleted file mode 100644 index b55868125e..0000000000 --- a/datachecker-appjoint/src/main/java/com/webank/wedatasphere/dss/appjoint/schedulis/jobtype/connector/DataDruidFactory.java +++ /dev/null @@ -1,122 +0,0 @@ -/* - * Copyright 2019 WeBank - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package com.webank.wedatasphere.dss.appjoint.schedulis.jobtype.connector; - -import com.alibaba.druid.pool.DruidDataSource; -import org.apache.commons.lang3.StringUtils; -import org.apache.log4j.Logger; - -import java.util.Base64; -import java.util.Properties; - -public class DataDruidFactory { - private static DruidDataSource jobInstance; - private static DruidDataSource msgInstance; - - public static DruidDataSource getJobInstance(Properties props, Logger log) { - if (jobInstance == null ) { - synchronized (DataDruidFactory.class) { - if(jobInstance == null) { - try { - jobInstance = createDataSource(props, log, "Job"); - } catch (Exception e) { - throw new RuntimeException("Error creating Druid DataSource", e); - } - } - } - } - return jobInstance; - } - - public static DruidDataSource getMsgInstance(Properties props, Logger log) { - if (msgInstance == null ) { - synchronized (DataDruidFactory.class) { - if(msgInstance == null) { - try { - msgInstance = createDataSource(props, log, "Msg"); - } catch (Exception e) { - throw new RuntimeException("Error creating Druid DataSource", e); - } - } - } - } - return msgInstance; - } - - private static DruidDataSource createDataSource(Properties props, Logger log, String type) { - String name = null; - String url = null; - String username = null; - String password = null; - - if (type.equals("Job")) { - name = props.getProperty("job.datachecker.jdo.option.name"); - url = props.getProperty("job.datachecker.jdo.option.url"); - username = props.getProperty("job.datachecker.jdo.option.username"); - password = props.getProperty("job.datachecker.jdo.option.password"); - try { -// password = new String(Base64.getDecoder().decode(props.getProperty("job.datachecker.jdo.option.password").getBytes()),"UTF-8"); - password = props.getProperty("job.datachecker.jdo.option.password"); - } catch (Exception e){ - log.error("password decore failed" + e); - } - } - int initialSize = Integer.valueOf(props.getProperty("datachecker.jdo.option.initial.size", "1")); - int maxActive = Integer.valueOf(props.getProperty("datachecker.jdo.option.max.active", "100")); - int minIdle = Integer.valueOf(props.getProperty("datachecker.jdo.option.min.idle", "1")); - long maxWait = Long.valueOf(props.getProperty("datachecker.jdo.option.max.wait", "60000")); - String validationQuery = props.getProperty("datachecker.jdo.option.validation.quert", "SELECT 'x'"); - long timeBetweenEvictionRunsMillis = Long.valueOf(props.getProperty("datachecker.jdo.option.time.between.eviction.runs.millis", "6000")); - long minEvictableIdleTimeMillis = Long.valueOf(props.getProperty("datachecker.jdo.option.evictable.idle,time.millis", "300000")); - boolean testOnBorrow = Boolean.valueOf(props.getProperty("datachecker.jdo.option.test.on.borrow", "true")); - int maxOpenPreparedStatements = Integer.valueOf(props.getProperty("datachecker.jdo.option.max.open.prepared.statements", "-1")); - - - if (timeBetweenEvictionRunsMillis > minEvictableIdleTimeMillis) { - timeBetweenEvictionRunsMillis = minEvictableIdleTimeMillis; - } - - DruidDataSource ds = new DruidDataSource(); - - if (StringUtils.isNotBlank(name)) { - ds.setName(name); - } - - ds.setUrl(url); - ds.setDriverClassName("com.mysql.jdbc.Driver"); - ds.setUsername(username); - ds.setPassword(password); - ds.setInitialSize(initialSize); - ds.setMinIdle(minIdle); - ds.setMaxActive(maxActive); - ds.setMaxWait(maxWait); - ds.setTestOnBorrow(testOnBorrow); - ds.setValidationQuery(validationQuery); - ds.setTimeBetweenEvictionRunsMillis(timeBetweenEvictionRunsMillis); - ds.setMinEvictableIdleTimeMillis(minEvictableIdleTimeMillis); - if (maxOpenPreparedStatements > 0) { - ds.setPoolPreparedStatements(true); - ds.setMaxPoolPreparedStatementPerConnectionSize( - maxOpenPreparedStatements); - } else { - ds.setPoolPreparedStatements(false); - } - log.info("Druid data source initialed!"); - return ds; - } -} diff --git a/datachecker-appjoint/src/main/java/com/webank/wedatasphere/dss/appjoint/schedulis/jobtype/entity/DataChecker.java b/datachecker-appjoint/src/main/java/com/webank/wedatasphere/dss/appjoint/schedulis/jobtype/entity/DataChecker.java deleted file mode 100644 index b108cc6b97..0000000000 --- a/datachecker-appjoint/src/main/java/com/webank/wedatasphere/dss/appjoint/schedulis/jobtype/entity/DataChecker.java +++ /dev/null @@ -1,90 +0,0 @@ -/* - * Copyright 2019 WeBank - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package com.webank.wedatasphere.dss.appjoint.schedulis.jobtype.entity; - -import com.webank.wedatasphere.dss.appjoint.execution.common.NodeExecutionState; -import com.webank.wedatasphere.dss.appjoint.schedulis.jobtype.connector.DataCheckerDao; -import com.webank.wedatasphere.dss.appjoint.schedulis.jobtype.execution.DataCheckerNodeExecutionAction; -import org.apache.log4j.Logger; - -import java.util.Properties; - -public class DataChecker { - public final static String SOURCE_TYPE = "source.type"; - public final static String DATA_OBJECT = "check.object"; - public final static String WAIT_TIME = "max.check.hours"; - public final static String QUERY_FREQUENCY = "query.frequency"; - public final static String TIME_SCAPE = "time.scape"; - - private Properties p; - private static final Logger logger = Logger.getRootLogger(); - DataCheckerDao wbDao = DataCheckerDao.getInstance(); - DataCheckerNodeExecutionAction dataCheckerAction = null; - public long maxWaitTime; - public int queryFrequency; - - public DataChecker(String jobName, Properties p,DataCheckerNodeExecutionAction action) { - this.p = p; - dataCheckerAction = action; - maxWaitTime = Long.valueOf(p.getProperty(DataChecker.WAIT_TIME, "1")) * 3600 * 1000; - queryFrequency = Integer.valueOf(p.getProperty(DataChecker.QUERY_FREQUENCY, "30000")); - - } - - public void run() { - dataCheckerAction.setState(NodeExecutionState.Running); - try { - if(p == null) { - throw new RuntimeException("Properties is null. Can't continue"); - } - if (!p.containsKey(SOURCE_TYPE)) { - logger.info("Properties " + SOURCE_TYPE + " value is Null !"); - } - if (!p.containsKey(DATA_OBJECT)) { - logger.info("Properties " + DATA_OBJECT + " value is Null !"); - } - begineCheck(); - }catch (Exception ex){ - dataCheckerAction.setState(NodeExecutionState.Failed); - throw new RuntimeException("get DataChecker result failed", ex); - } - - } - - public void begineCheck(){ - boolean success=false; - try { - success= wbDao.validateTableStatusFunction(p, logger); - }catch (Exception ex){ - dataCheckerAction.setState(NodeExecutionState.Failed); - logger.error("datacheck error",ex); - throw new RuntimeException("get DataChecker result failed", ex); - } - if(success) { - dataCheckerAction.setState(NodeExecutionState.Success); - }else { - dataCheckerAction.setState(NodeExecutionState.Running); - } - } - - public void cancel() { -// DataCheckerDao.closeDruidDataSource(); -// throw new RuntimeException("Kill this DataChecker job."); - } - -} \ No newline at end of file diff --git a/datachecker-appjoint/src/main/resources/appjoint.properties b/datachecker-appjoint/src/main/resources/appjoint.properties deleted file mode 100644 index 01e9ff0b4b..0000000000 --- a/datachecker-appjoint/src/main/resources/appjoint.properties +++ /dev/null @@ -1,24 +0,0 @@ -# -# Copyright 2019 WeBank -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# - -job.datachecker.jdo.option.name=job -job.datachecker.jdo.option.url=jdbc:mysql://127.0.0.1:3306/ -job.datachecker.jdo.option.username= -job.datachecker.jdo.option.password= - - - diff --git a/datachecker-appjoint/src/main/resources/log4j.properties b/datachecker-appjoint/src/main/resources/log4j.properties deleted file mode 100644 index 0807e60877..0000000000 --- a/datachecker-appjoint/src/main/resources/log4j.properties +++ /dev/null @@ -1,37 +0,0 @@ -# -# Copyright 2019 WeBank -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# - -### set log levels ### - -log4j.rootCategory=INFO,console - -log4j.appender.console=org.apache.log4j.ConsoleAppender -log4j.appender.console.Threshold=INFO -log4j.appender.console.layout=org.apache.log4j.PatternLayout -#log4j.appender.console.layout.ConversionPattern= %d{ISO8601} %-5p (%t) [%F:%M(%L)] - %m%n -log4j.appender.console.layout.ConversionPattern= %d{ISO8601} %-5p (%t) %p %c{1} - %m%n - - -log4j.appender.com.webank.bdp.ide.core=org.apache.log4j.DailyRollingFileAppender -log4j.appender.com.webank.bdp.ide.core.Threshold=INFO -log4j.additivity.com.webank.bdp.ide.core=false -log4j.appender.com.webank.bdp.ide.core.layout=org.apache.log4j.PatternLayout -log4j.appender.com.webank.bdp.ide.core.Append=true -log4j.appender.com.webank.bdp.ide.core.File=logs/linkis.log -log4j.appender.com.webank.bdp.ide.core.layout.ConversionPattern= %d{ISO8601} %-5p (%t) [%F:%M(%L)] - %m%n - -log4j.logger.org.springframework=INFO diff --git a/datachecker-appjoint/src/main/resources/log4j2.xml b/datachecker-appjoint/src/main/resources/log4j2.xml deleted file mode 100644 index 3923cd9f39..0000000000 --- a/datachecker-appjoint/src/main/resources/log4j2.xml +++ /dev/null @@ -1,39 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - diff --git a/datachecker-appjoint/src/main/scala/com/webank/wedatasphere/dss/appjoint/schedulis/jobtype/execution/DataCheckerAppJoint.scala b/datachecker-appjoint/src/main/scala/com/webank/wedatasphere/dss/appjoint/schedulis/jobtype/execution/DataCheckerAppJoint.scala deleted file mode 100644 index f71b8307dc..0000000000 --- a/datachecker-appjoint/src/main/scala/com/webank/wedatasphere/dss/appjoint/schedulis/jobtype/execution/DataCheckerAppJoint.scala +++ /dev/null @@ -1,51 +0,0 @@ -/* - * Copyright 2019 WeBank - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package com.webank.wedatasphere.dss.appjoint.schedulis.jobtype.execution - -import java.util - -import com.webank.wedatasphere.dss.appjoint.AppJoint -import com.webank.wedatasphere.dss.appjoint.execution.NodeExecution -import com.webank.wedatasphere.dss.appjoint.service.AppJointUrlImpl - -/** - * Created by enjoyyin on 2019/11/5. - */ -class DataCheckerAppJoint extends AppJointUrlImpl with AppJoint { - - private var params: util.Map[String, AnyRef] = _ - private var nodeExecution: NodeExecution = _ - - override def getAppJointName: String = "DataChecker" - - override def init(baseUrl: String, params: util.Map[String, AnyRef]): Unit = { - setBaseUrl(baseUrl) - this.params = params - } - - override def getNodeExecution: NodeExecution = { - if(nodeExecution == null) synchronized { - if(nodeExecution == null) { - nodeExecution = new DataCheckerExecution() - nodeExecution.setBaseUrl(getBaseUrl) - nodeExecution.init(params) - } - } - nodeExecution - } -} diff --git a/datachecker-appjoint/src/main/scala/com/webank/wedatasphere/dss/appjoint/schedulis/jobtype/execution/DataCheckerExecution.scala b/datachecker-appjoint/src/main/scala/com/webank/wedatasphere/dss/appjoint/schedulis/jobtype/execution/DataCheckerExecution.scala deleted file mode 100644 index c169d44311..0000000000 --- a/datachecker-appjoint/src/main/scala/com/webank/wedatasphere/dss/appjoint/schedulis/jobtype/execution/DataCheckerExecution.scala +++ /dev/null @@ -1,157 +0,0 @@ -/* - * Copyright 2019 WeBank - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package com.webank.wedatasphere.dss.appjoint.schedulis.jobtype.execution - -import java.util -import java.util.{Properties, UUID} - -import com.webank.wedatasphere.dss.appjoint.execution.common._ -import com.webank.wedatasphere.dss.appjoint.execution.core._ -import com.webank.wedatasphere.dss.appjoint.schedulis.jobtype.entity.DataChecker -import com.webank.wedatasphere.dss.appjoint.service.AppJointUrl -import com.webank.wedatasphere.dss.appjoint.service.session.Session -import com.webank.wedatasphere.linkis.common.utils.Utils -import org.slf4j.LoggerFactory; - -/** - * Created by allenlliu on 2019/11/11. - */ -class DataCheckerExecution extends LongTermNodeExecution with AppJointUrl with Killable with Procedure { - private val logger = LoggerFactory.getLogger(classOf[DataCheckerExecution]) - import scala.collection.JavaConversions.mapAsScalaMap - var appJointParams : scala.collection.mutable.Map[String,AnyRef]= null - - - /** - * 表示任务能否提交到该AppJoint去执行 - * - * @param node AppJointNode - * @return true is ok while false is not - */ - override def canExecute(node: AppJointNode, context: NodeContext, session: Session): Boolean = node.getNodeType.toLowerCase.contains("datachecker") - - protected def putErrorMsg(errorMsg: String, t: Throwable, action: DataCheckerNodeExecutionAction): DataCheckerNodeExecutionAction = t match { - - case t: Exception => - val response = action.response - response.setErrorMsg(errorMsg) - response.setException(t) - response.setIsSucceed(false) - action - } - - override def init(params: util.Map[String, AnyRef]): Unit = { - this.appJointParams = params - } - override def submit(node: AppJointNode, context: NodeContext, session:Session): NodeExecutionAction = { - val nodeAction = new DataCheckerNodeExecutionAction() - nodeAction.setId(UUID.randomUUID().toString()) - val jobName = node.getName - val scalaParams: scala.collection.mutable.Map[String,Object] =context.getRuntimeMap - val properties = new Properties() - this.appJointParams.foreach{ - case (key: String, value: Object) => - logger.info("appjoint params key : "+key+",value : "+value) - properties.put(key, value.toString) - } - scalaParams.foreach { case (key: String, value: Object) => - logger.info("request params key : "+key+",value : "+value) - properties.put(key, value.toString) - } - val dc = new DataChecker(jobName,properties,nodeAction) - dc.run() - nodeAction.setDc(dc) - nodeAction - } - - override def state(action: NodeExecutionAction): NodeExecutionState = { - action match { - case action: DataCheckerNodeExecutionAction => { - if (action.state.isCompleted) return action.state - action.dc.begineCheck() - action.state - } - case _ => NodeExecutionState.Failed - } - } - private var baseUrl:String ="" - - override def getBaseUrl: String = baseUrl - - override def setBaseUrl(basicUrl: String): Unit = { - this.baseUrl = basicUrl - } - - override def result(action: NodeExecutionAction, nodeContext: NodeContext): CompletedNodeExecutionResponse = { - val response = new CompletedNodeExecutionResponse - action match { - case action: DataCheckerNodeExecutionAction => { - if (action.state.equals(NodeExecutionState.Success)) { - response.setIsSucceed(true) - } else { - response.setIsSucceed(false) - } - response - } - case _ => { - response.setIsSucceed(false) - response - } - - - } - } - - override def kill(action: NodeExecutionAction): Boolean = action match { - case longTermAction: DataCheckerNodeExecutionAction => - getScheduler.removeAsyncResponse(longTermAction) - true - } - - override def progress(action: NodeExecutionAction): Float = { - return 0.5f - } - - override def log(action: NodeExecutionAction): String = { - action match { - case action: DataCheckerNodeExecutionAction => { - if (!action.state.isCompleted) { - "DataChecker is waiting for tables" - } else { - "DataChecker successfully received info of tables" - } - } - case _ => "Error for NodeExecutionAction " - } - - } - - override def createAsyncNodeExecutionResponse(node: AppJointNode, context: NodeContext, action: NodeExecutionAction): AsyncNodeExecutionResponse = { - action match { - case action: DataCheckerNodeExecutionAction => { - val response = new AsyncNodeExecutionResponse - response.setAction(action) - response.setAppJointNode(node) - response.setNodeContext(context) - response.setMaxLoopTime(action.dc.maxWaitTime) - response.setAskStatePeriod(action.dc.queryFrequency) - response - } - } - } -} diff --git a/datachecker-appjoint/src/main/scala/com/webank/wedatasphere/dss/appjoint/schedulis/jobtype/execution/DataCheckerNodeExecutionAction.scala b/datachecker-appjoint/src/main/scala/com/webank/wedatasphere/dss/appjoint/schedulis/jobtype/execution/DataCheckerNodeExecutionAction.scala deleted file mode 100644 index fff35e0679..0000000000 --- a/datachecker-appjoint/src/main/scala/com/webank/wedatasphere/dss/appjoint/schedulis/jobtype/execution/DataCheckerNodeExecutionAction.scala +++ /dev/null @@ -1,46 +0,0 @@ -/* - * Copyright 2019 WeBank - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package com.webank.wedatasphere.dss.appjoint.schedulis.jobtype.execution - -import com.webank.wedatasphere.dss.appjoint.execution.common.{AbstractNodeExecutionAction, CompletedNodeExecutionResponse, LongTermNodeExecutionAction, NodeExecutionState} -import com.webank.wedatasphere.dss.appjoint.schedulis.jobtype.entity.DataChecker - -/** - * Created by allenlliu on 2019/11/12. - */ -class DataCheckerNodeExecutionAction extends AbstractNodeExecutionAction with LongTermNodeExecutionAction { - private[this] var _state: NodeExecutionState = null - private var schedulerId: Int = _ - def state: NodeExecutionState = _state - - def setState(value: NodeExecutionState): Unit = { - _state = value - } - val response = new CompletedNodeExecutionResponse - private[this] var _dc: DataChecker = null - - def dc: DataChecker = _dc - - def setDc(value: DataChecker): Unit = { - _dc = value - } - - override def setSchedulerId(schedulerId: Int): Unit = this.schedulerId = schedulerId - - override def getSchedulerId: Int = schedulerId -} diff --git a/dss-appjoint-auth/pom.xml b/dss-appjoint-auth/pom.xml deleted file mode 100644 index 3a368b8666..0000000000 --- a/dss-appjoint-auth/pom.xml +++ /dev/null @@ -1,75 +0,0 @@ - - - - - - dss - com.webank.wedatasphere.dss - 0.9.1 - - 4.0.0 - - dss-appjoint-auth - - - com.webank.wedatasphere.linkis - linkis-gateway-httpclient-support - ${linkis.version} - - - com.webank.wedatasphere.linkis - linkis-common - ${linkis.version} - - - javax.servlet - javax.servlet-api - 3.1.0 - - - - - - - org.apache.maven.plugins - maven-deploy-plugin - - - net.alchim31.maven - scala-maven-plugin - - - org.apache.maven.plugins - maven-jar-plugin - - - org.apache.maven.plugins - maven-compiler-plugin - 3.3 - - 1.8 - 1.8 - UTF-8 - - - - - - \ No newline at end of file diff --git a/dss-appjoint-auth/src/main/java/com/webank/wedatasphere/dss/appjoint/auth/AppJointAuth.java b/dss-appjoint-auth/src/main/java/com/webank/wedatasphere/dss/appjoint/auth/AppJointAuth.java deleted file mode 100644 index aa03f8dc6e..0000000000 --- a/dss-appjoint-auth/src/main/java/com/webank/wedatasphere/dss/appjoint/auth/AppJointAuth.java +++ /dev/null @@ -1,38 +0,0 @@ -/* - * Copyright 2019 WeBank - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package com.webank.wedatasphere.dss.appjoint.auth; - -import com.webank.wedatasphere.dss.appjoint.auth.impl.AppJointAuthImpl$; - -import javax.servlet.http.HttpServletRequest; -import java.io.Closeable; - -/** - * Created by enjoyyin on 2019/11/6. - */ -public interface AppJointAuth extends Closeable { - - boolean isDssRequest(HttpServletRequest request); - - RedirectMsg getRedirectMsg(HttpServletRequest request); - - static AppJointAuth getAppJointAuth() { - return AppJointAuthImpl$.MODULE$.apply(); - } - -} diff --git a/dss-appjoint-auth/src/main/java/com/webank/wedatasphere/dss/appjoint/auth/RedirectMsg.java b/dss-appjoint-auth/src/main/java/com/webank/wedatasphere/dss/appjoint/auth/RedirectMsg.java deleted file mode 100644 index b4875a1960..0000000000 --- a/dss-appjoint-auth/src/main/java/com/webank/wedatasphere/dss/appjoint/auth/RedirectMsg.java +++ /dev/null @@ -1,33 +0,0 @@ -/* - * Copyright 2019 WeBank - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package com.webank.wedatasphere.dss.appjoint.auth; - -/** - * Created by enjoyyin on 2019/11/7. - */ -public interface RedirectMsg { - - String getRedirectUrl(); - - void setRedirectUrl(String redirectUrl); - - String getUser(); - - void setUser(String user); - -} diff --git a/dss-appjoint-auth/src/main/scala/com/webank/wedatasphere/dss/appjoint/auth/impl/AppJointAuthImpl.scala b/dss-appjoint-auth/src/main/scala/com/webank/wedatasphere/dss/appjoint/auth/impl/AppJointAuthImpl.scala deleted file mode 100644 index 15dd7e3072..0000000000 --- a/dss-appjoint-auth/src/main/scala/com/webank/wedatasphere/dss/appjoint/auth/impl/AppJointAuthImpl.scala +++ /dev/null @@ -1,99 +0,0 @@ -/* - * Copyright 2019 WeBank - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package com.webank.wedatasphere.dss.appjoint.auth.impl - -import java.net.URI -import java.util - -import com.webank.wedatasphere.dss.appjoint.auth.{AppJointAuth, RedirectMsg} -import com.webank.wedatasphere.linkis.common.utils.Logging -import com.webank.wedatasphere.linkis.httpclient.dws.DWSHttpClient -import com.webank.wedatasphere.linkis.httpclient.dws.config.DWSClientConfigBuilder -import javax.servlet.http.HttpServletRequest -import org.apache.commons.io.IOUtils -import org.apache.http.impl.cookie.BasicClientCookie - -import scala.collection.JavaConversions._ - -/** - * Created by enjoyyin on 2019/11/6. - */ -class AppJointAuthImpl private() extends AppJointAuth with Logging { - - private val dwsHttpClients = new util.HashMap[String, DWSHttpClient] - - private def getBaseUrl(dssUrl: String): String = { - val uri = new URI(dssUrl) - val dssPort = if(uri.getPort != -1) uri.getPort else 80 - uri.getScheme + "://" + uri.getHost + ":" + dssPort - } - - protected def getDWSClient(dssUrl: String): DWSHttpClient = { - val baseUrl = getBaseUrl(dssUrl) - if(!dwsHttpClients.containsKey(baseUrl)) baseUrl.intern synchronized { - if(!dwsHttpClients.containsKey(baseUrl)) { - info("create a new DSSClient for url " + baseUrl) - val clientConfig = DWSClientConfigBuilder.newBuilder().setDWSVersion(AppJointAuthImpl.dssVersion) - .addUJESServerUrl(baseUrl).connectionTimeout(30000).discoveryEnabled(false) - .maxConnectionSize(AppJointAuthImpl.maxConnection).readTimeout(30000).build() - val dwsHttpClient = new DWSHttpClient(clientConfig, "DSS-UserInfo-Ask-Client") - dwsHttpClients.put(baseUrl, dwsHttpClient) - } - } - dwsHttpClients.get(baseUrl) - } - - override def isDssRequest(request: HttpServletRequest): Boolean = request.getParameterMap.containsKey(AppJointAuthImpl.DSS_URL_KEY) - - override def getRedirectMsg(request: HttpServletRequest): RedirectMsg = { - val dssUrl = request.getParameter(AppJointAuthImpl.DSS_URL_KEY) - val dwsHttpClient = getDWSClient(dssUrl) - val userInfoAction = new UserInfoAction - val cookies = request.getParameter(AppJointAuthImpl.COOKIES_KEY) - cookies.split(";").foreach { cookie => - val index = cookie.indexOf("=") - val key = cookie.substring(0, index).trim - val value = cookie.substring(index + 1).trim - userInfoAction.addCookie(new BasicClientCookie(key, value)) - } - val redirectMsg = new RedirectMsgImpl - redirectMsg.setRedirectUrl(request.getParameter(AppJointAuthImpl.REDIRECT_KEY)) - dwsHttpClient.execute(userInfoAction) match { - case userInfoResult: UserInfoResult => - redirectMsg.setUser(userInfoResult.getUserName) - } - redirectMsg - } - - override def close(): Unit = dwsHttpClients.values().foreach(IOUtils.closeQuietly) -} - -object AppJointAuthImpl { - private val appJointAuth = new AppJointAuthImpl - private var dssVersion = "v1" - private var maxConnection = 5 - - private val REDIRECT_KEY = "redirect" - private val DSS_URL_KEY = "dssurl" - private val COOKIES_KEY = "cookies" - - def apply: AppJointAuthImpl = appJointAuth - - def setDSSVersion(version: String): Unit = this.dssVersion = version - def setMaxConnection(maxConnection: Int): Unit = this.maxConnection = maxConnection -} \ No newline at end of file diff --git a/dss-appjoint-auth/src/main/scala/com/webank/wedatasphere/dss/appjoint/auth/impl/RedirectMsgImpl.scala b/dss-appjoint-auth/src/main/scala/com/webank/wedatasphere/dss/appjoint/auth/impl/RedirectMsgImpl.scala deleted file mode 100644 index ef2c8da748..0000000000 --- a/dss-appjoint-auth/src/main/scala/com/webank/wedatasphere/dss/appjoint/auth/impl/RedirectMsgImpl.scala +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright 2019 WeBank - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package com.webank.wedatasphere.dss.appjoint.auth.impl - -import com.webank.wedatasphere.dss.appjoint.auth.RedirectMsg - -/** - * Created by enjoyyin on 2019/11/7. - */ -class RedirectMsgImpl extends RedirectMsg { - - private var redirectUrl: String = _ - private var user: String = _ - - override def getRedirectUrl: String = redirectUrl - - override def setRedirectUrl(redirectUrl: String): Unit = this.redirectUrl = redirectUrl - - override def getUser: String = user - - override def setUser(user: String): Unit = this.user = user -} diff --git a/dss-appjoint-auth/src/main/scala/com/webank/wedatasphere/dss/appjoint/auth/impl/UserInfoAction.scala b/dss-appjoint-auth/src/main/scala/com/webank/wedatasphere/dss/appjoint/auth/impl/UserInfoAction.scala deleted file mode 100644 index 6d454b2ff3..0000000000 --- a/dss-appjoint-auth/src/main/scala/com/webank/wedatasphere/dss/appjoint/auth/impl/UserInfoAction.scala +++ /dev/null @@ -1,28 +0,0 @@ -/* - * Copyright 2019 WeBank - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package com.webank.wedatasphere.dss.appjoint.auth.impl - -import com.webank.wedatasphere.linkis.httpclient.dws.request.DWSHttpAction -import com.webank.wedatasphere.linkis.httpclient.request.GetAction - -/** - * Created by enjoyyin on 2019/11/7. - */ -class UserInfoAction extends GetAction with DWSHttpAction { - override def suffixURLs: Array[String] = Array("user", "userInfo") -} diff --git a/dss-appjoint-auth/src/main/scala/com/webank/wedatasphere/dss/appjoint/auth/impl/UserInfoResult.scala b/dss-appjoint-auth/src/main/scala/com/webank/wedatasphere/dss/appjoint/auth/impl/UserInfoResult.scala deleted file mode 100644 index f0280b8cfe..0000000000 --- a/dss-appjoint-auth/src/main/scala/com/webank/wedatasphere/dss/appjoint/auth/impl/UserInfoResult.scala +++ /dev/null @@ -1,33 +0,0 @@ -/* - * Copyright 2019 WeBank - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package com.webank.wedatasphere.dss.appjoint.auth.impl - -import com.webank.wedatasphere.linkis.httpclient.dws.annotation.DWSHttpMessageResult -import com.webank.wedatasphere.linkis.httpclient.dws.response.DWSResult - -import scala.beans.BeanProperty - -/** - * Created by enjoyyin on 2019/11/7. - */ -@DWSHttpMessageResult("/api/rest_j/v\\d+/user/userInfo") -class UserInfoResult extends DWSResult { - - @BeanProperty var userName: String = _ - -} diff --git a/dss-appjoint-core/pom.xml b/dss-appjoint-core/pom.xml deleted file mode 100644 index 7140b33656..0000000000 --- a/dss-appjoint-core/pom.xml +++ /dev/null @@ -1,93 +0,0 @@ - - - - - - dss - com.webank.wedatasphere.dss - 0.9.1 - - 4.0.0 - - dss-appjoint-core - - - - com.webank.wedatasphere.linkis - linkis-protocol - ${linkis.version} - - - com.webank.wedatasphere.linkis - linkis-storage - ${linkis.version} - - - - - - - - - com.webank.wedatasphere.dss - dss-common - ${dss.version} - - - - com.webank.wedatasphere.linkis - linkis-httpclient - ${linkis.version} - - - - - - - - org.apache.maven.plugins - maven-deploy-plugin - - - net.alchim31.maven - scala-maven-plugin - - - org.apache.maven.plugins - maven-jar-plugin - - - org.apache.maven.plugins - maven-compiler-plugin - 3.3 - - 1.8 - 1.8 - UTF-8 - - - - - - - - - - \ No newline at end of file diff --git a/dss-appjoint-core/src/main/java/com/webank/wedatasphere/dss/appjoint/AppJoint.java b/dss-appjoint-core/src/main/java/com/webank/wedatasphere/dss/appjoint/AppJoint.java deleted file mode 100644 index 10d20f11c4..0000000000 --- a/dss-appjoint-core/src/main/java/com/webank/wedatasphere/dss/appjoint/AppJoint.java +++ /dev/null @@ -1,73 +0,0 @@ -/* - * Copyright 2019 WeBank - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package com.webank.wedatasphere.dss.appjoint; - -import com.webank.wedatasphere.dss.appjoint.exception.AppJointErrorException; -import com.webank.wedatasphere.dss.appjoint.execution.NodeExecution; -import com.webank.wedatasphere.dss.appjoint.service.NodeService; -import com.webank.wedatasphere.dss.appjoint.service.ProjectService; -import com.webank.wedatasphere.dss.appjoint.service.SecurityService; - -import java.util.Map; - -/** - * Created by enjoyyin on 2019/10/10. - */ -public interface AppJoint { - - /** - * 每一个AppJoint都需要有一个自己的标识名
- * 这个标识名是唯一的，如qualitis子系统，可以将名字设置为qualitis; visualis子系统，可以设置为visualis - */ - String getAppJointName(); - - void init(String baseUrl, Map params) throws AppJointErrorException; - - /** - * 如果不存在，直接返回null即可 - * @return - */ - default SecurityService getSecurityService() { - return null; - } - - /** - * 如果不存在，直接返回null即可 - * @return - */ - default ProjectService getProjectService() { - return null; - } - - /** - * 如果不存在，直接返回null即可 - * @return - */ - default NodeService getNodeService() { - return null; - } - - /** - * 如果不存在，直接返回null即可 - * @return - */ - default NodeExecution getNodeExecution() { - return null; - } - -} diff --git a/dss-appjoint-core/src/main/java/com/webank/wedatasphere/dss/appjoint/exception/AppJointErrorException.java b/dss-appjoint-core/src/main/java/com/webank/wedatasphere/dss/appjoint/exception/AppJointErrorException.java deleted file mode 100644 index 86d0749283..0000000000 --- a/dss-appjoint-core/src/main/java/com/webank/wedatasphere/dss/appjoint/exception/AppJointErrorException.java +++ /dev/null @@ -1,35 +0,0 @@ -/* - * Copyright 2019 WeBank - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package com.webank.wedatasphere.dss.appjoint.exception; - -import com.webank.wedatasphere.linkis.common.exception.ErrorException; - -/** - * Created by enjoyyin on 2019/11/5. - */ -public class AppJointErrorException extends ErrorException { - - public AppJointErrorException(int errCode, String desc) { - super(errCode, desc); - } - - public AppJointErrorException(int errCode, String desc, Throwable cause) { - super(errCode, desc); - initCause(cause); - } -} diff --git a/dss-appjoint-core/src/main/java/com/webank/wedatasphere/dss/appjoint/execution/NodeExecution.java b/dss-appjoint-core/src/main/java/com/webank/wedatasphere/dss/appjoint/execution/NodeExecution.java deleted file mode 100644 index 6cd1794927..0000000000 --- a/dss-appjoint-core/src/main/java/com/webank/wedatasphere/dss/appjoint/execution/NodeExecution.java +++ /dev/null @@ -1,53 +0,0 @@ -/* - * Copyright 2019 WeBank - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package com.webank.wedatasphere.dss.appjoint.execution; - - -import com.webank.wedatasphere.dss.appjoint.exception.AppJointErrorException; -import com.webank.wedatasphere.dss.appjoint.execution.common.NodeExecutionResponse; -import com.webank.wedatasphere.dss.appjoint.execution.core.AppJointNode; -import com.webank.wedatasphere.dss.appjoint.execution.core.NodeContext; -import com.webank.wedatasphere.dss.appjoint.service.AppJointUrl; -import com.webank.wedatasphere.dss.appjoint.service.session.Session; - - -/** - * created by enjoyyin on 2019/9/25 - * Description: - * NodeExecution - */ -public interface NodeExecution extends AppJointUrl { - - void init(java.util.Map params) throws AppJointErrorException; - - /** - * 表示任务能否提交到该AppJoint去执行 - * @param appJointNode AppJointNode - * @return true is ok while false is not - */ - boolean canExecute(AppJointNode appJointNode, NodeContext context, Session session); - - /** - * 相应的appJoint提交到外部系统执行 - * @param appJointNode 就是工作流节点 - * @param context 运行时参数的context - * @param session session是为了解决将任务提交给第三方系统时解决鉴权等问题 - */ - NodeExecutionResponse execute(AppJointNode appJointNode, NodeContext context, Session session) throws AppJointErrorException; - -} diff --git a/dss-appjoint-core/src/main/java/com/webank/wedatasphere/dss/appjoint/execution/async/NodeExecutionListener.java b/dss-appjoint-core/src/main/java/com/webank/wedatasphere/dss/appjoint/execution/async/NodeExecutionListener.java deleted file mode 100644 index 4502660f1b..0000000000 --- a/dss-appjoint-core/src/main/java/com/webank/wedatasphere/dss/appjoint/execution/async/NodeExecutionListener.java +++ /dev/null @@ -1,40 +0,0 @@ -/* - * Copyright 2019 WeBank - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package com.webank.wedatasphere.dss.appjoint.execution.async; - -import com.webank.wedatasphere.dss.appjoint.execution.common.AsyncNodeExecutionResponse; -import com.webank.wedatasphere.dss.appjoint.execution.common.CompletedNodeExecutionResponse; -import com.webank.wedatasphere.dss.appjoint.execution.common.NodeExecutionAction; -import com.webank.wedatasphere.dss.appjoint.execution.core.AppJointNode; -import com.webank.wedatasphere.dss.appjoint.execution.core.NodeContext; - -/** - * created by enjoyyin on 2019/10/8 - * Description: - */ -public interface NodeExecutionListener { - - void beforeSubmit(AppJointNode appJointNode, NodeContext nodeContext); - - void afterSubmit(AppJointNode appJointNode, NodeContext nodeContext, NodeExecutionAction action); - - void afterAsyncNodeExecutionResponse(AsyncNodeExecutionResponse response); - - void afterCompletedNodeExecutionResponse(AppJointNode appJointNode, NodeContext nodeContext, NodeExecutionAction action, - CompletedNodeExecutionResponse response); -} diff --git a/dss-appjoint-core/src/main/java/com/webank/wedatasphere/dss/appjoint/execution/async/NodeExecutionLogListener.java b/dss-appjoint-core/src/main/java/com/webank/wedatasphere/dss/appjoint/execution/async/NodeExecutionLogListener.java deleted file mode 100644 index 7959042ed4..0000000000 --- a/dss-appjoint-core/src/main/java/com/webank/wedatasphere/dss/appjoint/execution/async/NodeExecutionLogListener.java +++ /dev/null @@ -1,26 +0,0 @@ -/* - * Copyright 2019 WeBank - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package com.webank.wedatasphere.dss.appjoint.execution.async; - -/** - * created by enjoyyin on 2019/9/30 - * Description: - */ -public interface NodeExecutionLogListener { - public void onLogUpdated(String log); -} diff --git a/dss-appjoint-core/src/main/java/com/webank/wedatasphere/dss/appjoint/execution/async/NodeExecutionProgressListener.java b/dss-appjoint-core/src/main/java/com/webank/wedatasphere/dss/appjoint/execution/async/NodeExecutionProgressListener.java deleted file mode 100644 index b1955a617c..0000000000 --- a/dss-appjoint-core/src/main/java/com/webank/wedatasphere/dss/appjoint/execution/async/NodeExecutionProgressListener.java +++ /dev/null @@ -1,26 +0,0 @@ -/* - * Copyright 2019 WeBank - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package com.webank.wedatasphere.dss.appjoint.execution.async; - -/** - * created by enjoyyin on 2019/9/30 - * Description: - */ -public interface NodeExecutionProgressListener { - public void onProgressUpdated(double progress); -} diff --git a/dss-appjoint-core/src/main/java/com/webank/wedatasphere/dss/appjoint/execution/async/NodeExecutionResponseListener.java b/dss-appjoint-core/src/main/java/com/webank/wedatasphere/dss/appjoint/execution/async/NodeExecutionResponseListener.java deleted file mode 100644 index 8cc32bdb8b..0000000000 --- a/dss-appjoint-core/src/main/java/com/webank/wedatasphere/dss/appjoint/execution/async/NodeExecutionResponseListener.java +++ /dev/null @@ -1,30 +0,0 @@ -/* - * Copyright 2019 WeBank - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package com.webank.wedatasphere.dss.appjoint.execution.async; - -import com.webank.wedatasphere.dss.appjoint.execution.common.CompletedNodeExecutionResponse; - -/** - * created by enjoyyin on 2019/9/25 - * Description: - */ -public interface NodeExecutionResponseListener { - - void onNodeExecutionCompleted(CompletedNodeExecutionResponse response); - -} diff --git a/dss-appjoint-core/src/main/java/com/webank/wedatasphere/dss/appjoint/execution/async/NodeExecutionStatusListener.java b/dss-appjoint-core/src/main/java/com/webank/wedatasphere/dss/appjoint/execution/async/NodeExecutionStatusListener.java deleted file mode 100644 index a606638f3e..0000000000 --- a/dss-appjoint-core/src/main/java/com/webank/wedatasphere/dss/appjoint/execution/async/NodeExecutionStatusListener.java +++ /dev/null @@ -1,26 +0,0 @@ -/* - * Copyright 2019 WeBank - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package com.webank.wedatasphere.dss.appjoint.execution.async; - -/** - * created by enjoyyin on 2019/9/30 - * Description: - */ -public interface NodeExecutionStatusListener { - public void onStatusUpdated(String status); -} diff --git a/dss-appjoint-core/src/main/java/com/webank/wedatasphere/dss/appjoint/execution/common/AbstractNodeExecutionAction.java b/dss-appjoint-core/src/main/java/com/webank/wedatasphere/dss/appjoint/execution/common/AbstractNodeExecutionAction.java deleted file mode 100644 index 495f77397a..0000000000 --- a/dss-appjoint-core/src/main/java/com/webank/wedatasphere/dss/appjoint/execution/common/AbstractNodeExecutionAction.java +++ /dev/null @@ -1,35 +0,0 @@ -/* - * Copyright 2019 WeBank - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package com.webank.wedatasphere.dss.appjoint.execution.common; - -/** - * created by enjoyyin on 2019/9/26 - * Description: - */ -public abstract class AbstractNodeExecutionAction implements NodeExecutionAction { - - public String id; - - public String getId() { - return id; - } - - public void setId(String id) { - this.id = id; - } -} diff --git a/dss-appjoint-core/src/main/java/com/webank/wedatasphere/dss/appjoint/execution/common/AsyncNodeExecutionResponse.java b/dss-appjoint-core/src/main/java/com/webank/wedatasphere/dss/appjoint/execution/common/AsyncNodeExecutionResponse.java deleted file mode 100644 index a0fb502fd4..0000000000 --- a/dss-appjoint-core/src/main/java/com/webank/wedatasphere/dss/appjoint/execution/common/AsyncNodeExecutionResponse.java +++ /dev/null @@ -1,116 +0,0 @@ -/* - * Copyright 2019 WeBank - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package com.webank.wedatasphere.dss.appjoint.execution.common; - -import com.webank.wedatasphere.dss.appjoint.execution.async.NodeExecutionResponseListener; -import com.webank.wedatasphere.dss.appjoint.execution.core.LongTermNodeExecution; -import com.webank.wedatasphere.dss.appjoint.execution.core.AppJointNode; -import com.webank.wedatasphere.dss.appjoint.execution.core.NodeContext; - -import java.util.ArrayList; -import java.util.List; - -/** - * created by enjoyyin on 2019/9/25 - * Description: - */ -public class AsyncNodeExecutionResponse implements NodeExecutionResponse { - - private NodeExecutionAction action; - private LongTermNodeExecution nodeExecution; - private AppJointNode appJointNode; - private NodeContext nodeContext; - private boolean isCompleted = false; - private long startTime = System.currentTimeMillis(); - private long askStatePeriod = 1000; - private long maxLoopTime = -1; - - /** - * 通过监听的方式，任务一旦完成，我们就通知上层 - */ - private List listener = new ArrayList<>(); - - - public NodeExecutionAction getAction() { - return action; - } - - public void setAction(NodeExecutionAction action) { - this.action = action; - } - - public void addListener(NodeExecutionResponseListener responseListener){ - this.listener.add(responseListener); - } - - public List getListeners(){ - return this.listener; - } - - public AppJointNode getAppJointNode() { - return appJointNode; - } - - public void setAppJointNode(AppJointNode appJointNode) { - this.appJointNode = appJointNode; - } - - public NodeContext getNodeContext() { - return nodeContext; - } - - public void setNodeContext(NodeContext nodeContext) { - this.nodeContext = nodeContext; - } - - public LongTermNodeExecution getNodeExecution() { - return nodeExecution; - } - - public void setNodeExecution(LongTermNodeExecution nodeExecution) { - this.nodeExecution = nodeExecution; - } - - public long getAskStatePeriod() { - return askStatePeriod; - } - - public void setAskStatePeriod(long askStatePeriod) { - this.askStatePeriod = askStatePeriod; - } - - public boolean isCompleted() { - return isCompleted; - } - - public void setCompleted(boolean completed) { - isCompleted = completed; - } - - public long getMaxLoopTime() { - return maxLoopTime; - } - - public void setMaxLoopTime(long maxLoopTime) { - this.maxLoopTime = maxLoopTime; - } - - public long getStartTime() { - return startTime; - } -} diff --git a/dss-appjoint-core/src/main/java/com/webank/wedatasphere/dss/appjoint/execution/common/CompletedNodeExecutionResponse.java b/dss-appjoint-core/src/main/java/com/webank/wedatasphere/dss/appjoint/execution/common/CompletedNodeExecutionResponse.java deleted file mode 100644 index b2cbb57d99..0000000000 --- a/dss-appjoint-core/src/main/java/com/webank/wedatasphere/dss/appjoint/execution/common/CompletedNodeExecutionResponse.java +++ /dev/null @@ -1,66 +0,0 @@ -/* - * Copyright 2019 WeBank - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package com.webank.wedatasphere.dss.appjoint.execution.common; - -/** - * created by enjoyyin on 2019/9/25 - * Description: - */ -public class CompletedNodeExecutionResponse implements NodeExecutionResponse { - - private boolean isSucceed; - - private String errorMsg; - - private String succeedMsg; - - private Exception exception; - - public void setIsSucceed(boolean isSucceed){ - this.isSucceed = isSucceed; - } - - public boolean isSucceed() { - return this.isSucceed; - } - - public String getErrorMsg(){ - return this.errorMsg; - } - - public void setErrorMsg(String errorMsg){ - this.errorMsg = errorMsg; - } - - - public String getSucceedMsg() { - return succeedMsg; - } - - public void setSucceedMsg(String succeedMsg) { - this.succeedMsg = succeedMsg; - } - - public Exception getException() { - return exception; - } - - public void setException(Exception exception) { - this.exception = exception; - } -} diff --git a/dss-appjoint-core/src/main/java/com/webank/wedatasphere/dss/appjoint/execution/common/LongTermNodeExecutionAction.java b/dss-appjoint-core/src/main/java/com/webank/wedatasphere/dss/appjoint/execution/common/LongTermNodeExecutionAction.java deleted file mode 100644 index 2c798182b8..0000000000 --- a/dss-appjoint-core/src/main/java/com/webank/wedatasphere/dss/appjoint/execution/common/LongTermNodeExecutionAction.java +++ /dev/null @@ -1,26 +0,0 @@ -/* - * Copyright 2019 WeBank - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package com.webank.wedatasphere.dss.appjoint.execution.common; - -/** - * Created by enjoyyin on 2019/11/18. - */ -public interface LongTermNodeExecutionAction { - void setSchedulerId(int schedulerId); - int getSchedulerId(); -} diff --git a/dss-appjoint-core/src/main/java/com/webank/wedatasphere/dss/appjoint/execution/common/NodeExecutionAction.java b/dss-appjoint-core/src/main/java/com/webank/wedatasphere/dss/appjoint/execution/common/NodeExecutionAction.java deleted file mode 100644 index a7414e516e..0000000000 --- a/dss-appjoint-core/src/main/java/com/webank/wedatasphere/dss/appjoint/execution/common/NodeExecutionAction.java +++ /dev/null @@ -1,25 +0,0 @@ -/* - * Copyright 2019 WeBank - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package com.webank.wedatasphere.dss.appjoint.execution.common; - -/** - * created by enjoyyin on 2019/9/26 - * Description: - */ -public interface NodeExecutionAction { -} diff --git a/dss-appjoint-core/src/main/java/com/webank/wedatasphere/dss/appjoint/execution/common/NodeExecutionResponse.java b/dss-appjoint-core/src/main/java/com/webank/wedatasphere/dss/appjoint/execution/common/NodeExecutionResponse.java deleted file mode 100644 index ed6f3441c3..0000000000 --- a/dss-appjoint-core/src/main/java/com/webank/wedatasphere/dss/appjoint/execution/common/NodeExecutionResponse.java +++ /dev/null @@ -1,25 +0,0 @@ -/* - * Copyright 2019 WeBank - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package com.webank.wedatasphere.dss.appjoint.execution.common; - -/** - * created by enjoyyin on 2019/9/25 - * Description: - */ -public interface NodeExecutionResponse { -} diff --git a/dss-appjoint-core/src/main/java/com/webank/wedatasphere/dss/appjoint/execution/common/NodeExecutionState.java b/dss-appjoint-core/src/main/java/com/webank/wedatasphere/dss/appjoint/execution/common/NodeExecutionState.java deleted file mode 100644 index 19a4968cba..0000000000 --- a/dss-appjoint-core/src/main/java/com/webank/wedatasphere/dss/appjoint/execution/common/NodeExecutionState.java +++ /dev/null @@ -1,71 +0,0 @@ -/* - * Copyright 2019 WeBank - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package com.webank.wedatasphere.dss.appjoint.execution.common; - -/** - * created by enjoyyin on 2019/9/25 - * Description: - */ -public enum NodeExecutionState { - /** - * NodeExecution的状态枚举 - */ - Accepted(0, "Accepted"), - Running(1, "Running"), - Success(2, "Success"), - Failed(3, "Failed"), - Killed(4, "Killed"), - Alert(5, "Alert"); - - private int code; - private String status; - - private NodeExecutionState(int code, String status){ - this.code = code; - this.status = status; - } - - public int getCode() { - return code; - } - - public String getStatus() { - return status; - } - - public static boolean isCompleted(NodeExecutionState state){ - return Success.equals(state) || Failed.equals(state) || Killed.equals(state); - } - - public static boolean isCompleted(String state){ - return Success.status.equals(state) || - Failed.status.equals(state) || - Killed.status.equals(state); - } - - public boolean isCompleted(){ - return isCompleted(this); - } - - public boolean isSuccess(){ - return Success.equals(this); - } - - - -} diff --git a/dss-appjoint-core/src/main/java/com/webank/wedatasphere/dss/appjoint/execution/core/AbstractAppJointNode.java b/dss-appjoint-core/src/main/java/com/webank/wedatasphere/dss/appjoint/execution/core/AbstractAppJointNode.java deleted file mode 100644 index dad4d22b5a..0000000000 --- a/dss-appjoint-core/src/main/java/com/webank/wedatasphere/dss/appjoint/execution/core/AbstractAppJointNode.java +++ /dev/null @@ -1,144 +0,0 @@ -/* - * Copyright 2019 WeBank - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package com.webank.wedatasphere.dss.appjoint.execution.core; - -import java.util.Map; - -/** - * created by enjoyyin on 2019/9/25 - * Description: - */ -public abstract class AbstractAppJointNode implements AppJointNode{ - - String projectName; - - long projectId; - - - String flowName; - - long flowId; - - String nodeName; - - String nodeId; - - String nodeType; - - Map jobContent; - - - public AbstractAppJointNode(String projectName, long projectId, String flowName, long flowId, - String nodeName, String nodeTye, Map jobContent){ - this.projectId = projectId; - this.projectName = projectName; - this.flowName = flowName; - this.flowId = flowId; - this.nodeName = nodeName; - this.nodeType = nodeTye; - this.jobContent = jobContent; - } - - public AbstractAppJointNode(){ - - } - - - public void setProjectName(String projectName) { - this.projectName = projectName; - } - - public void setProjectId(long projectId) { - this.projectId = projectId; - } - - public void setFlowName(String flowName) { - this.flowName = flowName; - } - - public void setFlowId(long flowId) { - this.flowId = flowId; - } - - - @Override - public String getId() { - return this.nodeId; - } - - @Override - public void setId(String id) { - this.nodeId = id; - } - - @Override - public String getNodeType() { - return this.nodeType; - } - - @Override - public void setNodeType(String nodeType) { - this.nodeType = nodeType; - } - - @Override - public String getName() { - return this.nodeName; - } - - @Override - public void setName(String name) { - this.nodeName = name; - } - - @Override - public long getProjectId() { - return this.projectId; - } - - - - - - - @Override - public String getProjectName() { - return this.projectName; - } - - @Override - public String getFlowName() { - return this.flowName; - } - - @Override - public long getFlowId() { - return this.flowId; - } - - @Override - public Map getJobContent() { - return this.jobContent; - } - - @Override - public void setJobContent(Map jobContent) { - this.jobContent = jobContent; - } - -} diff --git a/dss-appjoint-core/src/main/java/com/webank/wedatasphere/dss/appjoint/execution/core/AppJointNode.java b/dss-appjoint-core/src/main/java/com/webank/wedatasphere/dss/appjoint/execution/core/AppJointNode.java deleted file mode 100644 index 8295026c0e..0000000000 --- a/dss-appjoint-core/src/main/java/com/webank/wedatasphere/dss/appjoint/execution/core/AppJointNode.java +++ /dev/null @@ -1,51 +0,0 @@ -/* - * Copyright 2019 WeBank - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package com.webank.wedatasphere.dss.appjoint.execution.core; - -import com.webank.wedatasphere.dss.common.entity.node.Node; - -import java.util.Map; - -/** - * created by enjoyyin on 2019/9/25 - * Description: - */ -public interface AppJointNode extends Node { - /** - * 获取的node所属的project的id - * @return - */ - long getProjectId(); - - /** - * 获取node所属的project的name - * @return - */ - - String getProjectName(); - String getFlowName(); - long getFlowId(); - - /** - * 获取到Node的执行内容,执行内容是以Map的形式的 - * @return - */ - Map getJobContent(); - void setJobContent(Map jobContent); -} - diff --git a/dss-appjoint-core/src/main/java/com/webank/wedatasphere/dss/appjoint/execution/core/CallbackLongTermNodeExecution.java b/dss-appjoint-core/src/main/java/com/webank/wedatasphere/dss/appjoint/execution/core/CallbackLongTermNodeExecution.java deleted file mode 100644 index d5389504f6..0000000000 --- a/dss-appjoint-core/src/main/java/com/webank/wedatasphere/dss/appjoint/execution/core/CallbackLongTermNodeExecution.java +++ /dev/null @@ -1,59 +0,0 @@ - -/* - * Copyright 2019 WeBank - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package com.webank.wedatasphere.dss.appjoint.execution.core; - - -import com.webank.wedatasphere.dss.appjoint.execution.common.AsyncNodeExecutionResponse; -import com.webank.wedatasphere.dss.appjoint.execution.common.CompletedNodeExecutionResponse; -import com.webank.wedatasphere.dss.appjoint.execution.common.NodeExecutionAction; -import com.webank.wedatasphere.dss.appjoint.execution.conf.NodeExecutionConfiguration; - -import java.util.Map; -import java.util.concurrent.ConcurrentHashMap; - -/** - * created by enjoyyin on 2019/9/26 - * Description: - */ -public abstract class CallbackLongTermNodeExecution extends LongTermNodeExecution { - - protected Map asyncResponses = new ConcurrentHashMap(); - - public String getCallbackURL(NodeContext nodeContext){ - String urlSuffix = NodeExecutionConfiguration.CALL_BACK_URL().getValue(); - return nodeContext.getGatewayUrl() + urlSuffix; - } - - public abstract void acceptCallback(Map callbackMap); - - protected void markCompleted(NodeExecutionAction action, CompletedNodeExecutionResponse resultResponse) { - AsyncNodeExecutionResponse response = asyncResponses.get(action); - asyncResponses.remove(action); - response.getListeners().forEach(l -> l.onNodeExecutionCompleted(resultResponse)); - response.setCompleted(true); - } - - @Override - protected AsyncNodeExecutionResponse createAsyncNodeExecutionResponse(AppJointNode appJointNode, NodeContext context, NodeExecutionAction action) { - AsyncNodeExecutionResponse response = super.createAsyncNodeExecutionResponse(appJointNode, context, action); - response.setAskStatePeriod(NodeExecutionConfiguration.CALLBACK_NODE_EXECUTION_REFRESH_INTERVAL().getValue().toLong()); - asyncResponses.put(action, response); - return response; - } -} diff --git a/dss-appjoint-core/src/main/java/com/webank/wedatasphere/dss/appjoint/execution/core/CommonAppJointNode.java b/dss-appjoint-core/src/main/java/com/webank/wedatasphere/dss/appjoint/execution/core/CommonAppJointNode.java deleted file mode 100644 index e3e352f822..0000000000 --- a/dss-appjoint-core/src/main/java/com/webank/wedatasphere/dss/appjoint/execution/core/CommonAppJointNode.java +++ /dev/null @@ -1,71 +0,0 @@ -/* - * Copyright 2019 WeBank - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package com.webank.wedatasphere.dss.appjoint.execution.core; - -import java.util.ArrayList; -import java.util.List; -import java.util.Map; - -/** - * created by enjoyyin on 2019/10/9 - * Description: - */ -public class CommonAppJointNode extends AbstractAppJointNode{ - - private Map params; - - - private List dependencys = new ArrayList<>(); - - public CommonAppJointNode(String projectName, long projectId , String flowName, long flowId, String nodeName, - String type, Map jobContent){ - super(projectName, projectId, flowName, flowId, nodeName, type, jobContent); - } - - public CommonAppJointNode(){ - - } - - - public void setParams(Map params){ - this.params = params; - } - - - - - @Override - public void addDependency(String nodeName) { - this.dependencys.add(nodeName); - } - - @Override - public void setDependency(List dependencys) { - this.dependencys = dependencys; - } - - @Override - public void removeDependency(String nodeName) { - this.dependencys.remove(nodeName); - } - - @Override - public List getDependencys() { - return this.dependencys; - } -} diff --git a/dss-appjoint-core/src/main/java/com/webank/wedatasphere/dss/appjoint/execution/core/HttpLongTermNodeExecution.java b/dss-appjoint-core/src/main/java/com/webank/wedatasphere/dss/appjoint/execution/core/HttpLongTermNodeExecution.java deleted file mode 100644 index e33dee7ccf..0000000000 --- a/dss-appjoint-core/src/main/java/com/webank/wedatasphere/dss/appjoint/execution/core/HttpLongTermNodeExecution.java +++ /dev/null @@ -1,9 +0,0 @@ -package com.webank.wedatasphere.dss.appjoint.execution.core; - -/** - * created by enjoyyin on 2019/11/26 - * Description: - */ -public abstract class HttpLongTermNodeExecution extends LongTermNodeExecution{ - -} diff --git a/dss-appjoint-core/src/main/java/com/webank/wedatasphere/dss/appjoint/execution/core/Killable.java b/dss-appjoint-core/src/main/java/com/webank/wedatasphere/dss/appjoint/execution/core/Killable.java deleted file mode 100644 index 10f0a3a6df..0000000000 --- a/dss-appjoint-core/src/main/java/com/webank/wedatasphere/dss/appjoint/execution/core/Killable.java +++ /dev/null @@ -1,28 +0,0 @@ -/* - * Copyright 2019 WeBank - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package com.webank.wedatasphere.dss.appjoint.execution.core; - -import com.webank.wedatasphere.dss.appjoint.execution.common.NodeExecutionAction; - -/** - * created by enjoyyin on 2019/9/26 - * Description: - */ -public interface Killable { - public boolean kill(NodeExecutionAction action); -} diff --git a/dss-appjoint-core/src/main/java/com/webank/wedatasphere/dss/appjoint/execution/core/LongTermNodeExecution.java b/dss-appjoint-core/src/main/java/com/webank/wedatasphere/dss/appjoint/execution/core/LongTermNodeExecution.java deleted file mode 100644 index b56d53c85f..0000000000 --- a/dss-appjoint-core/src/main/java/com/webank/wedatasphere/dss/appjoint/execution/core/LongTermNodeExecution.java +++ /dev/null @@ -1,94 +0,0 @@ -/* - * Copyright 2019 WeBank - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package com.webank.wedatasphere.dss.appjoint.execution.core; - -import com.webank.wedatasphere.dss.appjoint.execution.NodeExecution; -import com.webank.wedatasphere.dss.appjoint.execution.async.NodeExecutionListener; -import com.webank.wedatasphere.dss.appjoint.execution.common.*; -import com.webank.wedatasphere.dss.appjoint.execution.scheduler.LongTermNodeExecutionScheduler; -import com.webank.wedatasphere.dss.appjoint.service.session.Session; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.util.ArrayList; -import java.util.List; - -/** - * created by enjoyyin on 2019/9/26 - * Description: LongTermNodeExecution 是一个 - */ -public abstract class LongTermNodeExecution implements NodeExecution { - - private static final Logger logger = LoggerFactory.getLogger(LongTermNodeExecution.class); - - private List nodeExecutionListener = new ArrayList(); - private LongTermNodeExecutionScheduler scheduler = SchedulerManager.getScheduler(); - - public void addNodeExecutionListener(NodeExecutionListener nodeExecutionListener) { - this.nodeExecutionListener.add(nodeExecutionListener); - } - - public LongTermNodeExecutionScheduler getScheduler() { - return scheduler; - } - - public void setScheduler(LongTermNodeExecutionScheduler scheduler) { - this.scheduler = scheduler; - } - - protected abstract NodeExecutionAction submit(AppJointNode appJointNode, NodeContext nodeContext, Session session); - - public abstract NodeExecutionState state(NodeExecutionAction action); - - public abstract CompletedNodeExecutionResponse result(NodeExecutionAction action, NodeContext nodeContext); - - - - @Override - public NodeExecutionResponse execute(AppJointNode appJointNode, NodeContext context, Session session) { - nodeExecutionListener.forEach(l -> l.beforeSubmit(appJointNode, context)); - NodeExecutionAction action = submit(appJointNode, context, session); - nodeExecutionListener.forEach(l -> l.afterSubmit(appJointNode, context, action)); - NodeExecutionState state = state(action); - if(state != null && state.isCompleted()) { - CompletedNodeExecutionResponse response = result(action, context); - nodeExecutionListener.forEach(l -> l.afterCompletedNodeExecutionResponse(appJointNode, context, action, response)); - return response; - } else { - AsyncNodeExecutionResponse response = createAsyncNodeExecutionResponse(appJointNode, context, action); - response.setNodeExecution(this); - response.addListener(r -> { - nodeExecutionListener.forEach(l -> l.afterCompletedNodeExecutionResponse(appJointNode, context, action, r)); - }); - nodeExecutionListener.forEach(l -> l.afterAsyncNodeExecutionResponse(response)); - if(scheduler != null) { - scheduler.addAsyncResponse(response); - } - return response; - } - } - - protected AsyncNodeExecutionResponse createAsyncNodeExecutionResponse(AppJointNode appJointNode, NodeContext context, NodeExecutionAction action) { - AsyncNodeExecutionResponse response = new AsyncNodeExecutionResponse(); - response.setAction(action); - response.setAppJointNode(appJointNode); - response.setNodeContext(context); - return response; - } - -} diff --git a/dss-appjoint-core/src/main/java/com/webank/wedatasphere/dss/appjoint/execution/core/NodeContext.java b/dss-appjoint-core/src/main/java/com/webank/wedatasphere/dss/appjoint/execution/core/NodeContext.java deleted file mode 100644 index 1d86103a66..0000000000 --- a/dss-appjoint-core/src/main/java/com/webank/wedatasphere/dss/appjoint/execution/core/NodeContext.java +++ /dev/null @@ -1,112 +0,0 @@ -/* - * Copyright 2019 WeBank - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package com.webank.wedatasphere.dss.appjoint.execution.core; - -import com.webank.wedatasphere.linkis.common.io.FsPath; -import com.webank.wedatasphere.linkis.common.io.MetaData; -import com.webank.wedatasphere.linkis.common.io.Record; -import com.webank.wedatasphere.linkis.common.io.resultset.ResultSet; -import com.webank.wedatasphere.linkis.common.io.resultset.ResultSetReader; -import com.webank.wedatasphere.linkis.common.io.resultset.ResultSetWriter; -import com.webank.wedatasphere.linkis.protocol.query.RequestPersistTask; -import com.webank.wedatasphere.linkis.storage.LineMetaData; -import com.webank.wedatasphere.linkis.storage.LineRecord; -import com.webank.wedatasphere.linkis.storage.resultset.table.TableMetaData; -import com.webank.wedatasphere.linkis.storage.resultset.table.TableRecord; - -import java.util.HashMap; -import java.util.Map; - -/** - * created by enjoyyin on 2019/10/12 - * Description: - */ -public interface NodeContext { - - - static final String TOKEN_USER_STR = "Token-User"; - - static final String TOKEN_CODE_STR = "Token-Code"; - - - public boolean isReadNode(); - - public AppJointNode getAppJointNode(); - - public Map getRuntimeMap(); - - public long[] getJobIdsOfShareNode(); - - public RequestPersistTask getJobById(long jobId); - - - //获取任务的所有的结果集路径 - FsPath[] getResultSetPathsByJobId(long jobId); - - public void appendLog(String log); - - public void updateProgress(float progress); - - /** - * 获取本节点的操作用户 - * @return - */ - public String getUser(); - - void setStorePath(String storePath); - - String getStorePath(); - - ResultSetWriter createTableResultSetWriter(); - - ResultSetWriter createTableResultSetWriter(String resultSetAlias); - - ResultSetWriter createTextResultSetWriter(); - - ResultSetWriter createTextResultSetWriter(String resultSetAlias); - - ResultSetWriter createHTMLResultSetWriter(); - - ResultSetWriter createHTMLResultSetWriter(String resultSetAlias); - - ResultSetWriter createPictureResultSetWriter(); - - ResultSetWriter createPictureResultSetWriter(String resultSetAlias); - - - - ResultSetWritercreateResultSetWriter(ResultSet resultSet, String resultSetAlias); - - - public ResultSetReader getResultSetReader(FsPath fsPath); - - /** - * - * @return - */ - public String getGatewayUrl(); - - - default Map getTokenHeader(String user){ - Map tokenHeader = new HashMap<>(); - tokenHeader.put(TOKEN_CODE_STR, "dss-AUTH"); - tokenHeader.put(TOKEN_USER_STR, user); - return tokenHeader; - } - -} diff --git a/dss-appjoint-core/src/main/java/com/webank/wedatasphere/dss/appjoint/execution/core/NodeContextImpl.java b/dss-appjoint-core/src/main/java/com/webank/wedatasphere/dss/appjoint/execution/core/NodeContextImpl.java deleted file mode 100644 index 99ab157013..0000000000 --- a/dss-appjoint-core/src/main/java/com/webank/wedatasphere/dss/appjoint/execution/core/NodeContextImpl.java +++ /dev/null @@ -1,94 +0,0 @@ -/* - * Copyright 2019 WeBank - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package com.webank.wedatasphere.dss.appjoint.execution.core; - -import com.webank.wedatasphere.dss.appjoint.service.session.Session; - -import java.util.Map; - -/** - * created by enjoyyin on 2019/10/12 - * Description: - */ -public abstract class NodeContextImpl implements NodeContext{ - - - private boolean isReadNode; - - private AppJointNode appJointNode; - - private Map runtimeMap; - - private long[] jobIdsOfShareNode; - - private String user; - - private Session session; - - - public void setIsReadNode(boolean isReadNode){ - this.isReadNode = isReadNode; - } - - public void setAppJointNode(AppJointNode appJointNode){ - this.appJointNode = appJointNode; - } - - public void setRuntimeMap(Map runtimeMap){ - this.runtimeMap = runtimeMap; - } - - public void setJobIdsOfShareNode(long[] jobIds){ - this.jobIdsOfShareNode = jobIds; - } - - public void setUser(String user){ - this.user = user; - } - - - - @Override - public boolean isReadNode() { - return this.isReadNode; - } - - @Override - public AppJointNode getAppJointNode() { - return this.appJointNode; - } - - @Override - public Map getRuntimeMap() { - return this.runtimeMap; - } - - @Override - public long[] getJobIdsOfShareNode() { - return this.jobIdsOfShareNode; - } - - @Override - public String getUser() { - return this.user; - } - - - - -} diff --git a/dss-appjoint-core/src/main/java/com/webank/wedatasphere/dss/appjoint/execution/core/ParamsNode.java b/dss-appjoint-core/src/main/java/com/webank/wedatasphere/dss/appjoint/execution/core/ParamsNode.java deleted file mode 100644 index ac8152ca07..0000000000 --- a/dss-appjoint-core/src/main/java/com/webank/wedatasphere/dss/appjoint/execution/core/ParamsNode.java +++ /dev/null @@ -1,28 +0,0 @@ -/* - * Copyright 2019 WeBank - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package com.webank.wedatasphere.dss.appjoint.execution.core; - -import java.util.Map; - -/** - * created by enjoyyin on 2019/9/29 - * Description: - */ -public interface ParamsNode { - public Map getParams(); -} diff --git a/dss-appjoint-core/src/main/java/com/webank/wedatasphere/dss/appjoint/execution/core/Procedure.java b/dss-appjoint-core/src/main/java/com/webank/wedatasphere/dss/appjoint/execution/core/Procedure.java deleted file mode 100644 index 726177cc3b..0000000000 --- a/dss-appjoint-core/src/main/java/com/webank/wedatasphere/dss/appjoint/execution/core/Procedure.java +++ /dev/null @@ -1,33 +0,0 @@ -/* - * Copyright 2019 WeBank - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package com.webank.wedatasphere.dss.appjoint.execution.core; - -import com.webank.wedatasphere.dss.appjoint.execution.common.NodeExecutionAction; - -/** - * created by enjoyyin on 2019/9/29 - * Description: - */ -public interface Procedure { - - public float progress(NodeExecutionAction action); - - public String log(NodeExecutionAction action); - - -} diff --git a/dss-appjoint-core/src/main/java/com/webank/wedatasphere/dss/appjoint/execution/core/SchedulerManager.java b/dss-appjoint-core/src/main/java/com/webank/wedatasphere/dss/appjoint/execution/core/SchedulerManager.java deleted file mode 100644 index a51dee9d86..0000000000 --- a/dss-appjoint-core/src/main/java/com/webank/wedatasphere/dss/appjoint/execution/core/SchedulerManager.java +++ /dev/null @@ -1,38 +0,0 @@ -/* - * Copyright 2019 WeBank - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package com.webank.wedatasphere.dss.appjoint.execution.core; - -import com.webank.wedatasphere.dss.appjoint.execution.scheduler.ListenerEventBusNodeExecutionScheduler; -import com.webank.wedatasphere.dss.appjoint.execution.scheduler.LongTermNodeExecutionScheduler; - -/** - * created by enjoyyin on 2019/11/15 - * Description: - */ -public class SchedulerManager { - - private static LongTermNodeExecutionScheduler scheduler = new ListenerEventBusNodeExecutionScheduler(); - - static{ - ((ListenerEventBusNodeExecutionScheduler)scheduler).start(); - } - - public static LongTermNodeExecutionScheduler getScheduler(){ - return scheduler; - } -} diff --git a/dss-appjoint-core/src/main/java/com/webank/wedatasphere/dss/appjoint/execution/scheduler/AsyncNodeExecutionResponseEvent.java b/dss-appjoint-core/src/main/java/com/webank/wedatasphere/dss/appjoint/execution/scheduler/AsyncNodeExecutionResponseEvent.java deleted file mode 100644 index c35108d06c..0000000000 --- a/dss-appjoint-core/src/main/java/com/webank/wedatasphere/dss/appjoint/execution/scheduler/AsyncNodeExecutionResponseEvent.java +++ /dev/null @@ -1,51 +0,0 @@ -/* - * Copyright 2019 WeBank - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package com.webank.wedatasphere.dss.appjoint.execution.scheduler; - -import com.webank.wedatasphere.dss.appjoint.execution.common.AsyncNodeExecutionResponse; -import com.webank.wedatasphere.dss.appjoint.execution.core.LongTermNodeExecution; -import com.webank.wedatasphere.linkis.common.listener.Event; - -/** - * Created by enjoyyin on 2019/11/13. - */ -public class AsyncNodeExecutionResponseEvent implements Event { - - private AsyncNodeExecutionResponse response; - private long lastAskTime = 0; - - public AsyncNodeExecutionResponseEvent(AsyncNodeExecutionResponse response) { - this.response = response; - } - - public AsyncNodeExecutionResponse getResponse() { - return response; - } - - public void setResponse(AsyncNodeExecutionResponse response) { - this.response = response; - } - - public long getLastAskTime() { - return lastAskTime; - } - - public void setLastAskTime() { - this.lastAskTime = System.currentTimeMillis(); - } -} diff --git a/dss-appjoint-core/src/main/java/com/webank/wedatasphere/dss/appjoint/execution/scheduler/AsyncNodeExecutionSchedulerListener.java b/dss-appjoint-core/src/main/java/com/webank/wedatasphere/dss/appjoint/execution/scheduler/AsyncNodeExecutionSchedulerListener.java deleted file mode 100644 index ff33af2b8a..0000000000 --- a/dss-appjoint-core/src/main/java/com/webank/wedatasphere/dss/appjoint/execution/scheduler/AsyncNodeExecutionSchedulerListener.java +++ /dev/null @@ -1,38 +0,0 @@ -/* - * Copyright 2019 WeBank - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package com.webank.wedatasphere.dss.appjoint.execution.scheduler; - -import com.webank.wedatasphere.linkis.common.listener.Event; -import com.webank.wedatasphere.linkis.common.listener.EventListener; - -/** - * Created by enjoyyin on 2019/11/13. - */ -public interface AsyncNodeExecutionSchedulerListener extends EventListener { - - void onEvent(AsyncNodeExecutionResponseEvent event); - - void onEventError(AsyncNodeExecutionResponseEvent event, Throwable t); - - @Override - default void onEventError(Event event, Throwable t) { - if(event instanceof AsyncNodeExecutionResponseEvent) { - onEventError((AsyncNodeExecutionResponseEvent) event, t); - } - } -} diff --git a/dss-appjoint-core/src/main/java/com/webank/wedatasphere/dss/appjoint/execution/scheduler/LongTermNodeExecutionScheduler.java b/dss-appjoint-core/src/main/java/com/webank/wedatasphere/dss/appjoint/execution/scheduler/LongTermNodeExecutionScheduler.java deleted file mode 100644 index 99430ddea9..0000000000 --- a/dss-appjoint-core/src/main/java/com/webank/wedatasphere/dss/appjoint/execution/scheduler/LongTermNodeExecutionScheduler.java +++ /dev/null @@ -1,39 +0,0 @@ -/* - * Copyright 2019 WeBank - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package com.webank.wedatasphere.dss.appjoint.execution.scheduler; - -import com.webank.wedatasphere.dss.appjoint.execution.common.AsyncNodeExecutionResponse; -import com.webank.wedatasphere.dss.appjoint.execution.common.LongTermNodeExecutionAction; -import com.webank.wedatasphere.dss.appjoint.execution.common.NodeExecutionAction; - -/** - * Created by enjoyyin on 2019/11/13. - */ -public interface LongTermNodeExecutionScheduler { - - void addAsyncResponse(AsyncNodeExecutionResponse response); - - void removeAsyncResponse(LongTermNodeExecutionAction action); - - AsyncNodeExecutionResponse getAsyncResponse(LongTermNodeExecutionAction action); - - void start(); - - void stop(); - -} diff --git a/dss-appjoint-core/src/main/java/com/webank/wedatasphere/dss/appjoint/service/AppJointUrl.java b/dss-appjoint-core/src/main/java/com/webank/wedatasphere/dss/appjoint/service/AppJointUrl.java deleted file mode 100644 index d3b00c7e76..0000000000 --- a/dss-appjoint-core/src/main/java/com/webank/wedatasphere/dss/appjoint/service/AppJointUrl.java +++ /dev/null @@ -1,29 +0,0 @@ -/* - * Copyright 2019 WeBank - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package com.webank.wedatasphere.dss.appjoint.service; - -/** - * Created by enjoyyin on 2019/11/5. - */ -public interface AppJointUrl { - - String getBaseUrl(); - - void setBaseUrl(String basicUrl); - -} diff --git a/dss-appjoint-core/src/main/java/com/webank/wedatasphere/dss/appjoint/service/AppJointUrlImpl.java b/dss-appjoint-core/src/main/java/com/webank/wedatasphere/dss/appjoint/service/AppJointUrlImpl.java deleted file mode 100644 index 017de5b5ed..0000000000 --- a/dss-appjoint-core/src/main/java/com/webank/wedatasphere/dss/appjoint/service/AppJointUrlImpl.java +++ /dev/null @@ -1,36 +0,0 @@ -/* - * Copyright 2019 WeBank - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package com.webank.wedatasphere.dss.appjoint.service; - -/** - * Created by enjoyyin on 2019/11/5. - */ -public abstract class AppJointUrlImpl implements AppJointUrl { - - private String baseUrl; - - @Override - public String getBaseUrl() { - return baseUrl; - } - - @Override - public void setBaseUrl(String basicUrl) { - this.baseUrl = basicUrl; - } -} diff --git a/dss-appjoint-core/src/main/java/com/webank/wedatasphere/dss/appjoint/service/NodeService.java b/dss-appjoint-core/src/main/java/com/webank/wedatasphere/dss/appjoint/service/NodeService.java deleted file mode 100644 index 3a859b00ea..0000000000 --- a/dss-appjoint-core/src/main/java/com/webank/wedatasphere/dss/appjoint/service/NodeService.java +++ /dev/null @@ -1,59 +0,0 @@ -/* - * Copyright 2019 WeBank - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package com.webank.wedatasphere.dss.appjoint.service; - -import com.webank.wedatasphere.dss.appjoint.execution.core.AppJointNode; -import com.webank.wedatasphere.dss.appjoint.exception.AppJointErrorException; -import com.webank.wedatasphere.dss.appjoint.service.session.Session; - -import java.util.ArrayList; -import java.util.List; -import java.util.Map; - -/** - * Created by enjoyyin on 2019/10/10. - */ -public interface NodeService extends AppJointUrl { - - /** - * 根据参数创建外部节点 - * @param session 连接会话 - * @param node 节点信息 - * @param requestBody 创建节点需要的参数，前端带过来的Json结构字符串。 - * @return 返回jobContent的Map，工作流会将该Map存储起来，作为该节点的关键关联信息，用于后续的CRUD和执行。 - */ - Map createNode(Session session, AppJointNode node, - Map requestBody) throws AppJointErrorException; - - void deleteNode(Session session, AppJointNode node) throws AppJointErrorException; - - Map updateNode(Session session, AppJointNode node, Map requestBody) throws AppJointErrorException; - - default Map refresh(Session session, AppJointNode node) throws AppJointErrorException { - return null; - } - - default void copyNode(Session session, AppJointNode newNode, AppJointNode oldNode) throws AppJointErrorException { } - - default void setNodeReadOnly(Session session, AppJointNode node) throws AppJointErrorException {} - - default List listNodes(Session session, AppJointNode node) throws AppJointErrorException { - return new ArrayList<>(); - } - -} diff --git a/dss-appjoint-core/src/main/java/com/webank/wedatasphere/dss/appjoint/service/ProjectService.java b/dss-appjoint-core/src/main/java/com/webank/wedatasphere/dss/appjoint/service/ProjectService.java deleted file mode 100644 index 30edffd847..0000000000 --- a/dss-appjoint-core/src/main/java/com/webank/wedatasphere/dss/appjoint/service/ProjectService.java +++ /dev/null @@ -1,43 +0,0 @@ -/* - * Copyright 2019 WeBank - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package com.webank.wedatasphere.dss.appjoint.service; - - -import com.webank.wedatasphere.dss.appjoint.exception.AppJointErrorException; -import com.webank.wedatasphere.dss.appjoint.service.session.Session; -import com.webank.wedatasphere.dss.common.entity.project.Project; - -import java.util.List; - -/** - * Created by enjoyyin on 2019/9/16. - */ -public interface ProjectService extends AppJointUrl { - - List fetchProjects(Session session) throws AppJointErrorException; - - Project createProject(Project project, Session session) throws AppJointErrorException; - - void deleteProject(Project project, Session session) throws AppJointErrorException; - - default Project getProject(Project project, Session session) throws AppJointErrorException { - return null; - } - - void updateProject(Project project, Session session) throws AppJointErrorException; -} diff --git a/dss-appjoint-core/src/main/java/com/webank/wedatasphere/dss/appjoint/service/SecurityService.java b/dss-appjoint-core/src/main/java/com/webank/wedatasphere/dss/appjoint/service/SecurityService.java deleted file mode 100644 index a1a035032d..0000000000 --- a/dss-appjoint-core/src/main/java/com/webank/wedatasphere/dss/appjoint/service/SecurityService.java +++ /dev/null @@ -1,32 +0,0 @@ -/* - * Copyright 2019 WeBank - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package com.webank.wedatasphere.dss.appjoint.service; - -import com.webank.wedatasphere.dss.appjoint.exception.AppJointErrorException; -import com.webank.wedatasphere.dss.appjoint.service.session.Session; - -/** - * Created by enjoyyin on 2019/9/16. - */ -public interface SecurityService extends AppJointUrl { - - Session login(String user) throws AppJointErrorException; - - void logout(String user) throws AppJointErrorException; - -} diff --git a/dss-appjoint-core/src/main/java/com/webank/wedatasphere/dss/appjoint/service/session/Session.java b/dss-appjoint-core/src/main/java/com/webank/wedatasphere/dss/appjoint/service/session/Session.java deleted file mode 100644 index 4f0f956748..0000000000 --- a/dss-appjoint-core/src/main/java/com/webank/wedatasphere/dss/appjoint/service/session/Session.java +++ /dev/null @@ -1,41 +0,0 @@ -/* - * Copyright 2019 WeBank - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package com.webank.wedatasphere.dss.appjoint.service.session; - -import org.apache.http.Header; -import org.apache.http.cookie.Cookie; - -import java.util.Map; - -/** - * Created by enjoyyin on 2019/9/16. - */ -public interface Session { - - String getUser(); - - Cookie[] getCookies(); - - Header[] getHeaders(); - - Map getParameters(); - - long getLastAccessTime(); - - void updateLastAccessTime(); -} diff --git a/dss-appjoint-core/src/main/scala/com/webank/wedatasphere/dss/appjoint/execution/conf/NodeExecutionConfiguration.scala b/dss-appjoint-core/src/main/scala/com/webank/wedatasphere/dss/appjoint/execution/conf/NodeExecutionConfiguration.scala deleted file mode 100644 index dca56c995f..0000000000 --- a/dss-appjoint-core/src/main/scala/com/webank/wedatasphere/dss/appjoint/execution/conf/NodeExecutionConfiguration.scala +++ /dev/null @@ -1,36 +0,0 @@ -/* - * Copyright 2019 WeBank - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package com.webank.wedatasphere.dss.appjoint.execution.conf - -import com.webank.wedatasphere.linkis.common.conf.{CommonVars, TimeType} - -/** - * created by enjoyyin on 2019/9/29 - * Description: - */ -object NodeExecutionConfiguration { - val QUERY_APPLICATION_NAME = CommonVars("wds.dss.appjoint.query.service.name", "cloud-query") - val CALL_BACK_URL = CommonVars("wds.dss.appjoint.callback.url", "/api/rest_j/v1/entrance/callback") - - val ASYNC_NODE_EXECUTION_SCHEDULER_QUEUE_SIZE = CommonVars("wds.dss.appjoint.nodeexecution.scheduler.queue.size", 2000) - val ASYNC_NODE_EXECUTION_SCHEDULER_THREAD_SIZE = CommonVars("wds.dss.appjoint.nodeexecution.scheduler.thread.max", 20) - - val CALLBACK_NODE_EXECUTION_REFRESH_INTERVAL = CommonVars("wds.dss.appjoint.nodeexecution.callback.refresh.interval", new TimeType("2m")) - - val WORKFLOW_SHARED_NODES_JOBIDS = "workflow.shared.nodes.jobids" -} diff --git a/dss-appjoint-core/src/main/scala/com/webank/wedatasphere/dss/appjoint/execution/context/NodeExecutionContext.scala b/dss-appjoint-core/src/main/scala/com/webank/wedatasphere/dss/appjoint/execution/context/NodeExecutionContext.scala deleted file mode 100644 index 7381bae2c5..0000000000 --- a/dss-appjoint-core/src/main/scala/com/webank/wedatasphere/dss/appjoint/execution/context/NodeExecutionContext.scala +++ /dev/null @@ -1,81 +0,0 @@ -/* - * Copyright 2019 WeBank - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package com.webank.wedatasphere.dss.appjoint.execution.context - -import com.webank.wedatasphere.linkis.common.utils.Logging - -/** - * created by enjoyyin on 2019/9/27 - * Description: - */ -object NodeExecutionContext extends Logging{ - -// private lazy val querySender = Sender.getSender(NodeExecutionConfiguration.QUERY_APPLICATION_NAME.getValue) -// -// val SUCCESS_FLAG:Int = 0 -// -// val TASK_MAP_KEY:String = "task" -// -// /** -// * 通过的taskId将所有的node都拿到 -// * @param taskIds -// * @return -// */ -// -// def getNodes(taskIds:Array[Long]):Array[AppJointNode] = { -// val nodes = new ArrayBuffer[AppJointNode]() -// val requestPersistTasks = new ArrayBuffer[RequestPersistTask]() -// taskIds foreach { -// taskId => val requestQueryTask = new RequestQueryTask() -// requestQueryTask.setTaskID(taskId) -// Utils.tryCatch{ -// val taskResponse = querySender.ask(requestQueryTask) -// taskResponse match { -// case responsePersist:ResponsePersist => val status = responsePersist.getStatus -// if (status != SUCCESS_FLAG){ -// logger.error(s"query from jobHistory status failed, status is $status") -// throw new AppJointErrorException(75533, "query from jobHistory status failed") -// }else{ -// val data = responsePersist.getData -// data.get(TASK_MAP_KEY) match { -// case tasks:util.List[RequestPersistTask] => tasks.get(0) match { -// case requestPersistTask:RequestPersistTask => requestPersistTasks += requestPersistTask -// case _ => logger.error(s"query from jobhistory not a correct RequestPersistTask type taskId is $taskId") -// throw new AppJointErrorException(75533, s"query from jobhistory not a correct RequestPersistTask type taskId is $taskId") -// } -// case _ => logger.error(s"query from jobhistory not a correct List type taskId is $taskId") -// throw new AppJointErrorException(75533, s"query from jobhistory not a correct List type taskId is $taskId") -// } -// } -// case _ => logger.error("get query response incorrectly") -// throw new AppJointErrorException(75533, "get query response incorrectly") -// } -// }{ -// case errorException:ErrorException => logger.error(s"query taskId $taskId error", errorException) -// throw errorException -// case e:Exception => logger.error(s"query taskId $taskId error", e) -// throw new AppJointErrorException(75533, s"query taskId $taskId error") -// } -// } -// requestPersistTasks foreach { -// requestPersistTask => requestPersistTask.getSource -// } -// nodes.toArray -// } - -} diff --git a/dss-appjoint-core/src/main/scala/com/webank/wedatasphere/dss/appjoint/execution/scheduler/ListenerEventBusNodeExecutionScheduler.scala b/dss-appjoint-core/src/main/scala/com/webank/wedatasphere/dss/appjoint/execution/scheduler/ListenerEventBusNodeExecutionScheduler.scala deleted file mode 100644 index 51d167eaff..0000000000 --- a/dss-appjoint-core/src/main/scala/com/webank/wedatasphere/dss/appjoint/execution/scheduler/ListenerEventBusNodeExecutionScheduler.scala +++ /dev/null @@ -1,125 +0,0 @@ -/* - * Copyright 2019 WeBank - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package com.webank.wedatasphere.dss.appjoint.execution.scheduler - -import java.util.concurrent.ArrayBlockingQueue - -import com.webank.wedatasphere.dss.appjoint.exception.AppJointErrorException -import com.webank.wedatasphere.dss.appjoint.execution.common.{AsyncNodeExecutionResponse, CompletedNodeExecutionResponse, LongTermNodeExecutionAction} -import com.webank.wedatasphere.dss.appjoint.execution.conf.NodeExecutionConfiguration._ -import com.webank.wedatasphere.linkis.common.collection.BlockingLoopArray -import com.webank.wedatasphere.linkis.common.listener.ListenerEventBus -import com.webank.wedatasphere.linkis.common.utils.{ByteTimeUtils, Utils} -import org.apache.commons.lang.time.DateFormatUtils - -import scala.collection.JavaConversions._ - -/** - * Created by enjoyyin on 2019/11/13. - */ -class ListenerEventBusNodeExecutionScheduler(eventQueueCapacity: Int, name: String)(listenerConsumerThreadSize: Int) - extends LongTermNodeExecutionScheduler { - - private val listenerEventBus = new ListenerEventBus[AsyncNodeExecutionSchedulerListener, AsyncNodeExecutionResponseEvent](eventQueueCapacity, name)(listenerConsumerThreadSize) { - override protected val dropEvent: DropEvent = new DropEvent { - override def onDropEvent(event: AsyncNodeExecutionResponseEvent): Unit = throw new AppJointErrorException(95536, "LongTermNodeExecutionScheduler is full, please ask admin for help!") - override def onBusStopped(event: AsyncNodeExecutionResponseEvent): Unit = throw new AppJointErrorException(95536, "LongTermNodeExecutionScheduler is stopped, please ask admin for help!") - } - override def doPostEvent(listener: AsyncNodeExecutionSchedulerListener, event: AsyncNodeExecutionResponseEvent): Unit = { - listener.onEvent(event) - } - } - - def this() = { - this(ASYNC_NODE_EXECUTION_SCHEDULER_QUEUE_SIZE.getValue, "Async-NodeExecution-Scheduler")(ASYNC_NODE_EXECUTION_SCHEDULER_THREAD_SIZE.getValue) - getAsyncNodeExecutionSchedulerListeners.foreach(listenerEventBus.addListener) - } - - private val eventQueue = { - val ru = scala.reflect.runtime.universe - val classMirror = ru.runtimeMirror(getClass.getClassLoader) - val listenerEventBusClass = classMirror.reflect(listenerEventBus) - val field1 = ru.typeOf[ListenerEventBus[_, _]].decl(ru.TermName("eventQueue")).asMethod - val result = listenerEventBusClass.reflectMethod(field1) - result() match { - case queue: ArrayBlockingQueue[AsyncNodeExecutionResponseEvent] => queue - } - } - - protected def getAsyncNodeExecutionSchedulerListeners: Array[AsyncNodeExecutionSchedulerListener] = { - Array(new AsyncNodeExecutionSchedulerListener() { - override def onEvent(event: AsyncNodeExecutionResponseEvent): Unit = if(!event.getResponse.isCompleted) { - val response = event.getResponse - if(response.getMaxLoopTime > 0 && System.currentTimeMillis - response.getStartTime >= response.getMaxLoopTime) { - onEventError(event, new AppJointErrorException(75533, s"AppJointNode Execution is overtime! StartTime is ${DateFormatUtils.format(response.getStartTime, "yyyy-MM-dd HH:mm:ss")}, maxWaitTime is " + - ByteTimeUtils.msDurationToString(response.getMaxLoopTime))) - return - } - val period = System.currentTimeMillis() - event.getLastAskTime - if(period < response.getAskStatePeriod) { - if(period < 10) Utils.sleepQuietly(100) - if(!response.isCompleted) addEvent(event) - return - } - val state = response.getNodeExecution.state(response.getAction) - if(state.isCompleted) { - val resultResponse = response.getNodeExecution.result(response.getAction, response.getNodeContext) - onEventCompleted(event, resultResponse) - } else if(!response.isCompleted) { - event.setLastAskTime() - addEvent(event) - } - } - - private def onEventCompleted(event: AsyncNodeExecutionResponseEvent, response: CompletedNodeExecutionResponse): Unit = { - event.getResponse.getListeners.foreach(_.onNodeExecutionCompleted(response)) - event.getResponse.setCompleted(true) - } - - override def onEventError(event: AsyncNodeExecutionResponseEvent, t: Throwable): Unit = t match { - case e: Exception => - val response = new CompletedNodeExecutionResponse - response.setIsSucceed(false) - response.setException(e) - onEventCompleted(event, response) - } - }) - } - - override def addAsyncResponse(response: AsyncNodeExecutionResponse): Unit = - addEvent(new AsyncNodeExecutionResponseEvent(response)) - - protected def addEvent(event: AsyncNodeExecutionResponseEvent): Unit = synchronized { - listenerEventBus.post(event) -// event.getResponse.getAction match { -// case longTermAction: LongTermNodeExecutionAction => -// longTermAction.setSchedulerId(eventQueue.max) -// case _ => -// } - } - - override def removeAsyncResponse(action: LongTermNodeExecutionAction): Unit = { - - } - - override def getAsyncResponse(action: LongTermNodeExecutionAction): AsyncNodeExecutionResponse = null - - override def start(): Unit = listenerEventBus.start() - - override def stop(): Unit = listenerEventBus.stop() -} \ No newline at end of file diff --git a/dss-appjoint-loader/pom.xml b/dss-appjoint-loader/pom.xml deleted file mode 100644 index 99ab6ac8be..0000000000 --- a/dss-appjoint-loader/pom.xml +++ /dev/null @@ -1,71 +0,0 @@ - - - - - - dss - com.webank.wedatasphere.dss - 0.9.1 - - 4.0.0 - - dss-appjoint-loader - 0.9.1 - - - - com.webank.wedatasphere.dss - dss-appjoint-core - ${dss.version} - - - - - - org.apache.maven.plugins - maven-deploy-plugin - - - - net.alchim31.maven - scala-maven-plugin - - - org.apache.maven.plugins - maven-jar-plugin - - - **/*.yml - **/*.properties - **/*.sh - **/log4j2.xml - - - - - - - src/main/resources - - - - - - \ No newline at end of file diff --git a/dss-appjoint-loader/src/main/java/com/webank/wedatasphere/dss/appjoint/clazzloader/AppJointClassLoader.java b/dss-appjoint-loader/src/main/java/com/webank/wedatasphere/dss/appjoint/clazzloader/AppJointClassLoader.java deleted file mode 100644 index c0c04dde6f..0000000000 --- a/dss-appjoint-loader/src/main/java/com/webank/wedatasphere/dss/appjoint/clazzloader/AppJointClassLoader.java +++ /dev/null @@ -1,59 +0,0 @@ -/* - * Copyright 2019 WeBank - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package com.webank.wedatasphere.dss.appjoint.clazzloader; - -import com.google.common.collect.ImmutableList; -import com.google.common.collect.Iterables; -import com.webank.wedatasphere.dss.appjoint.loader.AppJointLoader; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.lang.reflect.InvocationTargetException; -import java.lang.reflect.Method; -import java.net.URL; -import java.net.URLClassLoader; -import java.util.List; -import java.util.ServiceLoader; - -import static java.util.Objects.requireNonNull; - -/** - * created by cooperyang on 2019/11/8 - * Description: - */ -public class AppJointClassLoader extends URLClassLoader{ - - private static final Logger logger = LoggerFactory.getLogger(AppJointClassLoader.class); - - - - public AppJointClassLoader(URL[] urls, ClassLoader parent) { - super(urls, parent); - } - - - @Override - public Class loadClass(String name) throws ClassNotFoundException { - return super.loadClass(name); - } - - @Override - protected Class loadClass(String name, boolean resolve) throws ClassNotFoundException { - return super.loadClass(name, resolve); - } -} diff --git a/dss-appjoint-loader/src/main/java/com/webank/wedatasphere/dss/appjoint/loader/AppJointLoader.java b/dss-appjoint-loader/src/main/java/com/webank/wedatasphere/dss/appjoint/loader/AppJointLoader.java deleted file mode 100644 index 5b890333a3..0000000000 --- a/dss-appjoint-loader/src/main/java/com/webank/wedatasphere/dss/appjoint/loader/AppJointLoader.java +++ /dev/null @@ -1,59 +0,0 @@ -/* - * Copyright 2019 WeBank - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package com.webank.wedatasphere.dss.appjoint.loader; - -import com.webank.wedatasphere.dss.appjoint.AppJoint; -import com.webank.wedatasphere.linkis.common.exception.ErrorException; - -import java.util.Map; -/** - * created by cooperyang on 2019/11/8 - * Description: AppJointLoader 是为了上层模块，如server，entrance等模块能够获取到指定的AppJoint - */ -public interface AppJointLoader { - - - String DIR_NAME = "appjoints"; - - String APPJOINT_DIR_NAME = "dss-appjoints"; - - String PROPERTIES_NAME = "appjoint.properties"; - - String CONF_NAME = "conf"; - - String LIB_NAME = "lib"; - - String JAR_SUF_NAME = ".jar"; - - String FILE_SCHEMA = "file://"; - - /** - * 获取一个appjoint,需要通过baseUrl,name 和 params进行获取 - * @param baseUrl appjoint代理的外部系统的url - * @return 一个appjoint - */ - - AppJoint getAppJoint(String baseUrl, String appJointName, Map params) throws Exception; - - - static AppJointLoader getAppJointLoader(){ - return AppJointLoaderFactory.getAppJointLoader(); - } - - -} diff --git a/dss-appjoint-loader/src/main/java/com/webank/wedatasphere/dss/appjoint/loader/AppJointLoaderFactory.java b/dss-appjoint-loader/src/main/java/com/webank/wedatasphere/dss/appjoint/loader/AppJointLoaderFactory.java deleted file mode 100644 index fead09f7c3..0000000000 --- a/dss-appjoint-loader/src/main/java/com/webank/wedatasphere/dss/appjoint/loader/AppJointLoaderFactory.java +++ /dev/null @@ -1,67 +0,0 @@ -/* - * Copyright 2019 WeBank - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package com.webank.wedatasphere.dss.appjoint.loader; - -import com.webank.wedatasphere.dss.appjoint.conf.AppJointLoaderConf; -import org.apache.commons.lang.ClassUtils; -import org.apache.commons.lang.StringUtils; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** - * created by cooperyang on 2019/11/8 - * Description: - */ -class AppJointLoaderFactory { - - - private static Class clazz = CommonAppJointLoader.class; - private static AppJointLoader appJointLoader = null; - private static final Logger logger = LoggerFactory.getLogger(AppJointLoaderFactory.class); - - - static AppJointLoader getAppJointLoader(){ - if (appJointLoader == null){ - synchronized (AppJointLoaderFactory.class){ - if (appJointLoader == null){ - String className = AppJointLoaderConf.CLASS_LOADER_CLASS_NAME().getValue(); - if (clazz == CommonAppJointLoader.class && StringUtils.isNotBlank(className)){ - try{ - clazz = ClassUtils.getClass(className); - }catch(ClassNotFoundException e){ - logger.warn("can not get class {}", className, e); - } - try { - appJointLoader = clazz.newInstance(); - } catch (Exception e) { - logger.error("can not initialize class", e); - } - }else{ - try { - appJointLoader = clazz.newInstance(); - } catch (Exception e) { - logger.error("can not initialize class", e); - } - } - } - } - } - return appJointLoader; - } -} diff --git a/dss-appjoint-loader/src/main/java/com/webank/wedatasphere/dss/appjoint/loader/CommonAppJointLoader.java b/dss-appjoint-loader/src/main/java/com/webank/wedatasphere/dss/appjoint/loader/CommonAppJointLoader.java deleted file mode 100644 index 681f8de21d..0000000000 --- a/dss-appjoint-loader/src/main/java/com/webank/wedatasphere/dss/appjoint/loader/CommonAppJointLoader.java +++ /dev/null @@ -1,150 +0,0 @@ -/* - * Copyright 2019 WeBank - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package com.webank.wedatasphere.dss.appjoint.loader; - -import com.webank.wedatasphere.dss.appjoint.AppJoint; -import com.webank.wedatasphere.dss.appjoint.clazzloader.AppJointClassLoader; -import com.webank.wedatasphere.dss.appjoint.utils.AppJointUtils; -import com.webank.wedatasphere.dss.appjoint.utils.ExceptionHelper; -import com.webank.wedatasphere.linkis.common.exception.ErrorException; -import org.apache.commons.lang.StringUtils; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.*; -import java.net.MalformedURLException; -import java.net.URL; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Properties; - -/** - * created by cooperyang on 2019/11/8 - * Description: - */ -public class CommonAppJointLoader implements AppJointLoader{ - - - private static final Logger logger = LoggerFactory.getLogger(CommonAppJointLoader.class); - - private final Map appJoints = new HashMap<>(); - - - /** - * 用来存放每一个appjoint所用到的classloader，这样的话，每一个appjoint的classloader都是不一样的 - */ - private final Map classLoaders = new HashMap<>(); - - - - /** - * - * @param baseUrl appjoint代理的外部系统的url - * @param appJointName appJoint的名字 - * @param params 参数用来进行init - * @return - * 命名规范必须是 ${DSS_HOME}/appjoints/${appjointName}/ - */ - @Override - public AppJoint getAppJoint(String baseUrl, String appJointName, Map params) throws Exception{ - synchronized (appJoints){ - if (appJoints.containsKey(appJointName)){ - return appJoints.get(appJointName); - } - } - if (params == null) { - params = new HashMap(); - } - ClassLoader oldClassLoader = Thread.currentThread().getContextClassLoader(); - ClassLoader newClassLoader = null; - synchronized (classLoaders){ - if (classLoaders.containsKey(appJointName)){ - newClassLoader = classLoaders.get(appJointName); - } - } - URL classpathUrl = oldClassLoader.getResource(""); - logger.info("classpathUrl is {}", classpathUrl.getPath()); - if(null == classpathUrl){ - throw new ErrorException(70059, "classPathUrl is null"); - } - - - - String basePathUrlStr = classpathUrl.getPath() + ".." + File.separator + ".." + File.separator + AppJointLoader.APPJOINT_DIR_NAME; - - String libPathUrlStr = basePathUrlStr + File.separator + appJointName + - File.separator + AppJointLoader.LIB_NAME; - String propertiesUrlStr = basePathUrlStr + File.separator + appJointName + - File.separator + AppJointLoader.PROPERTIES_NAME; - try{ - params.putAll(readFromProperties(propertiesUrlStr)); - }catch(IOException e){ - logger.warn("cannot get properties from {}", propertiesUrlStr, e); - } - URL finalURL = null; - try { - finalURL = new URL(AppJointLoader.FILE_SCHEMA + libPathUrlStr + "/*"); - } catch (MalformedURLException e) { - ExceptionHelper.dealErrorException(70061, libPathUrlStr + " url is wrong", e); - } - List jars = AppJointUtils.getJarsUrlsOfPath(libPathUrlStr); - if (newClassLoader == null){ - newClassLoader = new AppJointClassLoader(jars.toArray(new URL[100]) ,oldClassLoader); - } - synchronized (classLoaders){ - classLoaders.put(appJointName, newClassLoader); - } - Thread.currentThread().setContextClassLoader(newClassLoader); - String fullClassName = AppJointUtils.getAppJointClassName(appJointName, libPathUrlStr, newClassLoader); - Class clazz = null; - try { - clazz = newClassLoader.loadClass(fullClassName); - } catch (ClassNotFoundException e) { - ExceptionHelper.dealErrorException(70062, fullClassName + " class not found ", e); - } - if (clazz == null){ - Thread.currentThread().setContextClassLoader(oldClassLoader); - return null; - }else{ - AppJoint retAppjoint = (AppJoint) clazz.newInstance(); - if (StringUtils.isEmpty(baseUrl) && params.get("baseUrl") != null){ - baseUrl = params.get("baseUrl").toString(); - } - retAppjoint.init(baseUrl, params); - Thread.currentThread().setContextClassLoader(oldClassLoader); - synchronized (appJoints){ - appJoints.put(appJointName, retAppjoint); - } - logger.info("appJointName is {}, retAppJoint is {}", appJointName, retAppjoint.getClass().toString()); - return retAppjoint; - } - } - - private Map readFromProperties(String propertiesFile) throws IOException { - Properties properties = new Properties(); - BufferedReader reader = new BufferedReader(new FileReader(propertiesFile)); - properties.load(reader); - Map map = new HashMap((Map)properties); - return map; - } - - - - -} diff --git a/dss-appjoint-loader/src/main/java/com/webank/wedatasphere/dss/appjoint/utils/AppJointUtils.java b/dss-appjoint-loader/src/main/java/com/webank/wedatasphere/dss/appjoint/utils/AppJointUtils.java deleted file mode 100644 index be7ceb0b3c..0000000000 --- a/dss-appjoint-loader/src/main/java/com/webank/wedatasphere/dss/appjoint/utils/AppJointUtils.java +++ /dev/null @@ -1,160 +0,0 @@ -/* - * Copyright 2019 WeBank - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package com.webank.wedatasphere.dss.appjoint.utils; - -import com.webank.wedatasphere.dss.appjoint.AppJoint; -import com.webank.wedatasphere.dss.appjoint.exception.NoSuchAppJointException; -import com.webank.wedatasphere.dss.appjoint.loader.AppJointLoader; -import com.webank.wedatasphere.linkis.common.exception.ErrorException; -import org.apache.commons.lang.StringUtils; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.File; -import java.io.IOException; -import java.lang.reflect.Modifier; -import java.net.MalformedURLException; -import java.net.URL; -import java.util.ArrayList; -import java.util.Enumeration; -import java.util.List; -import java.util.jar.JarEntry; -import java.util.jar.JarFile; - -/** - * created by cooperyang on 2019/11/9 - * Description: - */ -public class AppJointUtils { - - - private static final Logger logger = LoggerFactory.getLogger(AppJointUtils.class); - - - private static Class PARENT_CLASS = AppJoint.class; - /** - * 通过appJointName去获取到相应的appjoint的全限定类名 - * @param appJointName appJointname - * @param libPath ${DSS_HOME}目录 - * @return appjointName对应的的appjoint的全限定类名 - */ - - public static String getAppJointClassName(String appJointName, String libPath, ClassLoader classLoader)throws NoSuchAppJointException { - //1.获取目录下面所有的jar包 - List jars = getJarsOfPath(libPath); - //2.从所有的jar中获取到AppJoint的子类 - boolean flag = false; - String appjoint = null; - for (String jar : jars){ - for(String clazzName : getClassNameFrom(jar)){ - if (isChildClass(clazzName, PARENT_CLASS, classLoader)){ - flag = true; - appjoint = clazzName; - break; - } - } - if (flag){ - break; - } - } - if (StringUtils.isEmpty(appjoint)){ - throw new NoSuchAppJointException(appJointName + " does not exist"); - } - return appjoint; - } - - - public static List getJarsOfPath(String path){ - File file = new File(path); - List jars = new ArrayList<>(); - if (file.listFiles() != null){ - for(File f : file.listFiles()){ - if (!f.isDirectory() && f.getName().endsWith(AppJointLoader.JAR_SUF_NAME) && f.getName().startsWith("dss")){ - jars.add(f.getPath()); - } - } - } - return jars; - } - - - public static List getJarsUrlsOfPath(String path){ - File file = new File(path); - List jars = new ArrayList<>(); - if (file.listFiles() != null){ - for(File f : file.listFiles()){ - if (!f.isDirectory() && f.getName().endsWith(AppJointLoader.JAR_SUF_NAME)){ - try { - jars.add(f.toURI().toURL()); - } catch (MalformedURLException e) { - logger.warn("url {} cannot be added", AppJointLoader.FILE_SCHEMA + f.getPath()); - } - } - } - } - return jars; - } - - - /** - * 从jar包读取所有的class文件名 - */ - private static List getClassNameFrom(String jarName) { - List fileList = new ArrayList(); - try { - JarFile jarFile = new JarFile(new File(jarName)); - Enumeration en = jarFile.entries(); - while (en.hasMoreElements()) { - String name1 = en.nextElement().getName(); - if (!name1.endsWith(".class")) { - continue; - } - String name2 = name1.substring(0, name1.lastIndexOf(".class")); - String name3 = name2.replaceAll("/", "."); - fileList.add(name3); - } - } catch (IOException e) { - e.printStackTrace(); - } - - return fileList; - } - - - private static boolean isChildClass(String className, Class parentClazz, ClassLoader classLoader) { - if (StringUtils.isEmpty(className)) { - return false; - } - Class clazz = null; - try { - clazz = classLoader.loadClass(className); - //忽略抽象类和接口 - if (Modifier.isAbstract(clazz.getModifiers())) { - return false; - } - if (Modifier.isInterface(clazz.getModifiers())) { - return false; - } - } catch (Throwable t) { - logger.error("className {} can not be instanced", className); - return false; - } - return parentClazz.isAssignableFrom(clazz); - } - -} diff --git a/dss-appjoint-loader/src/main/java/com/webank/wedatasphere/dss/appjoint/utils/ExceptionHelper.java b/dss-appjoint-loader/src/main/java/com/webank/wedatasphere/dss/appjoint/utils/ExceptionHelper.java deleted file mode 100644 index 39a80227a2..0000000000 --- a/dss-appjoint-loader/src/main/java/com/webank/wedatasphere/dss/appjoint/utils/ExceptionHelper.java +++ /dev/null @@ -1,32 +0,0 @@ -/* - * Copyright 2019 WeBank - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package com.webank.wedatasphere.dss.appjoint.utils; - -import com.webank.wedatasphere.linkis.common.exception.ErrorException; - -/** - * created by cooperyang on 2019/11/10 - * Description: - */ -public class ExceptionHelper { - public static void dealErrorException(int errorCode, String errorMsg, Throwable t) throws ErrorException { - ErrorException errorException = new ErrorException(errorCode, errorMsg); - errorException.initCause(t); - throw errorException; - } -} diff --git a/dss-appjoint-loader/src/main/scala/com/webank/wedatasphere/dss/appjoint/exception/NoSuchAppJointException.scala b/dss-appjoint-loader/src/main/scala/com/webank/wedatasphere/dss/appjoint/exception/NoSuchAppJointException.scala deleted file mode 100644 index 735b893ee9..0000000000 --- a/dss-appjoint-loader/src/main/scala/com/webank/wedatasphere/dss/appjoint/exception/NoSuchAppJointException.scala +++ /dev/null @@ -1,26 +0,0 @@ -/* - * Copyright 2019 WeBank - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package com.webank.wedatasphere.dss.appjoint.exception - -import com.webank.wedatasphere.linkis.common.exception.ErrorException - -/** - * created by cooperyang on 2019/11/10 - * Description: - */ -case class NoSuchAppJointException(errDesc:String) extends ErrorException(70059, errDesc) diff --git a/dss-application/pom.xml b/dss-application/pom.xml deleted file mode 100644 index 8be0e04348..0000000000 --- a/dss-application/pom.xml +++ /dev/null @@ -1,86 +0,0 @@ - - - - - 4.0.0 - - dss - com.webank.wedatasphere.dss - 0.9.1 - - dss-application - - - - junit - junit - 4.12 - test - - - - com.webank.wedatasphere.linkis - linkis-module - ${linkis.version} - provided - - - - com.webank.wedatasphere.dss - dss-appjoint-loader - ${dss.version} - - - org.apache.htrace - htrace-core - 3.2.0-incubating - compile - - - - - - - - - - org.apache.maven.plugins - maven-deploy-plugin - - - - net.alchim31.maven - scala-maven-plugin - - - org.apache.maven.plugins - maven-jar-plugin - - - - - src/main/java - - **/*.xml - - - - - \ No newline at end of file diff --git a/dss-application/src/main/java/com/webank/wedatasphere/dss/application/conf/ApplicationConf.java b/dss-application/src/main/java/com/webank/wedatasphere/dss/application/conf/ApplicationConf.java deleted file mode 100644 index 1194eb4271..0000000000 --- a/dss-application/src/main/java/com/webank/wedatasphere/dss/application/conf/ApplicationConf.java +++ /dev/null @@ -1,52 +0,0 @@ -/* - * Copyright 2019 WeBank - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package com.webank.wedatasphere.dss.application.conf; - - -import com.webank.wedatasphere.linkis.common.conf.CommonVars; - -/** - * Created by chaogefeng on 2019/10/10. - */ -public class ApplicationConf { - - public static final CommonVars FAQ = CommonVars.apply("wds.linkis.application.dws.params",""); - - public static final String SUPER_USER_NAME = CommonVars.apply("wds.linkis.super.user.name","").getValue(); - public static final String WORKSPACE_USER_ROOT_PATH = CommonVars.apply("wds.linkis.workspace.user.root.path","").getValue(); - public static final String HDFS_USER_ROOT_PATH = CommonVars.apply("wds.linkis.hdfs.user.root.path","").getValue(); - public static final String RESULT_SET_ROOT_PATH = CommonVars.apply("wds.linkis.result.set.root.path","").getValue(); - public static final String WDS_SCHEDULER_PATH = CommonVars.apply("wds.linkis.scheduler.path","").getValue(); - public static final String WDS_USER_PATH = CommonVars.apply("wds.linkis.user.path","hdfs:///user").getValue(); - public static final String DSS_INSTALL_DIR = CommonVars.apply("wds.linkis.dss.install.dir","").getValue(); - public static final String AZKABAN_INSTALL_DIR = CommonVars.apply("wds.linkis.azkaban.install.dir","").getValue(); - - - - - - - - - - - - - - -} diff --git a/dss-application/src/main/java/com/webank/wedatasphere/dss/application/dao/ApplicationMapper.java b/dss-application/src/main/java/com/webank/wedatasphere/dss/application/dao/ApplicationMapper.java deleted file mode 100644 index a77f71b4fc..0000000000 --- a/dss-application/src/main/java/com/webank/wedatasphere/dss/application/dao/ApplicationMapper.java +++ /dev/null @@ -1,36 +0,0 @@ -/* - * Copyright 2019 WeBank - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package com.webank.wedatasphere.dss.application.dao; - - -import com.webank.wedatasphere.dss.application.entity.Application; - -import java.util.List; - -/** - * Created by chaogefeng on 2019/10/10. - */ -public interface ApplicationMapper { - List listApplicationNames(); - - List listApplications(); - - Application getApplication(String appName); - - Application getApplicationbyNodeType(String nodeType); -} diff --git a/dss-application/src/main/java/com/webank/wedatasphere/dss/application/dao/DSSApplicationUserMapper.java b/dss-application/src/main/java/com/webank/wedatasphere/dss/application/dao/DSSApplicationUserMapper.java deleted file mode 100644 index f2f59e06b1..0000000000 --- a/dss-application/src/main/java/com/webank/wedatasphere/dss/application/dao/DSSApplicationUserMapper.java +++ /dev/null @@ -1,31 +0,0 @@ -/* - * Copyright 2019 WeBank - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package com.webank.wedatasphere.dss.application.dao; - -import com.webank.wedatasphere.dss.application.entity.DSSUser; - -/** - * Created by chaogefeng on 2019/10/11. - */ -public interface DSSApplicationUserMapper { - DSSUser getUserByName(String username); - - void registerDssUser(DSSUser userDb); - - void updateUserFirstLogin(Long userId); -} diff --git a/dss-application/src/main/java/com/webank/wedatasphere/dss/application/dao/LinkisUserMapper.java b/dss-application/src/main/java/com/webank/wedatasphere/dss/application/dao/LinkisUserMapper.java deleted file mode 100644 index b6d8a10487..0000000000 --- a/dss-application/src/main/java/com/webank/wedatasphere/dss/application/dao/LinkisUserMapper.java +++ /dev/null @@ -1,12 +0,0 @@ -package com.webank.wedatasphere.dss.application.dao; - -import com.webank.wedatasphere.dss.application.entity.LinkisUser; - -/** - * Created by chaogefeng on 2019/11/29. - */ -public interface LinkisUserMapper { - LinkisUser getUserByName(String username); - - void registerLinkisUser(LinkisUser userDb); -} diff --git a/dss-application/src/main/java/com/webank/wedatasphere/dss/application/dao/impl/ApplicationMapper.xml b/dss-application/src/main/java/com/webank/wedatasphere/dss/application/dao/impl/ApplicationMapper.xml deleted file mode 100644 index 604cf39bfa..0000000000 --- a/dss-application/src/main/java/com/webank/wedatasphere/dss/application/dao/impl/ApplicationMapper.xml +++ /dev/null @@ -1,46 +0,0 @@ - - - - - - - - - - - - - - - - - \ No newline at end of file diff --git a/dss-application/src/main/java/com/webank/wedatasphere/dss/application/dao/impl/DSSApplicationUserMapper.xml b/dss-application/src/main/java/com/webank/wedatasphere/dss/application/dao/impl/DSSApplicationUserMapper.xml deleted file mode 100644 index 040cb3eb02..0000000000 --- a/dss-application/src/main/java/com/webank/wedatasphere/dss/application/dao/impl/DSSApplicationUserMapper.xml +++ /dev/null @@ -1,41 +0,0 @@ - - - - - - - - - - id,`username`,`name`,`is_first_login` - - - - - - INSERT INTO dss_user() - VALUES (#{id},#{username},#{name},#{isFirstLogin}) - - - - UPDATE FROM dss_user SET is_first_login = FALSE WHERE id = #{userId} - - - \ No newline at end of file diff --git a/dss-application/src/main/java/com/webank/wedatasphere/dss/application/dao/impl/LinkisUserMapper.xml b/dss-application/src/main/java/com/webank/wedatasphere/dss/application/dao/impl/LinkisUserMapper.xml deleted file mode 100644 index be55ca1368..0000000000 --- a/dss-application/src/main/java/com/webank/wedatasphere/dss/application/dao/impl/LinkisUserMapper.xml +++ /dev/null @@ -1,34 +0,0 @@ - - - - - - - - - - - - INSERT INTO linkis_user VALUES (#{id},#{email},#{userName},#{password},#{admin},#{active},#{name},#{description}, - #{department},#{avatar},#{createTime},#{createBy},#{updateTime},#{updateBy},#{isFirstLogin}) - - - - \ No newline at end of file diff --git a/dss-application/src/main/java/com/webank/wedatasphere/dss/application/entity/Application.java b/dss-application/src/main/java/com/webank/wedatasphere/dss/application/entity/Application.java deleted file mode 100644 index 80654846a5..0000000000 --- a/dss-application/src/main/java/com/webank/wedatasphere/dss/application/entity/Application.java +++ /dev/null @@ -1,135 +0,0 @@ -/* - * Copyright 2019 WeBank - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package com.webank.wedatasphere.dss.application.entity; - -/** - * Created by chaogefeng on 2019/10/10. - * “一个应用”的抽象，比如等等 - */ -public class Application { - private Integer id; - private String name; - private String url; - private Boolean isUserNeedInit; - private Integer level; - private String userInitUrl; - private Boolean existsProjectService;//有些应用可能没有project这么一个概念 - private String projectUrl; - private String enhanceJson; - private Boolean ifIframe; - private String homepageUrl; - private String redirectUrl; - - public Integer getId() { - return id; - } - - public void setId(Integer id) { - this.id = id; - } - - public String getName() { - return name; - } - - public void setName(String name) { - this.name = name; - } - - public String getUrl() { - return url; - } - - public void setUrl(String url) { - this.url = url; - } - - public Boolean getUserNeedInit() { - return isUserNeedInit; - } - - public void setUserNeedInit(Boolean userNeedInit) { - isUserNeedInit = userNeedInit; - } - - public Integer getLevel() { - return level; - } - - public void setLevel(Integer level) { - this.level = level; - } - - public String getUserInitUrl() { - return userInitUrl; - } - - public void setUserInitUrl(String userInitUrl) { - this.userInitUrl = userInitUrl; - } - - public Boolean getExistsProjectService() { - return existsProjectService; - } - - public void setExistsProjectService(Boolean existsProjectService) { - this.existsProjectService = existsProjectService; - } - - public String getProjectUrl() { - return projectUrl; - } - - public void setProjectUrl(String projectUrl) { - this.projectUrl = projectUrl; - } - - public String getEnhanceJson() { - return enhanceJson; - } - - public void setEnhanceJson(String enhanceJson) { - this.enhanceJson = enhanceJson; - } - - public Boolean getIfIframe() { - return ifIframe; - } - - public String getHomepageUrl() { - return homepageUrl; - } - - public void setHomepageUrl(String homepageUrl) { - this.homepageUrl = homepageUrl; - } - - public String getRedirectUrl() { - return redirectUrl; - } - - public void setRedirectUrl(String redirectUrl) { - this.redirectUrl = redirectUrl; - } - - public void setIfIframe(Boolean ifIframe) { - this.ifIframe = ifIframe; - } - - -} diff --git a/dss-application/src/main/java/com/webank/wedatasphere/dss/application/entity/DSSUser.java b/dss-application/src/main/java/com/webank/wedatasphere/dss/application/entity/DSSUser.java deleted file mode 100644 index 73dd36aca0..0000000000 --- a/dss-application/src/main/java/com/webank/wedatasphere/dss/application/entity/DSSUser.java +++ /dev/null @@ -1,68 +0,0 @@ -/* - * Copyright 2019 WeBank - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package com.webank.wedatasphere.dss.application.entity; - -/** - * Created by chaogefeng on 2019/10/11. - */ -public class DSSUser { - private Long id; - private String username; - private String name; - private Boolean isFirstLogin; - private boolean isSuperUser = false; - - public boolean getIsSuperUser() { - return isSuperUser; - } - public void setIsSuperUser(boolean isSuperUser) { - this.isSuperUser = isSuperUser; - } - - public Long getId() { - return id; - } - - public void setId(Long id) { - this.id = id; - } - - public String getUsername() { - return username; - } - - public void setUsername(String username) { - this.username = username; - } - - public String getName() { - return name; - } - - public void setName(String name) { - this.name = name; - } - - public Boolean getFirstLogin() { - return isFirstLogin; - } - - public void setFirstLogin(Boolean firstLogin) { - isFirstLogin = firstLogin; - } -} diff --git a/dss-application/src/main/java/com/webank/wedatasphere/dss/application/entity/DSSUserVO.java b/dss-application/src/main/java/com/webank/wedatasphere/dss/application/entity/DSSUserVO.java deleted file mode 100644 index 7c229c530b..0000000000 --- a/dss-application/src/main/java/com/webank/wedatasphere/dss/application/entity/DSSUserVO.java +++ /dev/null @@ -1,35 +0,0 @@ -/* - * Copyright 2019 WeBank - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package com.webank.wedatasphere.dss.application.entity; - - - -/** - * Created by chaogefeng on 2019/10/11. - */ -public class DSSUserVO { - private DSSUser basic; - - public DSSUser getBasic() { - return basic; - } - - public void setBasic(DSSUser basic) { - this.basic = basic; - } -} diff --git a/dss-application/src/main/java/com/webank/wedatasphere/dss/application/entity/LinkisUser.java b/dss-application/src/main/java/com/webank/wedatasphere/dss/application/entity/LinkisUser.java deleted file mode 100644 index 240f38ce2f..0000000000 --- a/dss-application/src/main/java/com/webank/wedatasphere/dss/application/entity/LinkisUser.java +++ /dev/null @@ -1,144 +0,0 @@ -package com.webank.wedatasphere.dss.application.entity; - -import java.util.Date; - -/** - * Created by chaogefeng on 2019/11/29. - */ -public class LinkisUser { - private Long id; - private String email; - private String userName; - private String password; - private Boolean admin; - private Boolean active; - private String name; - private String description; - private String department; - private String avatar; - private Date createTime; - private Long createBy; - private Date updateTime; - private Long updateBy; - private Boolean isFirstLogin; - - public Long getId() { - return id; - } - - public void setId(Long id) { - this.id = id; - } - - public String getEmail() { - return email; - } - - public void setEmail(String email) { - this.email = email; - } - - public String getUserName() { - return userName; - } - - public void setUserName(String userName) { - this.userName = userName; - } - - public String getPassword() { - return password; - } - - public void setPassword(String password) { - this.password = password; - } - - public Boolean getAdmin() { - return admin; - } - - public void setAdmin(Boolean admin) { - this.admin = admin; - } - - public Boolean getActive() { - return active; - } - - public void setActive(Boolean active) { - this.active = active; - } - - public String getName() { - return name; - } - - public void setName(String name) { - this.name = name; - } - - public String getDescription() { - return description; - } - - public void setDescription(String description) { - this.description = description; - } - - public String getDepartment() { - return department; - } - - public void setDepartment(String department) { - this.department = department; - } - - public String getAvatar() { - return avatar; - } - - public void setAvatar(String avatar) { - this.avatar = avatar; - } - - public Date getCreateTime() { - return createTime; - } - - public void setCreateTime(Date createTime) { - this.createTime = createTime; - } - - public Long getCreateBy() { - return createBy; - } - - public void setCreateBy(Long createBy) { - this.createBy = createBy; - } - - public Date getUpdateTime() { - return updateTime; - } - - public void setUpdateTime(Date updateTime) { - this.updateTime = updateTime; - } - - public Long getUpdateBy() { - return updateBy; - } - - public void setUpdateBy(Long updateBy) { - this.updateBy = updateBy; - } - - public Boolean getFirstLogin() { - return isFirstLogin; - } - - public void setFirstLogin(Boolean firstLogin) { - isFirstLogin = firstLogin; - } -} diff --git a/dss-application/src/main/java/com/webank/wedatasphere/dss/application/entity/UserInitResult.java b/dss-application/src/main/java/com/webank/wedatasphere/dss/application/entity/UserInitResult.java deleted file mode 100644 index 87660e7966..0000000000 --- a/dss-application/src/main/java/com/webank/wedatasphere/dss/application/entity/UserInitResult.java +++ /dev/null @@ -1,69 +0,0 @@ -/* - * Copyright 2019 WeBank - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package com.webank.wedatasphere.dss.application.entity; - -/** - * Created by chaogefeng on 2019/10/10. - */ -public class UserInitResult { - private Long id; - private Integer applicationId; - private String result; - private Long userId; - private Boolean isInitSuccess; - - public Long getId() { - return id; - } - - public void setId(Long id) { - this.id = id; - } - - public Integer getApplicationId() { - return applicationId; - } - - public void setApplicationId(Integer applicationId) { - this.applicationId = applicationId; - } - - public String getResult() { - return result; - } - - public void setResult(String result) { - this.result = result; - } - - public Long getUserId() { - return userId; - } - - public void setUserId(Long userId) { - this.userId = userId; - } - - public Boolean getInitSuccess() { - return isInitSuccess; - } - - public void setInitSuccess(Boolean initSuccess) { - isInitSuccess = initSuccess; - } -} diff --git a/dss-application/src/main/java/com/webank/wedatasphere/dss/application/entity/WorkSpacePath.java b/dss-application/src/main/java/com/webank/wedatasphere/dss/application/entity/WorkSpacePath.java deleted file mode 100644 index 4f2b258915..0000000000 --- a/dss-application/src/main/java/com/webank/wedatasphere/dss/application/entity/WorkSpacePath.java +++ /dev/null @@ -1,59 +0,0 @@ -package com.webank.wedatasphere.dss.application.entity; - -public class WorkSpacePath { - private String workspaceRootPath; - private String hdfsRootPath; - private String resultRootPath; - private String schedulerPath; - private String userPath; - - public String getUserPath() { - return userPath; - } - - public void setUserPath(String userPath) { - this.userPath = userPath; - } - - - - public String getWorkspaceRootPath() { - return workspaceRootPath; - } - - public void setWorkspaceRootPath(String workspaceRootPath) { - this.workspaceRootPath = workspaceRootPath; - } - - public String getHdfsRootPath() { - return hdfsRootPath; - } - - public void setHdfsRootPath(String hdfsRootPath) { - this.hdfsRootPath = hdfsRootPath; - } - - public String getResultRootPath() { - return resultRootPath; - } - - public void setResultRootPath(String resultRootPath) { - this.resultRootPath = resultRootPath; - } - - public String getSchedulerPath() { - return schedulerPath; - } - - public void setSchedulerPath(String schedulerPath) { - this.schedulerPath = schedulerPath; - } - - - - - - - - -} diff --git a/dss-application/src/main/java/com/webank/wedatasphere/dss/application/handler/ApplicationHandlerChain.java b/dss-application/src/main/java/com/webank/wedatasphere/dss/application/handler/ApplicationHandlerChain.java deleted file mode 100644 index eff4b9518c..0000000000 --- a/dss-application/src/main/java/com/webank/wedatasphere/dss/application/handler/ApplicationHandlerChain.java +++ /dev/null @@ -1,46 +0,0 @@ -/* - * Copyright 2019 WeBank - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package com.webank.wedatasphere.dss.application.handler; - -import com.webank.wedatasphere.dss.application.entity.DSSUser; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.stereotype.Component; - -import java.util.Arrays; - -/** - * Created by chaogefeng on 2019/10/14. - */ -@Component -public class ApplicationHandlerChain { - - private Logger logger = LoggerFactory.getLogger(this.getClass()); - - @Autowired - private Handler[] handlers; - - public void handle(String user){ - DSSUser dataworkisUser = new DSSUser(); - dataworkisUser.setUsername(user); - dataworkisUser.setName(user); - Arrays.stream(handlers).sorted((h1,h2)-> h1.getOrder() - h2.getOrder()).forEach(handler -> handler.handle(dataworkisUser)); - } - -} diff --git a/dss-application/src/main/java/com/webank/wedatasphere/dss/application/handler/Handler.java b/dss-application/src/main/java/com/webank/wedatasphere/dss/application/handler/Handler.java deleted file mode 100644 index eb8f262049..0000000000 --- a/dss-application/src/main/java/com/webank/wedatasphere/dss/application/handler/Handler.java +++ /dev/null @@ -1,27 +0,0 @@ -/* - * Copyright 2019 WeBank - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package com.webank.wedatasphere.dss.application.handler; - -import com.webank.wedatasphere.dss.application.entity.DSSUser; - -/** - * Created by chaogefeng on 2019/10/15. - */ -public interface Handler extends Order { - void handle(DSSUser user); -} diff --git a/dss-application/src/main/java/com/webank/wedatasphere/dss/application/handler/LinkisUserFirstLoginHandler.java b/dss-application/src/main/java/com/webank/wedatasphere/dss/application/handler/LinkisUserFirstLoginHandler.java deleted file mode 100644 index 1a83caa72c..0000000000 --- a/dss-application/src/main/java/com/webank/wedatasphere/dss/application/handler/LinkisUserFirstLoginHandler.java +++ /dev/null @@ -1,63 +0,0 @@ -/* - * Copyright 2019 WeBank - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ -package com.webank.wedatasphere.dss.application.handler; - -import com.webank.wedatasphere.dss.application.entity.DSSUser; -import com.webank.wedatasphere.dss.application.entity.LinkisUser; -import com.webank.wedatasphere.dss.application.service.LinkisUserService; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.stereotype.Component; - -import java.util.Date; - -/** - * Created by chaogefeng on 2019/11/29. - */ -@Component -public class LinkisUserFirstLoginHandler implements Handler{ - - private Logger logger = LoggerFactory.getLogger(this.getClass()); - - @Autowired - private LinkisUserService linkisUserService; - - @Override - public int getOrder() { - return -1; - } - - @Override - public void handle(DSSUser user) { - logger.info("LinkisUserFirstLoginHandler:"); - synchronized (user.getUsername().intern()){ - LinkisUser userDb = linkisUserService.getUserByName(user.getUsername()); - if(userDb == null){ - logger.info("User first enter linkis, insert table linkis_user and dss_user"); - userDb = new LinkisUser(); - userDb.setUserName(user.getUsername()); - userDb.setName(user.getName()); - userDb.setFirstLogin(true); - userDb.setCreateTime(new Date()); - userDb.setUpdateTime(new Date()); - linkisUserService.registerLinkisAndDssUser(userDb); - } - user.setId(userDb.getId()); - } - } -} diff --git a/dss-application/src/main/java/com/webank/wedatasphere/dss/application/handler/Order.java b/dss-application/src/main/java/com/webank/wedatasphere/dss/application/handler/Order.java deleted file mode 100644 index 30fb482c9a..0000000000 --- a/dss-application/src/main/java/com/webank/wedatasphere/dss/application/handler/Order.java +++ /dev/null @@ -1,27 +0,0 @@ -/* - * Copyright 2019 WeBank - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package com.webank.wedatasphere.dss.application.handler; - -/** - * Created by chaogefeng on 2019/9/30. - */ -public interface Order { - - int getOrder(); - -} diff --git a/dss-application/src/main/java/com/webank/wedatasphere/dss/application/handler/ProjectUserInitHandler.java b/dss-application/src/main/java/com/webank/wedatasphere/dss/application/handler/ProjectUserInitHandler.java deleted file mode 100644 index 38f06b10ac..0000000000 --- a/dss-application/src/main/java/com/webank/wedatasphere/dss/application/handler/ProjectUserInitHandler.java +++ /dev/null @@ -1,43 +0,0 @@ -/* - * Copyright 2019 WeBank - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package com.webank.wedatasphere.dss.application.handler; - - -import com.webank.wedatasphere.dss.application.entity.DSSUser; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.springframework.stereotype.Component; - -/** - * Created by chaogefeng on 2019/10/15. - */ -@Component -public class ProjectUserInitHandler implements Handler { - - private Logger logger = LoggerFactory.getLogger(this.getClass()); - - @Override - public void handle(DSSUser user) { - logger.info("ProjectUserInitHandler:"); - } - - @Override - public int getOrder() { - return 1; - } -} diff --git a/dss-application/src/main/java/com/webank/wedatasphere/dss/application/handler/UserFirstLoginHandler.java b/dss-application/src/main/java/com/webank/wedatasphere/dss/application/handler/UserFirstLoginHandler.java deleted file mode 100644 index f902c5ba3d..0000000000 --- a/dss-application/src/main/java/com/webank/wedatasphere/dss/application/handler/UserFirstLoginHandler.java +++ /dev/null @@ -1,62 +0,0 @@ -/* - * Copyright 2019 WeBank - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package com.webank.wedatasphere.dss.application.handler; - -import com.webank.wedatasphere.dss.application.entity.DSSUser; -import com.webank.wedatasphere.dss.application.service.DSSApplicationUserService; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.stereotype.Component; - -/** - * Created by chaogefeng on 2019/10/15. - */ -@Component -public class UserFirstLoginHandler implements Handler { - - private Logger logger = LoggerFactory.getLogger(this.getClass()); - - @Autowired - private DSSApplicationUserService dssApplicationUserService; - - @Override - public int getOrder() { - return 0; - } - - @Override - public void handle(DSSUser user) { - logger.info("UserFirstLoginHandler:"); - synchronized (user.getUsername().intern()){ - DSSUser userDb = dssApplicationUserService.getUserByName(user.getUsername()); - if(userDb == null){ - logger.info("User first enter dss, insert table dss_user"); - userDb = new DSSUser(); - userDb.setUsername(user.getUsername()); - userDb.setName(user.getName()); - userDb.setFirstLogin(true); - userDb.setId(user.getId()); - dssApplicationUserService.registerDssUser(userDb); - } - // TODO: 2019/11/29 update firstLogin - user = userDb; - } - - } -} diff --git a/dss-application/src/main/java/com/webank/wedatasphere/dss/application/restful/ApplicationRestfulApi.java b/dss-application/src/main/java/com/webank/wedatasphere/dss/application/restful/ApplicationRestfulApi.java deleted file mode 100644 index 10552b25f2..0000000000 --- a/dss-application/src/main/java/com/webank/wedatasphere/dss/application/restful/ApplicationRestfulApi.java +++ /dev/null @@ -1,104 +0,0 @@ -/* - * Copyright 2019 WeBank - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package com.webank.wedatasphere.dss.application.restful; - -import com.webank.wedatasphere.dss.application.conf.ApplicationConf; -import com.webank.wedatasphere.dss.application.entity.Application; -import com.webank.wedatasphere.dss.application.entity.DSSUser; -import com.webank.wedatasphere.dss.application.entity.DSSUserVO; -import com.webank.wedatasphere.dss.application.entity.WorkSpacePath; -import com.webank.wedatasphere.dss.application.handler.ApplicationHandlerChain; -import com.webank.wedatasphere.dss.application.service.ApplicationService; -import com.webank.wedatasphere.dss.application.service.DSSApplicationUserService; -import com.webank.wedatasphere.dss.application.util.ApplicationUtils; -import com.webank.wedatasphere.linkis.server.Message; -import com.webank.wedatasphere.linkis.server.security.SecurityFilter; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.stereotype.Component; - -import javax.servlet.http.HttpServletRequest; -import javax.ws.rs.Consumes; -import javax.ws.rs.GET; -import javax.ws.rs.Path; -import javax.ws.rs.Produces; -import javax.ws.rs.core.Context; -import javax.ws.rs.core.MediaType; -import javax.ws.rs.core.Response; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; - -/** - * Created by chaogefeng on 2019/10/10. - */ -@Component -@Path("/dss") -@Produces(MediaType.APPLICATION_JSON) -@Consumes(MediaType.APPLICATION_JSON) -public class ApplicationRestfulApi { - - @Autowired - private ApplicationService applicationService; - @Autowired - private DSSApplicationUserService dataworkisUserService; - @Autowired - private ApplicationHandlerChain applicationHandlerChain; - - @GET - @Path("getBaseInfo") - public Response getBaseInfo(@Context HttpServletRequest req){ - String username = SecurityFilter.getLoginUsername(req); - applicationHandlerChain.handle(username); - List applicationList = applicationService.listApplications(); - for (Application application : applicationList) { - String redirectUrl = application.getRedirectUrl(); - if(redirectUrl != null) { - application.setHomepageUrl(ApplicationUtils.redirectUrlFormat(redirectUrl,application.getHomepageUrl())); - application.setProjectUrl(ApplicationUtils.redirectUrlFormat(redirectUrl,application.getProjectUrl())); - } - } - DSSUser dssUser = dataworkisUserService.getUserByName(username); - DSSUserVO dataworkisUserVO = new DSSUserVO(); - String superUserName = ApplicationConf.SUPER_USER_NAME; - if(username.equals(superUserName)){ - dssUser.setIsSuperUser(true); - }else{ - dssUser.setIsSuperUser(false); - } - - dataworkisUserVO.setBasic(dssUser); - return Message.messageToResponse(Message.ok().data("applications",applicationList).data("userInfo",dataworkisUserVO)); - } - - - @GET - @Path("paths") - public Response getWorkSpace(@Context HttpServletRequest req) throws Exception { - WorkSpacePath workSpacePath = new WorkSpacePath(); - workSpacePath.setHdfsRootPath(ApplicationConf.HDFS_USER_ROOT_PATH); - workSpacePath.setResultRootPath(ApplicationConf.RESULT_SET_ROOT_PATH); - workSpacePath.setSchedulerPath(ApplicationConf.WDS_SCHEDULER_PATH); - workSpacePath.setWorkspaceRootPath(ApplicationConf.WORKSPACE_USER_ROOT_PATH); - workSpacePath.setUserPath(ApplicationConf.WDS_USER_PATH); - ArrayList> responses = ApplicationUtils.convertToMap(workSpacePath); - return Message.messageToResponse(Message.ok().data("paths",responses) - .data("dssInstallDir", ApplicationConf.DSS_INSTALL_DIR) - .data("azkakanDir", ApplicationConf.AZKABAN_INSTALL_DIR)); - - } -} diff --git a/dss-application/src/main/java/com/webank/wedatasphere/dss/application/service/ApplicationService.java b/dss-application/src/main/java/com/webank/wedatasphere/dss/application/service/ApplicationService.java deleted file mode 100644 index fa571e36f0..0000000000 --- a/dss-application/src/main/java/com/webank/wedatasphere/dss/application/service/ApplicationService.java +++ /dev/null @@ -1,43 +0,0 @@ -/* - * Copyright 2019 WeBank - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package com.webank.wedatasphere.dss.application.service; - - -import com.webank.wedatasphere.dss.appjoint.AppJoint; -import com.webank.wedatasphere.dss.appjoint.exception.AppJointErrorException; -import com.webank.wedatasphere.dss.application.entity.Application; - -import java.util.List; - -/** - * Created by chaogefeng on 2019/10/10. - */ -public interface ApplicationService { - - List listApplicationNames(); - - List listApplications(); - - Application getApplication(String appName); - - AppJoint getAppjoint(String nodeType) throws AppJointErrorException; - - List listAppjoint() throws AppJointErrorException; - - Application getApplicationbyNodeType(String nodeType); -} diff --git a/dss-application/src/main/java/com/webank/wedatasphere/dss/application/service/DSSApplicationUserService.java b/dss-application/src/main/java/com/webank/wedatasphere/dss/application/service/DSSApplicationUserService.java deleted file mode 100644 index a798dc4357..0000000000 --- a/dss-application/src/main/java/com/webank/wedatasphere/dss/application/service/DSSApplicationUserService.java +++ /dev/null @@ -1,32 +0,0 @@ -/* - * Copyright 2019 WeBank - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package com.webank.wedatasphere.dss.application.service; - -import com.webank.wedatasphere.dss.application.entity.DSSUser; - -/** - * Created by chaogefeng on 2019/10/11. - */ -public interface DSSApplicationUserService { - - DSSUser getUserByName(String username); - - void registerDssUser(DSSUser userDb); - - void updateUserFirstLogin(Long id); -} diff --git a/dss-application/src/main/java/com/webank/wedatasphere/dss/application/service/LinkisUserService.java b/dss-application/src/main/java/com/webank/wedatasphere/dss/application/service/LinkisUserService.java deleted file mode 100644 index 464de78497..0000000000 --- a/dss-application/src/main/java/com/webank/wedatasphere/dss/application/service/LinkisUserService.java +++ /dev/null @@ -1,31 +0,0 @@ -/* - * Copyright 2019 WeBank - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ -package com.webank.wedatasphere.dss.application.service; - -import com.webank.wedatasphere.dss.application.entity.LinkisUser; - -/** - * Created by chaogefeng on 2019/11/29. - */ -public interface LinkisUserService { - - LinkisUser getUserByName(String username); - - void registerLinkisAndDssUser(LinkisUser userDb); - - void registerDSSUser(LinkisUser userDb); -} diff --git a/dss-application/src/main/java/com/webank/wedatasphere/dss/application/service/impl/ApplicationServiceImpl.java b/dss-application/src/main/java/com/webank/wedatasphere/dss/application/service/impl/ApplicationServiceImpl.java deleted file mode 100644 index 271acac867..0000000000 --- a/dss-application/src/main/java/com/webank/wedatasphere/dss/application/service/impl/ApplicationServiceImpl.java +++ /dev/null @@ -1,110 +0,0 @@ -/* - * Copyright 2019 WeBank - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package com.webank.wedatasphere.dss.application.service.impl; - -import com.webank.wedatasphere.dss.appjoint.AppJoint; -import com.webank.wedatasphere.dss.appjoint.exception.AppJointErrorException; -import com.webank.wedatasphere.dss.appjoint.loader.AppJointLoader; -import com.webank.wedatasphere.dss.application.dao.ApplicationMapper; -import com.webank.wedatasphere.dss.application.entity.Application; -import com.webank.wedatasphere.dss.application.service.ApplicationService; -import com.webank.wedatasphere.dss.common.exception.DSSRuntimeException; -import com.webank.wedatasphere.dss.common.utils.DSSExceptionUtils; -import org.apache.commons.lang.StringUtils; -import org.apache.htrace.fasterxml.jackson.databind.JavaType; -import org.apache.htrace.fasterxml.jackson.databind.ObjectMapper; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.stereotype.Service; - -import java.util.List; -import java.util.Map; -import java.util.stream.Collectors; - -/** - * Created by chaogefeng on 2019/10/10. - */ -@Service -public class ApplicationServiceImpl implements ApplicationService { - - private final Logger logger = LoggerFactory.getLogger(this.getClass()); - - @Autowired - private ApplicationMapper applicationMapper; - - @Override - public List listApplicationNames() { - return applicationMapper.listApplicationNames(); - } - - @Override - public List listApplications() { - return applicationMapper.listApplications(); - } - - @Override - public Application getApplication(String appName) { - return applicationMapper.getApplication(appName); - } - - @Override - public AppJoint getAppjoint(String nodeType) throws AppJointErrorException { - Application application; - if(nodeType.equals("schedulis")){ - application = getApplication(nodeType); - }else { - application = getApplicationbyNodeType(nodeType); - } - AppJoint appJoint = null; - try { - appJoint = loadAppjoint(application); - } catch (Exception e) { - logger.error("load appJoint failed:",e); - throw new AppJointErrorException(90000,"load appJoint failed:"+e.getMessage()); - } - return appJoint; - } - - @Override - public List listAppjoint()throws AppJointErrorException{ - List applications = listApplications(); - List appJoints = null; - try { - appJoints = applications.stream().filter(Application::getExistsProjectService).map(DSSExceptionUtils.map(this::loadAppjoint)).collect(Collectors.toList()); - }catch (DSSRuntimeException e){ - logger.error("load appJoint failed:",e); - throw new AppJointErrorException(90000,"load appJoint failed:"+e.getMessage()); - } - return appJoints; - } - - @Override - public Application getApplicationbyNodeType(String nodeType) { - return applicationMapper.getApplicationbyNodeType(nodeType); - } - - private AppJoint loadAppjoint(Application application) throws Exception { - String enhanceJson = application.getEnhanceJson(); - if(StringUtils.isBlank(enhanceJson)) enhanceJson = "{}"; - ObjectMapper om = new ObjectMapper(); - JavaType javaType = om.getTypeFactory().constructParametricType(Map.class,String.class,Object.class); - Map parmas = om.readValue(enhanceJson, javaType); - return AppJointLoader.getAppJointLoader().getAppJoint(application.getUrl(), application.getName(), parmas); - } -} diff --git a/dss-application/src/main/java/com/webank/wedatasphere/dss/application/service/impl/DSSApplicationUserServiceImpl.java b/dss-application/src/main/java/com/webank/wedatasphere/dss/application/service/impl/DSSApplicationUserServiceImpl.java deleted file mode 100644 index c8fae7a631..0000000000 --- a/dss-application/src/main/java/com/webank/wedatasphere/dss/application/service/impl/DSSApplicationUserServiceImpl.java +++ /dev/null @@ -1,49 +0,0 @@ -/* - * Copyright 2019 WeBank - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package com.webank.wedatasphere.dss.application.service.impl; - -import com.webank.wedatasphere.dss.application.dao.DSSApplicationUserMapper; -import com.webank.wedatasphere.dss.application.entity.DSSUser; -import com.webank.wedatasphere.dss.application.service.DSSApplicationUserService; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.stereotype.Service; - -/** - * Created by chaogefeng on 2019/10/11. - */ -@Service -public class DSSApplicationUserServiceImpl implements DSSApplicationUserService { - - @Autowired - private DSSApplicationUserMapper dssApplicationUserMapper; - - @Override - public DSSUser getUserByName(String username) { - return dssApplicationUserMapper.getUserByName(username); - } - - @Override - public void registerDssUser(DSSUser userDb) { - dssApplicationUserMapper.registerDssUser( userDb); - } - - @Override - public void updateUserFirstLogin(Long id) { - dssApplicationUserMapper.updateUserFirstLogin(id); - } -} diff --git a/dss-application/src/main/java/com/webank/wedatasphere/dss/application/service/impl/LinkisUserServiceImpl.java b/dss-application/src/main/java/com/webank/wedatasphere/dss/application/service/impl/LinkisUserServiceImpl.java deleted file mode 100644 index f40f286ecc..0000000000 --- a/dss-application/src/main/java/com/webank/wedatasphere/dss/application/service/impl/LinkisUserServiceImpl.java +++ /dev/null @@ -1,60 +0,0 @@ -/* - * Copyright 2019 WeBank - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ -package com.webank.wedatasphere.dss.application.service.impl; - -import com.webank.wedatasphere.dss.application.dao.DSSApplicationUserMapper; -import com.webank.wedatasphere.dss.application.dao.LinkisUserMapper; -import com.webank.wedatasphere.dss.application.entity.DSSUser; -import com.webank.wedatasphere.dss.application.entity.LinkisUser; -import com.webank.wedatasphere.dss.application.service.LinkisUserService; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.stereotype.Component; - -/** - * Created by chaogefeng on 2019/11/29. - */ -@Component -public class LinkisUserServiceImpl implements LinkisUserService { - - @Autowired - private LinkisUserMapper linkisUserMapper; - @Autowired - private DSSApplicationUserMapper dssApplicationUserMapper; - - @Override - public LinkisUser getUserByName(String username) { - return linkisUserMapper.getUserByName(username); - } - - - @Override - public void registerLinkisAndDssUser(LinkisUser userDb) { - // TODO: 2019/11/29 @transactional - linkisUserMapper.registerLinkisUser(userDb); - registerDSSUser(userDb); - } - - @Override - public void registerDSSUser(LinkisUser userDb) { - DSSUser dssUser = new DSSUser(); - dssUser.setId(userDb.getId()); - dssUser.setName(userDb.getName()); - dssUser.setUsername(userDb.getUserName()); - dssUser.setFirstLogin(userDb.getFirstLogin()); - dssApplicationUserMapper.registerDssUser(dssUser); - } -} diff --git a/dss-application/src/main/java/com/webank/wedatasphere/dss/application/util/ApplicationUtils.java b/dss-application/src/main/java/com/webank/wedatasphere/dss/application/util/ApplicationUtils.java deleted file mode 100644 index af65bd3227..0000000000 --- a/dss-application/src/main/java/com/webank/wedatasphere/dss/application/util/ApplicationUtils.java +++ /dev/null @@ -1,85 +0,0 @@ -/* - * Copyright 2019 WeBank - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package com.webank.wedatasphere.dss.application.util; - -import com.webank.wedatasphere.dss.application.entity.WorkSpacePath; -import com.webank.wedatasphere.dss.common.exception.DSSErrorException; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.UnsupportedEncodingException; -import java.lang.reflect.Field; -import java.net.URLEncoder; -import java.util.ArrayList; -import java.util.HashMap; - -/** - * Created by chaogefeng on 2019/11/20. - */ -public class ApplicationUtils { - - private static final Logger LOGGER = LoggerFactory.getLogger(ApplicationUtils.class); - - private static final String REDIRECT_FORMAT = "%s?redirect=%s&dssurl=${dssurl}&cookies=${cookies}"; - - private static String URLEndoder(String str){ - try { - return URLEncoder.encode(str,"utf-8"); - } catch (UnsupportedEncodingException e) { - LOGGER.warn("endoe failed:",e); - return str; - } - } - - public static String redirectUrlFormat(String redirectUrl,String url){ - return String.format(REDIRECT_FORMAT,redirectUrl,URLEndoder(url)); - } - - - public static ArrayList> convertToMap(Object obj) - throws Exception { - - ArrayList> mapList = new ArrayList(); - Field[] fields = obj.getClass().getDeclaredFields(); - for (int i = 0, len = fields.length; i < len; i++) { - String varName = fields[i].getName(); - boolean accessFlag = fields[i].isAccessible(); - fields[i].setAccessible(true); - HashMap map = new HashMap<>(); - Object o = fields[i].get(obj); - if (o != null){ - map.put("key",varName); - map.put("value",o.toString()); - mapList.add(map); - } - - fields[i].setAccessible(accessFlag); - } - - return mapList; - } - - public static void main(String[] args) throws Exception { -// System.out.println(redirectUrlFormat("http://127.0..0.1:8090/qualitis/api/v1/redirect","http://127.0..0.1:8090/#/projects/list?id={projectId}&flow=true")); - - WorkSpacePath workSpacePath = new WorkSpacePath(); - workSpacePath.setWorkspaceRootPath("/"); - System.out.println(convertToMap(workSpacePath)); - } - -} diff --git a/dss-azkaban-scheduler-appjoint/pom.xml b/dss-azkaban-scheduler-appjoint/pom.xml deleted file mode 100644 index 4f67a636b8..0000000000 --- a/dss-azkaban-scheduler-appjoint/pom.xml +++ /dev/null @@ -1,105 +0,0 @@ - - - - - - dss - com.webank.wedatasphere.dss - 0.9.1 - - 4.0.0 - - dss-azkaban-scheduler-appjoint - - - - - - - - - com.webank.wedatasphere.dss - dss-scheduler-appjoint-core - ${dss.version} - provided - true - - - org.apache.httpcomponents - httpmime - 4.5.4 - - - com.webank.wedatasphere.dss - dss-application - ${dss.version} - provided - true - - - - - - - org.apache.maven.plugins - maven-deploy-plugin - - - - net.alchim31.maven - scala-maven-plugin - - - org.apache.maven.plugins - maven-jar-plugin - - - org.apache.maven.plugins - maven-assembly-plugin - 2.3 - false - - - make-assembly - package - - single - - - - src/main/assembly/distribution.xml - - - - - - false - dss-azkaban-appjoint - false - false - - src/main/assembly/distribution.xml - - - - - - - \ No newline at end of file diff --git a/dss-azkaban-scheduler-appjoint/src/main/assembly/distribution.xml b/dss-azkaban-scheduler-appjoint/src/main/assembly/distribution.xml deleted file mode 100644 index 2898561240..0000000000 --- a/dss-azkaban-scheduler-appjoint/src/main/assembly/distribution.xml +++ /dev/null @@ -1,136 +0,0 @@ - - - - dss-azkaban-appjoint - - zip - - true - schedulis - - - - - - lib - true - true - false - true - true - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - ${basedir}/src/main/resources - - appjoint.properties - - 0777 - / - unix - - - - ${basedir}/src/main/resources - - log4j.properties - log4j2.xml - - 0777 - conf - unix - - - - - - diff --git a/dss-azkaban-scheduler-appjoint/src/main/java/com/webank/wedatasphere/dss/appjoint/scheduler/azkaban/AzkabanSchedulerAppJoint.java b/dss-azkaban-scheduler-appjoint/src/main/java/com/webank/wedatasphere/dss/appjoint/scheduler/azkaban/AzkabanSchedulerAppJoint.java deleted file mode 100644 index 8f6e4bb8a9..0000000000 --- a/dss-azkaban-scheduler-appjoint/src/main/java/com/webank/wedatasphere/dss/appjoint/scheduler/azkaban/AzkabanSchedulerAppJoint.java +++ /dev/null @@ -1,85 +0,0 @@ -package com.webank.wedatasphere.dss.appjoint.scheduler.azkaban; - -import com.webank.wedatasphere.dss.appjoint.scheduler.SchedulerAppJoint; -import com.webank.wedatasphere.dss.appjoint.scheduler.azkaban.conf.AzkabanConf; -import com.webank.wedatasphere.dss.appjoint.scheduler.azkaban.hooks.LinkisAzkabanProjectPublishHook; -import com.webank.wedatasphere.dss.appjoint.scheduler.azkaban.parser.AzkabanProjectParser; -import com.webank.wedatasphere.dss.appjoint.scheduler.azkaban.tuning.AzkabanProjectTuning; -import com.webank.wedatasphere.dss.appjoint.scheduler.hooks.ProjectPublishHook; -import com.webank.wedatasphere.dss.appjoint.scheduler.parser.ProjectParser; -import com.webank.wedatasphere.dss.appjoint.scheduler.service.SchedulerProjectService; -import com.webank.wedatasphere.dss.appjoint.scheduler.service.SchedulerSecurityService; -import com.webank.wedatasphere.dss.appjoint.scheduler.azkaban.service.AzkabanProjectService; -import com.webank.wedatasphere.dss.appjoint.scheduler.azkaban.service.AzkabanSecurityService; -import com.webank.wedatasphere.dss.appjoint.scheduler.tuning.ProjectTuning; -import com.webank.wedatasphere.dss.appjoint.service.AppJointUrlImpl; -import com.webank.wedatasphere.dss.application.service.ApplicationService; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.springframework.util.StringUtils; -import java.io.IOException; -import java.util.Map; - -/** - * Created by cooperyang on 2019/9/16. - */ - -public final class AzkabanSchedulerAppJoint extends AppJointUrlImpl implements SchedulerAppJoint { - - private static final Logger LOGGER = LoggerFactory.getLogger(AzkabanSchedulerAppJoint.class); - - private SchedulerSecurityService securityService; - private SchedulerProjectService projectService; - private ApplicationService applicationService; - - @Override - public String getAppJointName() { - return "schedulis"; - } - - - @Override - public void init(String basicUrl, Map params) { - LOGGER.info("read schedulerAppJoint url from db{}",basicUrl); - if(StringUtils.isEmpty(basicUrl)){ - basicUrl = AzkabanConf.AZKABAN_BASE_URL.getValue(); - LOGGER.warn("basic url in db is empty,read it from conf{}",basicUrl); - } - securityService = new AzkabanSecurityService(); - securityService.setBaseUrl(basicUrl); - - projectService = new AzkabanProjectService(); - projectService.setBaseUrl(basicUrl); - } - - @Override - public SchedulerSecurityService getSecurityService() { - return this.securityService; - } - - @Override - public ProjectParser getProjectParser() { - return new AzkabanProjectParser(); - } - - @Override - public ProjectTuning getProjectTuning() { - return new AzkabanProjectTuning(); - } - - @Override - public ProjectPublishHook[] getProjectPublishHooks() { - ProjectPublishHook[] projectPublishHooks = {new LinkisAzkabanProjectPublishHook()}; - return projectPublishHooks; - } - - @Override - public SchedulerProjectService getProjectService() { - return this.projectService; - } - - @Override - public void close() throws IOException { - - } -} diff --git a/dss-azkaban-scheduler-appjoint/src/main/java/com/webank/wedatasphere/dss/appjoint/scheduler/azkaban/conf/AzkabanConf.java b/dss-azkaban-scheduler-appjoint/src/main/java/com/webank/wedatasphere/dss/appjoint/scheduler/azkaban/conf/AzkabanConf.java deleted file mode 100644 index 49ea9948ad..0000000000 --- a/dss-azkaban-scheduler-appjoint/src/main/java/com/webank/wedatasphere/dss/appjoint/scheduler/azkaban/conf/AzkabanConf.java +++ /dev/null @@ -1,14 +0,0 @@ -package com.webank.wedatasphere.dss.appjoint.scheduler.azkaban.conf; - - -import com.webank.wedatasphere.linkis.common.conf.CommonVars; - -/** - * Created by cooperyang on 2019/9/16. - */ -public class AzkabanConf { - - public static final CommonVars AZKABAN_BASE_URL = CommonVars.apply("wds.dss.appjoint.scheduler.azkaban.address", ""); - public static final CommonVars DEFAULT_STORE_PATH = CommonVars.apply("wds.dss.appjoint.scheduler.project.store.dir", "/appcom/tmp/wds/dss"); - public static final CommonVars AZKABAN_LOGIN_PWD = CommonVars.apply("wds.dss.appjoint.scheduler.azkaban.login.passwd", "password"); -} diff --git a/dss-azkaban-scheduler-appjoint/src/main/java/com/webank/wedatasphere/dss/appjoint/scheduler/azkaban/constant/AzkabanConstant.java b/dss-azkaban-scheduler-appjoint/src/main/java/com/webank/wedatasphere/dss/appjoint/scheduler/azkaban/constant/AzkabanConstant.java deleted file mode 100644 index eda5820b59..0000000000 --- a/dss-azkaban-scheduler-appjoint/src/main/java/com/webank/wedatasphere/dss/appjoint/scheduler/azkaban/constant/AzkabanConstant.java +++ /dev/null @@ -1,17 +0,0 @@ -package com.webank.wedatasphere.dss.appjoint.scheduler.azkaban.constant; - -/** - * Created by cooperyang on 2019/9/27. - */ -public class AzkabanConstant { - public final static String LINKIS_FLOW_VARIABLE_KEY = "flow.variable."; - public final static String AZKABAN_JOB_SUFFIX = ".job"; - public final static String AZKABAN_PROPERTIES_SUFFIX = ".properties"; - public final static String LINKIS_JOB_RESOURCES_KEY = "resources="; - public final static String ZAKABAN_DEPENDENCIES_KEY = "dependencies"; - public final static String TOKEN_FILE_NAME = "token.properties"; - public final static String JOB_TYPE = "type"; - public final static String LINKIS_TYPE = "linkistype"; - public final static String JOB_COMMAND = "command"; - -} diff --git a/dss-azkaban-scheduler-appjoint/src/main/java/com/webank/wedatasphere/dss/appjoint/scheduler/azkaban/entity/AzkabanSchedulerFlow.java b/dss-azkaban-scheduler-appjoint/src/main/java/com/webank/wedatasphere/dss/appjoint/scheduler/azkaban/entity/AzkabanSchedulerFlow.java deleted file mode 100644 index a8a6aae1d8..0000000000 --- a/dss-azkaban-scheduler-appjoint/src/main/java/com/webank/wedatasphere/dss/appjoint/scheduler/azkaban/entity/AzkabanSchedulerFlow.java +++ /dev/null @@ -1,17 +0,0 @@ -package com.webank.wedatasphere.dss.appjoint.scheduler.azkaban.entity; - -import com.webank.wedatasphere.dss.appjoint.scheduler.entity.SchedulerFlow; - -public class AzkabanSchedulerFlow extends SchedulerFlow { - - private String storePath; - - public String getStorePath() { - return storePath; - } - - public void setStorePath(String storePath) { - this.storePath = storePath; - } - -} diff --git a/dss-azkaban-scheduler-appjoint/src/main/java/com/webank/wedatasphere/dss/appjoint/scheduler/azkaban/entity/AzkabanSchedulerNode.java b/dss-azkaban-scheduler-appjoint/src/main/java/com/webank/wedatasphere/dss/appjoint/scheduler/azkaban/entity/AzkabanSchedulerNode.java deleted file mode 100644 index d4b3861d07..0000000000 --- a/dss-azkaban-scheduler-appjoint/src/main/java/com/webank/wedatasphere/dss/appjoint/scheduler/azkaban/entity/AzkabanSchedulerNode.java +++ /dev/null @@ -1,22 +0,0 @@ -package com.webank.wedatasphere.dss.appjoint.scheduler.azkaban.entity; - - -import com.webank.wedatasphere.dss.appjoint.scheduler.entity.AbstractSchedulerNode; -import com.webank.wedatasphere.linkis.common.exception.ErrorException; - -/** - * Created by cooperyang on 2019/9/15 - */ -public class AzkabanSchedulerNode extends AbstractSchedulerNode { - - private String storePath; - - public String getStorePath() { - return storePath; - } - - public void setStorePath(String storePath) { - this.storePath = storePath; - } - -} diff --git a/dss-azkaban-scheduler-appjoint/src/main/java/com/webank/wedatasphere/dss/appjoint/scheduler/azkaban/entity/AzkabanSchedulerProject.java b/dss-azkaban-scheduler-appjoint/src/main/java/com/webank/wedatasphere/dss/appjoint/scheduler/azkaban/entity/AzkabanSchedulerProject.java deleted file mode 100644 index 4e28e26554..0000000000 --- a/dss-azkaban-scheduler-appjoint/src/main/java/com/webank/wedatasphere/dss/appjoint/scheduler/azkaban/entity/AzkabanSchedulerProject.java +++ /dev/null @@ -1,21 +0,0 @@ -package com.webank.wedatasphere.dss.appjoint.scheduler.azkaban.entity; - -import com.webank.wedatasphere.dss.appjoint.scheduler.entity.AbstractSchedulerProject; - -/** - * Created by cooperyang on 2019/9/19 - */ -public class AzkabanSchedulerProject extends AbstractSchedulerProject { - - private String storePath; - public static final String DATE_FORMAT = "yyyy-MM-dd"; - - public String getStorePath() { - return storePath; - } - - public void setStorePath(String storePath) { - this.storePath = storePath; - } - -} diff --git a/dss-azkaban-scheduler-appjoint/src/main/java/com/webank/wedatasphere/dss/appjoint/scheduler/azkaban/entity/LinkisAzkabanReadNode.java b/dss-azkaban-scheduler-appjoint/src/main/java/com/webank/wedatasphere/dss/appjoint/scheduler/azkaban/entity/LinkisAzkabanReadNode.java deleted file mode 100644 index ccee05d432..0000000000 --- a/dss-azkaban-scheduler-appjoint/src/main/java/com/webank/wedatasphere/dss/appjoint/scheduler/azkaban/entity/LinkisAzkabanReadNode.java +++ /dev/null @@ -1,37 +0,0 @@ -package com.webank.wedatasphere.dss.appjoint.scheduler.azkaban.entity; - -import com.webank.wedatasphere.dss.appjoint.scheduler.azkaban.linkisjob.LinkisJobConverter; -import com.webank.wedatasphere.dss.appjoint.scheduler.entity.ReadNode; -import com.webank.wedatasphere.dss.appjoint.scheduler.entity.SchedulerNode; - -public class LinkisAzkabanReadNode extends LinkisAzkabanSchedulerNode implements ReadNode { - - private SchedulerNode schedulerNode; - private String[] nodeIds; - - @Override - public String[] getShareNodeIds() { - return this.nodeIds; - } - - @Override - public void setShareNodeIds(String[] nodeIds) { - this.nodeIds = nodeIds; - } - - @Override - public SchedulerNode getSchedulerNode() { - return schedulerNode; - } - - @Override - public void setSchedulerNode(SchedulerNode schedulerNode) { - this.schedulerNode = schedulerNode; - } - - @Override - public String toJobString(LinkisJobConverter converter) { - return converter.conversion(this); - } - -} diff --git a/dss-azkaban-scheduler-appjoint/src/main/java/com/webank/wedatasphere/dss/appjoint/scheduler/azkaban/entity/LinkisAzkabanSchedulerNode.java b/dss-azkaban-scheduler-appjoint/src/main/java/com/webank/wedatasphere/dss/appjoint/scheduler/azkaban/entity/LinkisAzkabanSchedulerNode.java deleted file mode 100644 index 750f4d1e13..0000000000 --- a/dss-azkaban-scheduler-appjoint/src/main/java/com/webank/wedatasphere/dss/appjoint/scheduler/azkaban/entity/LinkisAzkabanSchedulerNode.java +++ /dev/null @@ -1,17 +0,0 @@ -package com.webank.wedatasphere.dss.appjoint.scheduler.azkaban.entity; - -import com.webank.wedatasphere.dss.appjoint.scheduler.azkaban.linkisjob.LinkisJobConverter; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** - * Created by cooperyang on 2019/9/29. - */ -public class LinkisAzkabanSchedulerNode extends AzkabanSchedulerNode { - - private final Logger logger = LoggerFactory.getLogger(this.getClass()); - - public String toJobString(LinkisJobConverter converter) { - return converter.conversion(this); - } -} diff --git a/dss-azkaban-scheduler-appjoint/src/main/java/com/webank/wedatasphere/dss/appjoint/scheduler/azkaban/entity/LinkisAzkabanShareNode.java b/dss-azkaban-scheduler-appjoint/src/main/java/com/webank/wedatasphere/dss/appjoint/scheduler/azkaban/entity/LinkisAzkabanShareNode.java deleted file mode 100644 index 44ac7d26ba..0000000000 --- a/dss-azkaban-scheduler-appjoint/src/main/java/com/webank/wedatasphere/dss/appjoint/scheduler/azkaban/entity/LinkisAzkabanShareNode.java +++ /dev/null @@ -1,35 +0,0 @@ -package com.webank.wedatasphere.dss.appjoint.scheduler.azkaban.entity; - -import com.webank.wedatasphere.dss.appjoint.scheduler.azkaban.linkisjob.LinkisJobConverter; -import com.webank.wedatasphere.dss.appjoint.scheduler.entity.SchedulerNode; -import com.webank.wedatasphere.dss.appjoint.scheduler.entity.ShareNode; - -public class LinkisAzkabanShareNode extends LinkisAzkabanSchedulerNode implements ShareNode { - private SchedulerNode schedulerNode; - private int shareTimes; - - @Override - public int getShareTimes() { - return shareTimes; - } - - @Override - public void setShareTimes(int num) { - this.shareTimes = num; - } - - @Override - public SchedulerNode getSchedulerNode() { - return schedulerNode; - } - - @Override - public void setSchedulerNode(SchedulerNode node) { - this.schedulerNode = node; - } - - @Override - public String toJobString(LinkisJobConverter converter) { - return converter.conversion(this); - } -} diff --git a/dss-azkaban-scheduler-appjoint/src/main/java/com/webank/wedatasphere/dss/appjoint/scheduler/azkaban/hooks/LinkisAzkabanFlowPublishHook.java b/dss-azkaban-scheduler-appjoint/src/main/java/com/webank/wedatasphere/dss/appjoint/scheduler/azkaban/hooks/LinkisAzkabanFlowPublishHook.java deleted file mode 100644 index 73cec2164c..0000000000 --- a/dss-azkaban-scheduler-appjoint/src/main/java/com/webank/wedatasphere/dss/appjoint/scheduler/azkaban/hooks/LinkisAzkabanFlowPublishHook.java +++ /dev/null @@ -1,113 +0,0 @@ -package com.webank.wedatasphere.dss.appjoint.scheduler.azkaban.hooks; - - -import com.google.gson.Gson; -import com.webank.wedatasphere.dss.appjoint.scheduler.hooks.AbstractFlowPublishHook; -import com.webank.wedatasphere.dss.appjoint.scheduler.hooks.NodePublishHook; -import com.webank.wedatasphere.dss.appjoint.scheduler.azkaban.constant.AzkabanConstant; -import com.webank.wedatasphere.dss.appjoint.scheduler.azkaban.entity.AzkabanSchedulerFlow; -import com.webank.wedatasphere.dss.common.entity.Resource; -import com.webank.wedatasphere.dss.appjoint.scheduler.entity.SchedulerFlow; -import com.webank.wedatasphere.dss.common.exception.DSSErrorException; -import org.apache.commons.io.FileUtils; -import org.apache.commons.io.IOUtils; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import java.io.File; -import java.io.FileOutputStream; -import java.util.List; -import java.util.Map; - -/** - * Created by allenlliu on 2019/9/20. - */ - -public class LinkisAzkabanFlowPublishHook extends AbstractFlowPublishHook { - - private static final Logger LOGGER = LoggerFactory.getLogger(LinkisAzkabanFlowPublishHook.class); - - public LinkisAzkabanFlowPublishHook(){ - NodePublishHook[] nodePublishHooks = {new LinkisAzkabanNodePublishHook()}; - setNodeHooks(nodePublishHooks); - } - - @Override - public void prePublish(SchedulerFlow flow) throws DSSErrorException { - // 处理资源生成文件等等 - writeFlowResourcesToLocal(flow); - writeFlowPropertiesToLocal(flow); - super.prePublish(flow); - } - - private void writeFlowResourcesToLocal(SchedulerFlow flow) throws DSSErrorException { - List flowResources = flow.getFlowResources(); - FileOutputStream os = null; - try { - String storePath = ((AzkabanSchedulerFlow)flow).getStorePath(); - File flowDir = new File(storePath); - FileUtils.forceMkdir(flowDir); - if(flowResources == null || flowResources.isEmpty()) {return;} - String projectStorePath = getProjectStorePath(storePath); - String flowResourceStringPrefix = getFlowResourceStringPrefix(projectStorePath,storePath); - String flowtResourceString = flowResourceStringPrefix + new Gson().toJson(flowResources) + "\n"; - File projectResourcesFile = new File(projectStorePath, "project.properties"); - os = FileUtils.openOutputStream(projectResourcesFile,true); - os.write(flowtResourceString.getBytes()); - }catch (Exception e){ - LOGGER.error("write FlowResources to local failed,reason:",e); - throw new DSSErrorException(90006,e.getMessage()); - }finally { - IOUtils.closeQuietly(os); - } - } - - private String getFlowResourceStringPrefix(String projectStorePath, String storePath) { - // TODO: 2019/9/30 需要做用户工作流命名为subFlows的情况的判断 - String substring = storePath.substring(projectStorePath.length() + 1); - String prefix = substring.replaceAll("\\" + File.separator +"subFlows" + "\\" + File.separator,"."); - return "flow." + prefix + "_.resources="; - } - - private String getProjectStorePath(String storePath) { - int indexOf = storePath.indexOf("subFlows"); - if(indexOf != -1){ - storePath = storePath.substring(0, indexOf-1); - } - return storePath.substring(0, storePath.lastIndexOf(File.separator)); - } - - private void writeFlowPropertiesToLocal(SchedulerFlow flow)throws DSSErrorException { - List> flowProperties = flow.getFlowProperties(); - if(flowProperties == null || flowProperties.isEmpty()) {return;} - FileOutputStream os = null; - try { - String storePath = ((AzkabanSchedulerFlow)flow).getStorePath(); - File flowPrpsFile = new File(storePath,flow.getName() + AzkabanConstant.AZKABAN_PROPERTIES_SUFFIX); - flowPrpsFile.createNewFile(); - os = FileUtils.openOutputStream(flowPrpsFile,true); - StringBuilder stringBuilder = new StringBuilder(); - flowProperties.forEach(p ->p.forEach((k,v)->{ - stringBuilder.append(AzkabanConstant.LINKIS_FLOW_VARIABLE_KEY + k + "=" + v + "\n"); - })); - os.write(stringBuilder.toString().getBytes()); - }catch (Exception e){ - LOGGER.error("write flowProperties to local faailed,reason:",e); - throw new DSSErrorException(90007,e.getMessage()); - }finally { - IOUtils.closeQuietly(os); - } - } - - @Override - public void postPublish(SchedulerFlow flow) { - super.postPublish(flow); - } - - - @Override - public void setNodeHooks(NodePublishHook[] nodePublishHooks) { - super.setNodeHooks(nodePublishHooks); - } - - -} diff --git a/dss-azkaban-scheduler-appjoint/src/main/java/com/webank/wedatasphere/dss/appjoint/scheduler/azkaban/hooks/LinkisAzkabanNodePublishHook.java b/dss-azkaban-scheduler-appjoint/src/main/java/com/webank/wedatasphere/dss/appjoint/scheduler/azkaban/hooks/LinkisAzkabanNodePublishHook.java deleted file mode 100644 index 702275f40c..0000000000 --- a/dss-azkaban-scheduler-appjoint/src/main/java/com/webank/wedatasphere/dss/appjoint/scheduler/azkaban/hooks/LinkisAzkabanNodePublishHook.java +++ /dev/null @@ -1,79 +0,0 @@ -package com.webank.wedatasphere.dss.appjoint.scheduler.azkaban.hooks; - -import com.google.gson.Gson; -import com.webank.wedatasphere.dss.appjoint.scheduler.azkaban.entity.LinkisAzkabanSchedulerNode; -import com.webank.wedatasphere.dss.appjoint.scheduler.hooks.AbstractNodePublishHook; -import com.webank.wedatasphere.dss.appjoint.scheduler.azkaban.constant.AzkabanConstant; -import com.webank.wedatasphere.dss.appjoint.scheduler.azkaban.entity.AzkabanSchedulerNode; -import com.webank.wedatasphere.dss.common.entity.Resource; -import com.webank.wedatasphere.dss.appjoint.scheduler.entity.SchedulerNode; -import com.webank.wedatasphere.dss.common.exception.DSSErrorException; -import com.webank.wedatasphere.dss.appjoint.scheduler.azkaban.linkisjob.LinkisJobConverter; -import org.apache.commons.io.FileUtils; -import org.apache.commons.io.IOUtils; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import java.io.File; -import java.io.FileOutputStream; -import java.util.List; - - -public class LinkisAzkabanNodePublishHook extends AbstractNodePublishHook { - - private static final Logger LOGGER = LoggerFactory.getLogger(LinkisAzkabanNodePublishHook.class); - private LinkisJobConverter linkisJobConverter; - - public LinkisAzkabanNodePublishHook(){ - this.linkisJobConverter = new LinkisJobConverter(); - } - - @Override - public void prePublish(SchedulerNode schedulerNode) throws DSSErrorException { - writeNodeTojobLocal(schedulerNode); - writeNodeResourcesToLocal(schedulerNode); - } - - private void writeNodeTojobLocal(SchedulerNode schedulerNode) throws DSSErrorException { - LinkisAzkabanSchedulerNode azkabanSchedulerNode = (LinkisAzkabanSchedulerNode) schedulerNode; - FileOutputStream os = null; - try { - String storePath = azkabanSchedulerNode.getStorePath(); - File jobDirFile = new File(storePath); - FileUtils.forceMkdir(jobDirFile); - File jobFile = new File(storePath,schedulerNode.getName() + AzkabanConstant.AZKABAN_JOB_SUFFIX); - jobFile.createNewFile(); - String nodeString = azkabanSchedulerNode.toJobString(linkisJobConverter); - os = FileUtils.openOutputStream(jobFile,true); - os.write(nodeString.getBytes()); - }catch (Exception e){ - LOGGER.error("write AppJointNode to jobLocal failed,reason:",e); - throw new DSSErrorException(90017,e.getMessage()); - }finally { - IOUtils.closeQuietly(os); - } - } - - @Override - public void postPublish(SchedulerNode schedulerNode) { - - } - - private void writeNodeResourcesToLocal(SchedulerNode schedulerNode) throws DSSErrorException { - List nodeResources = schedulerNode.getDssNode().getResources(); - if(nodeResources == null || nodeResources.isEmpty()) {return;} - FileOutputStream os = null; - try { - String storePath = ((AzkabanSchedulerNode)schedulerNode).getStorePath(); - File jobFile = new File(storePath,schedulerNode.getName() + AzkabanConstant.AZKABAN_JOB_SUFFIX); - String nodeResourceString = AzkabanConstant.LINKIS_JOB_RESOURCES_KEY + new Gson().toJson(nodeResources); - os = FileUtils.openOutputStream(jobFile,true); - os.write(nodeResourceString.getBytes()); - }catch (Exception e){ - LOGGER.error("write nodeResources to local failed,reason:",e); - throw new DSSErrorException(90018,e.getMessage()); - }finally { - IOUtils.closeQuietly(os); - } - } - -} diff --git a/dss-azkaban-scheduler-appjoint/src/main/java/com/webank/wedatasphere/dss/appjoint/scheduler/azkaban/hooks/LinkisAzkabanProjectPublishHook.java b/dss-azkaban-scheduler-appjoint/src/main/java/com/webank/wedatasphere/dss/appjoint/scheduler/azkaban/hooks/LinkisAzkabanProjectPublishHook.java deleted file mode 100644 index 40effab61b..0000000000 --- a/dss-azkaban-scheduler-appjoint/src/main/java/com/webank/wedatasphere/dss/appjoint/scheduler/azkaban/hooks/LinkisAzkabanProjectPublishHook.java +++ /dev/null @@ -1,127 +0,0 @@ -package com.webank.wedatasphere.dss.appjoint.scheduler.azkaban.hooks; - -import com.google.gson.Gson; -import com.webank.wedatasphere.dss.appjoint.scheduler.azkaban.entity.AzkabanSchedulerProject; -import com.webank.wedatasphere.dss.appjoint.scheduler.azkaban.util.AzkabanUtilsScala; -import com.webank.wedatasphere.dss.appjoint.scheduler.entity.SchedulerFlow; -import com.webank.wedatasphere.dss.appjoint.scheduler.entity.SchedulerProject; -import com.webank.wedatasphere.dss.appjoint.scheduler.entity.SchedulerProjectVersionForFlows; -import com.webank.wedatasphere.dss.appjoint.scheduler.hooks.AbstractProjectPublishHook; -import com.webank.wedatasphere.dss.appjoint.scheduler.hooks.FlowPublishHook; -import com.webank.wedatasphere.dss.common.entity.Resource; -import com.webank.wedatasphere.dss.common.exception.DSSErrorException; -import org.apache.commons.io.FileUtils; -import org.apache.commons.io.IOUtils; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import java.io.File; -import java.io.FileOutputStream; -import java.util.ArrayList; -import java.util.List; - -/** - * Created by cooperyang on 2019/9/18 - */ - -public class LinkisAzkabanProjectPublishHook extends AbstractProjectPublishHook { - - private static final Logger LOGGER = LoggerFactory.getLogger( LinkisAzkabanProjectPublishHook.class); - public LinkisAzkabanProjectPublishHook(){ - FlowPublishHook[] flowPublishHooks = {new LinkisAzkabanFlowPublishHook()}; - setFlowPublishHooks(flowPublishHooks); - } - - @Override - public void setFlowPublishHooks(FlowPublishHook[] flowPublishHooks) { - super.setFlowPublishHooks(flowPublishHooks); - } - - @Override - public void prePublish(SchedulerProject project) throws DSSErrorException { - //1.检查重复的节点名 - AzkabanSchedulerProject publishProject = (AzkabanSchedulerProject)project; - SchedulerProjectVersionForFlows schedulerProjectVersionForFlows = publishProject.getProjectVersions().get(0); - List allSchedulerFlows = schedulerProjectVersionForFlows.getFlows(); - List repeatNode = AzkabanUtilsScala.getRepeatNodeName(getAllNodeName(allSchedulerFlows)); - if(repeatNode.size()>0){ - throw new DSSErrorException(80001,"重复的节点名称："+repeatNode.toString()); - } - //删除可能未处理的zip包和文件夹 - removeProjectStoreDirAndzip(publishProject); - // 2.处理资源 - writeProjectResourcesToLocal(publishProject); - //3.调用flowPublishHooks - super.prePublish(project); - } - - private void removeProjectStoreDirAndzip(AzkabanSchedulerProject publishProject) throws DSSErrorException { - String storePath = publishProject.getStorePath(); - File projectDir = new File(storePath); - try { - if(projectDir.exists()) { - LOGGER.info("exist project dir{} before publish ,now remove it",storePath); - FileUtils.deleteDirectory(projectDir); - } - String projectZip = projectDir.getParent() + File.separator + publishProject.getName() + ".zip"; - File zipFile = new File(projectZip); - if(zipFile.exists()){ - LOGGER.info("exist project zip{} before publish ,now remove it",projectZip); - zipFile.delete(); - } - }catch (Exception e){ - LOGGER.error("delete project dir or zip failed,reaseon:",e); - throw new DSSErrorException(90020,e.getMessage()); - } - } - - private void writeProjectResourcesToLocal(AzkabanSchedulerProject publishProject)throws DSSErrorException { - List resources = publishProject.getDssProject().getProjectResources(); - FileOutputStream os = null; - try { - String storePath = publishProject.getStorePath(); - File projectDir = new File(storePath); - FileUtils.forceMkdir(projectDir); - File projectResourcesFile = new File(storePath, "project.properties"); - projectResourcesFile.createNewFile(); - if(resources == null || resources.isEmpty()) {return;} - String projectResourceStringPrefix = "project." + publishProject.getName() + ".resources="; - String projectResourceString = projectResourceStringPrefix + new Gson().toJson(resources) + "\n"; - os = FileUtils.openOutputStream(projectResourcesFile); - os.write(projectResourceString.getBytes()); - }catch (Exception e){ - LOGGER.error("write projectResources to local failed,reason:",e); - throw new DSSErrorException(90015,e.getMessage()); - }finally { - IOUtils.closeQuietly(os); - } - } - - @Override - public void postPublish(SchedulerProject project) throws DSSErrorException { - // TODO: 2019/9/26 1.删除zip包和文件夹 - AzkabanSchedulerProject publishProject = (AzkabanSchedulerProject)project; - String storePath = publishProject.getStorePath(); - try { - String projectName = publishProject.getName(); - File file = new File(storePath); - new File(file.getParent() + File.separator + projectName + ".zip" ).delete(); - }catch (Exception e){ - LOGGER.error("删除发布后的zip包失败",e); - throw new DSSErrorException(90001,"发布后删除工程目录失败"); - } - // TODO: 2019/9/26 flowPublishHooks ... - } - - /** - * 直接从allFlows 中获取所有节点的名字，免去一次递归 - * @param allSchedulerFlows - * @return - */ - public List getAllNodeName(List allSchedulerFlows) { - List nodeNames = new ArrayList<>(); - allSchedulerFlows.forEach(flow ->flow.getSchedulerNodes().forEach(node ->nodeNames.add(node.getName()))); - return nodeNames; - } - - -} diff --git a/dss-azkaban-scheduler-appjoint/src/main/java/com/webank/wedatasphere/dss/appjoint/scheduler/azkaban/linkisjob/AzkabanSubFlowJobTuning.java b/dss-azkaban-scheduler-appjoint/src/main/java/com/webank/wedatasphere/dss/appjoint/scheduler/azkaban/linkisjob/AzkabanSubFlowJobTuning.java deleted file mode 100644 index 4b31aa9dd4..0000000000 --- a/dss-azkaban-scheduler-appjoint/src/main/java/com/webank/wedatasphere/dss/appjoint/scheduler/azkaban/linkisjob/AzkabanSubFlowJobTuning.java +++ /dev/null @@ -1,23 +0,0 @@ -package com.webank.wedatasphere.dss.appjoint.scheduler.azkaban.linkisjob; - - - -/** - * Created by cooperyang on 2019/11/1. - */ - -public class AzkabanSubFlowJobTuning implements LinkisJobTuning { - - @Override - public LinkisJob tuningJob(LinkisJob job) { - job.setType("flow"); - job.setLinkistype(null); - job.getConf().put("flow.name",job.getName() + "_"); - return job; - } - - @Override - public boolean ifJobCantuning(String nodeType) { - return "workflow.subflow".equals(nodeType); - } -} diff --git a/dss-azkaban-scheduler-appjoint/src/main/java/com/webank/wedatasphere/dss/appjoint/scheduler/azkaban/linkisjob/LinkisJob.java b/dss-azkaban-scheduler-appjoint/src/main/java/com/webank/wedatasphere/dss/appjoint/scheduler/azkaban/linkisjob/LinkisJob.java deleted file mode 100644 index fb19c05776..0000000000 --- a/dss-azkaban-scheduler-appjoint/src/main/java/com/webank/wedatasphere/dss/appjoint/scheduler/azkaban/linkisjob/LinkisJob.java +++ /dev/null @@ -1,74 +0,0 @@ -package com.webank.wedatasphere.dss.appjoint.scheduler.azkaban.linkisjob; - -import java.util.HashMap; -import java.util.Map; - -/** - * Created by cooperyang on 2019/11/1. - */ -public class LinkisJob { - private String name; - private String type; - private String linkistype; - private String proxyUser; - private String dependencies; - private Map conf; - private String command; - - public String getName() { - return name; - } - - public void setName(String name) { - this.name = name; - } - - public String getType() { - return type; - } - - public void setType(String type) { - this.type = type; - } - - public String getLinkistype() { - return linkistype; - } - - public void setLinkistype(String linkistype) { - this.linkistype = linkistype; - } - - public String getProxyUser() { - return proxyUser; - } - - public void setProxyUser(String proxyUser) { - this.proxyUser = proxyUser; - } - - public String getDependencies() { - return dependencies; - } - - public void setDependencies(String dependencies) { - this.dependencies = dependencies; - } - - public Map getConf() { - return conf; - } - - public void setConf(Map conf) { - this.conf = conf; - } - - public String getCommand() { - return command; - } - - public void setCommand(String command) { - this.command = command; - } - -} diff --git a/dss-azkaban-scheduler-appjoint/src/main/java/com/webank/wedatasphere/dss/appjoint/scheduler/azkaban/linkisjob/LinkisJobConverter.java b/dss-azkaban-scheduler-appjoint/src/main/java/com/webank/wedatasphere/dss/appjoint/scheduler/azkaban/linkisjob/LinkisJobConverter.java deleted file mode 100644 index ef6f3f3b1c..0000000000 --- a/dss-azkaban-scheduler-appjoint/src/main/java/com/webank/wedatasphere/dss/appjoint/scheduler/azkaban/linkisjob/LinkisJobConverter.java +++ /dev/null @@ -1,112 +0,0 @@ -package com.webank.wedatasphere.dss.appjoint.scheduler.azkaban.linkisjob; - -import com.google.gson.Gson; -import com.webank.wedatasphere.dss.appjoint.scheduler.azkaban.constant.AzkabanConstant; -import com.webank.wedatasphere.dss.appjoint.scheduler.azkaban.entity.LinkisAzkabanReadNode; -import com.webank.wedatasphere.dss.appjoint.scheduler.azkaban.entity.LinkisAzkabanSchedulerNode; -import com.webank.wedatasphere.dss.appjoint.scheduler.azkaban.entity.LinkisAzkabanShareNode; -import com.webank.wedatasphere.dss.appjoint.scheduler.constant.SchedulerAppJointConstant; -import org.apache.commons.lang.StringUtils; -import java.util.Arrays; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -/** - * Created by cooperyang on 2019/11/1. - */ - -public class LinkisJobConverter { - - public LinkisJobConverter(){ - LinkisJobTuning[] linkisJobTunings = {new AzkabanSubFlowJobTuning()}; - this.linkisJobTunings = linkisJobTunings; - } - private LinkisJobTuning[] linkisJobTunings; - - public String conversion(LinkisAzkabanSchedulerNode schedulerNode){ - return baseConversion(schedulerNode); - } - - public String conversion(LinkisAzkabanReadNode schedulerNode){ - String tmpConversion = baseConversion(schedulerNode); - String[] shareNodeArray = schedulerNode.getShareNodeIds(); - String readNodePrefix = "read.nodes = "; - String shareNodeIds = StringUtils.join(shareNodeArray, ","); - return tmpConversion + readNodePrefix + shareNodeIds + "\n"; - } - - public String conversion(LinkisAzkabanShareNode schedulerNode){ - String tmpConversion = baseConversion(schedulerNode); - String shareNodePrefix = "share.num = "; - String shareNum = Integer.toString((schedulerNode).getShareTimes()); - return tmpConversion + shareNodePrefix + shareNum + "\n"; - } - - private String baseConversion(LinkisAzkabanSchedulerNode schedulerNode){ - LinkisJob job = new LinkisJob(); - job.setConf(new HashMap()); - job.setName(schedulerNode.getName()); - convertHead(schedulerNode,job); - convertDependencies(schedulerNode,job); - convertProxyUser(schedulerNode,job); - convertConfiguration(schedulerNode,job); - convertJobCommand(schedulerNode,job); - Arrays.stream(linkisJobTunings).forEach(t ->{ - if(t.ifJobCantuning(schedulerNode.getNodeType())) t.tuningJob(job); - }); - return convertJobtoString(job); - } - - private String convertJobtoString(LinkisJob job){ - HashMap map = new HashMap<>(); - map.put(AzkabanConstant.JOB_TYPE,job.getType()); - map.put(AzkabanConstant.LINKIS_TYPE,job.getLinkistype()); - map.put(AzkabanConstant.ZAKABAN_DEPENDENCIES_KEY,job.getDependencies()); - map.put(SchedulerAppJointConstant.PROXY_USER,job.getProxyUser()); - map.put(AzkabanConstant.JOB_COMMAND,job.getCommand()); - map.putAll(job.getConf()); - StringBuilder stringBuilder = new StringBuilder(); - map.forEach((k,v)->{ - if(v != null) stringBuilder.append(k).append("=").append(v).append("\n"); - }); - return stringBuilder.toString(); - } - - private void convertHead(LinkisAzkabanSchedulerNode schedulerNode,LinkisJob job){ - job.setType("linkis"); - job.setLinkistype(schedulerNode.getNodeType()); - } - - private void convertDependencies(LinkisAzkabanSchedulerNode schedulerNode,LinkisJob job){ - List dependencys = schedulerNode.getDssNode().getDependencys(); - if(dependencys != null && !dependencys.isEmpty()) { - StringBuilder dependencies = new StringBuilder(); - dependencys.forEach(d ->dependencies.append(d + ",")); - job.setDependencies(dependencies.substring(0,dependencies.length()-1)); - } - } - - private void convertProxyUser(LinkisAzkabanSchedulerNode schedulerNode,LinkisJob job){ - String userProxy = schedulerNode.getDssNode().getUserProxy(); - if(!StringUtils.isEmpty(userProxy)) job.setProxyUser(userProxy); - } - - private void convertConfiguration(LinkisAzkabanSchedulerNode schedulerNode,LinkisJob job){ - Map params = schedulerNode.getDssNode().getParams(); - if (params != null && !params.isEmpty()) { - Object configuration = params.get("configuration"); - String confprefix = "node.conf."; - ((Map>)configuration).forEach((k,v)->v.forEach((k2,v2)->job.getConf().put(confprefix + k + "." + k2,v2.toString()))); - } - - } - - private void convertJobCommand(LinkisAzkabanSchedulerNode schedulerNode,LinkisJob job){ - Map jobContent = schedulerNode.getDssNode().getJobContent(); - if(jobContent != null) { - jobContent.remove("jobParams"); - job.setCommand(new Gson().toJson(jobContent)); - } - } -} diff --git a/dss-azkaban-scheduler-appjoint/src/main/java/com/webank/wedatasphere/dss/appjoint/scheduler/azkaban/linkisjob/LinkisJobTuning.java b/dss-azkaban-scheduler-appjoint/src/main/java/com/webank/wedatasphere/dss/appjoint/scheduler/azkaban/linkisjob/LinkisJobTuning.java deleted file mode 100644 index 0884c1b5e6..0000000000 --- a/dss-azkaban-scheduler-appjoint/src/main/java/com/webank/wedatasphere/dss/appjoint/scheduler/azkaban/linkisjob/LinkisJobTuning.java +++ /dev/null @@ -1,11 +0,0 @@ -package com.webank.wedatasphere.dss.appjoint.scheduler.azkaban.linkisjob; - -/** - * Created by cooperyang on 2019/11/1. - */ -public interface LinkisJobTuning { - - LinkisJob tuningJob(LinkisJob job); - - boolean ifJobCantuning(String nodeType); -} diff --git a/dss-azkaban-scheduler-appjoint/src/main/java/com/webank/wedatasphere/dss/appjoint/scheduler/azkaban/parser/AzkabanFlowParser.java b/dss-azkaban-scheduler-appjoint/src/main/java/com/webank/wedatasphere/dss/appjoint/scheduler/azkaban/parser/AzkabanFlowParser.java deleted file mode 100644 index 0fe7350837..0000000000 --- a/dss-azkaban-scheduler-appjoint/src/main/java/com/webank/wedatasphere/dss/appjoint/scheduler/azkaban/parser/AzkabanFlowParser.java +++ /dev/null @@ -1,38 +0,0 @@ -package com.webank.wedatasphere.dss.appjoint.scheduler.azkaban.parser; - -import com.webank.wedatasphere.dss.appjoint.scheduler.parser.AbstractFlowParser; -import com.webank.wedatasphere.dss.appjoint.scheduler.parser.NodeParser; -import com.webank.wedatasphere.dss.appjoint.scheduler.azkaban.entity.AzkabanSchedulerFlow; -import com.webank.wedatasphere.dss.common.entity.flow.DSSJSONFlow; -import com.webank.wedatasphere.dss.appjoint.scheduler.entity.SchedulerFlow; -import java.util.ArrayList; - - -public class AzkabanFlowParser extends AbstractFlowParser { - - public AzkabanFlowParser(){ - ArrayList list = new ArrayList<>(); - list.add(new LinkisAzkabanNodeParser()); - list.add(new LinkisAzkabanSendEmailNodeParser()); - NodeParser[] nodeParsers =new NodeParser[list.size()]; - list.toArray(nodeParsers); - setNodeParsers(nodeParsers); - } - - @Override - protected SchedulerFlow createSchedulerFlow() { - return new AzkabanSchedulerFlow(); - } - - @Override - public void setNodeParsers(NodeParser[] nodeParsers) { - super.setNodeParsers(nodeParsers); - } - - @Override - public Boolean ifFlowCanParse(DSSJSONFlow flow) { - return true; - } - - -} diff --git a/dss-azkaban-scheduler-appjoint/src/main/java/com/webank/wedatasphere/dss/appjoint/scheduler/azkaban/parser/AzkabanNodeParser.java b/dss-azkaban-scheduler-appjoint/src/main/java/com/webank/wedatasphere/dss/appjoint/scheduler/azkaban/parser/AzkabanNodeParser.java deleted file mode 100644 index 156f14ba5a..0000000000 --- a/dss-azkaban-scheduler-appjoint/src/main/java/com/webank/wedatasphere/dss/appjoint/scheduler/azkaban/parser/AzkabanNodeParser.java +++ /dev/null @@ -1,17 +0,0 @@ -package com.webank.wedatasphere.dss.appjoint.scheduler.azkaban.parser; - - -import com.webank.wedatasphere.dss.appjoint.scheduler.entity.SchedulerEdge; -import com.webank.wedatasphere.dss.appjoint.scheduler.entity.SchedulerNode; -import com.webank.wedatasphere.dss.appjoint.scheduler.parser.AbstractNodeParser; - -import java.util.List; - -public abstract class AzkabanNodeParser extends AbstractNodeParser { - - protected Boolean generateDependencies(SchedulerNode node, List flowEdges){return null;}; - - protected Boolean assertNodeType(){return null;}; - - protected String getNodeResourcesStr(){return null;}; -} diff --git a/dss-azkaban-scheduler-appjoint/src/main/java/com/webank/wedatasphere/dss/appjoint/scheduler/azkaban/parser/AzkabanProjectParser.java b/dss-azkaban-scheduler-appjoint/src/main/java/com/webank/wedatasphere/dss/appjoint/scheduler/azkaban/parser/AzkabanProjectParser.java deleted file mode 100644 index 3895d7c687..0000000000 --- a/dss-azkaban-scheduler-appjoint/src/main/java/com/webank/wedatasphere/dss/appjoint/scheduler/azkaban/parser/AzkabanProjectParser.java +++ /dev/null @@ -1,32 +0,0 @@ -package com.webank.wedatasphere.dss.appjoint.scheduler.azkaban.parser; - -import com.webank.wedatasphere.dss.appjoint.scheduler.azkaban.entity.AzkabanSchedulerProject; -import com.webank.wedatasphere.dss.appjoint.scheduler.entity.AbstractSchedulerProject; -import com.webank.wedatasphere.dss.appjoint.scheduler.parser.AbstractProjectParser; -import com.webank.wedatasphere.dss.appjoint.scheduler.parser.FlowParser; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -/** - * Created by allenlliu on 2019/9/16. - */ - -public class AzkabanProjectParser extends AbstractProjectParser { - - public AzkabanProjectParser(){ - FlowParser[] flowParsers = {new AzkabanFlowParser()}; - setFlowParsers(flowParsers); - } - - private static final Logger logger = LoggerFactory.getLogger(AzkabanProjectParser.class); - - @Override - protected AbstractSchedulerProject createSchedulerProject() { - return new AzkabanSchedulerProject(); - } - - @Override - public void setFlowParsers(FlowParser[] flowParsers) { - super.setFlowParsers(flowParsers); - } - -} diff --git a/dss-azkaban-scheduler-appjoint/src/main/java/com/webank/wedatasphere/dss/appjoint/scheduler/azkaban/parser/LinkisAzkabanNodeParser.java b/dss-azkaban-scheduler-appjoint/src/main/java/com/webank/wedatasphere/dss/appjoint/scheduler/azkaban/parser/LinkisAzkabanNodeParser.java deleted file mode 100644 index e9c8b86a90..0000000000 --- a/dss-azkaban-scheduler-appjoint/src/main/java/com/webank/wedatasphere/dss/appjoint/scheduler/azkaban/parser/LinkisAzkabanNodeParser.java +++ /dev/null @@ -1,31 +0,0 @@ -package com.webank.wedatasphere.dss.appjoint.scheduler.azkaban.parser; - -import com.webank.wedatasphere.dss.appjoint.scheduler.azkaban.entity.LinkisAzkabanSchedulerNode; -import com.webank.wedatasphere.dss.common.entity.node.DSSNode; -import com.webank.wedatasphere.dss.appjoint.scheduler.entity.SchedulerNode; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -public class LinkisAzkabanNodeParser extends AzkabanNodeParser { - - private static final Logger LOGGER = LoggerFactory.getLogger(LinkisAzkabanNodeParser.class); - - - @Override - public SchedulerNode parseNode(DSSNode dssNode) { - LinkisAzkabanSchedulerNode schedulerNode = new LinkisAzkabanSchedulerNode(); - schedulerNode.setDssNode(dssNode); - return schedulerNode; - } - - @Override - public Boolean ifNodeCanParse(DSSNode dssNode) { - //预留 - return true; - } - - @Override - public int getOrder() { - return 1; - } -} diff --git a/dss-azkaban-scheduler-appjoint/src/main/java/com/webank/wedatasphere/dss/appjoint/scheduler/azkaban/parser/LinkisAzkabanSendEmailNodeParser.java b/dss-azkaban-scheduler-appjoint/src/main/java/com/webank/wedatasphere/dss/appjoint/scheduler/azkaban/parser/LinkisAzkabanSendEmailNodeParser.java deleted file mode 100644 index 723b35f7e8..0000000000 --- a/dss-azkaban-scheduler-appjoint/src/main/java/com/webank/wedatasphere/dss/appjoint/scheduler/azkaban/parser/LinkisAzkabanSendEmailNodeParser.java +++ /dev/null @@ -1,27 +0,0 @@ -package com.webank.wedatasphere.dss.appjoint.scheduler.azkaban.parser; - -import com.webank.wedatasphere.dss.appjoint.scheduler.azkaban.entity.LinkisAzkabanReadNode; -import com.webank.wedatasphere.dss.appjoint.scheduler.azkaban.entity.LinkisAzkabanSchedulerNode; -import com.webank.wedatasphere.dss.appjoint.scheduler.entity.ReadNode; -import com.webank.wedatasphere.dss.appjoint.scheduler.entity.SchedulerNode; -import com.webank.wedatasphere.dss.appjoint.scheduler.parser.SendEmailNodeParser; - - - -public class LinkisAzkabanSendEmailNodeParser extends SendEmailNodeParser { - - @Override - protected ReadNode createReadNode() { - return new LinkisAzkabanReadNode(); - } - - @Override - protected SchedulerNode createSchedulerNode() { - return new LinkisAzkabanSchedulerNode(); - } - - @Override - public int getOrder() { - return 2; - } -} diff --git a/dss-azkaban-scheduler-appjoint/src/main/java/com/webank/wedatasphere/dss/appjoint/scheduler/azkaban/service/AzkabanProjectService.java b/dss-azkaban-scheduler-appjoint/src/main/java/com/webank/wedatasphere/dss/appjoint/scheduler/azkaban/service/AzkabanProjectService.java deleted file mode 100644 index eb536b14b7..0000000000 --- a/dss-azkaban-scheduler-appjoint/src/main/java/com/webank/wedatasphere/dss/appjoint/scheduler/azkaban/service/AzkabanProjectService.java +++ /dev/null @@ -1,223 +0,0 @@ -package com.webank.wedatasphere.dss.appjoint.scheduler.azkaban.service; - - -import com.webank.wedatasphere.dss.appjoint.exception.AppJointErrorException; -import com.webank.wedatasphere.dss.appjoint.service.AppJointUrlImpl; -import com.webank.wedatasphere.dss.appjoint.scheduler.azkaban.entity.AzkabanSchedulerProject; -import com.webank.wedatasphere.dss.appjoint.scheduler.service.SchedulerProjectService; -import com.webank.wedatasphere.dss.appjoint.service.session.Session; -import com.webank.wedatasphere.dss.common.entity.project.Project; -import com.webank.wedatasphere.dss.common.exception.DSSErrorException; -import com.webank.wedatasphere.dss.appjoint.scheduler.azkaban.util.AzkabanUtils; -import com.webank.wedatasphere.dss.common.utils.DSSExceptionUtils; -import com.webank.wedatasphere.dss.common.utils.ZipHelper; -import org.apache.commons.io.IOUtils; -import org.apache.commons.lang.StringUtils; -import org.apache.http.Consts; -import org.apache.http.Header; -import org.apache.http.HttpEntity; -import org.apache.http.NameValuePair; -import org.apache.http.client.CookieStore; -import org.apache.http.client.entity.EntityBuilder; -import org.apache.http.client.entity.UrlEncodedFormEntity; -import org.apache.http.client.methods.CloseableHttpResponse; -import org.apache.http.client.methods.HttpGet; -import org.apache.http.client.methods.HttpPost; -import org.apache.http.client.protocol.HttpClientContext; -import org.apache.http.cookie.Cookie; -import org.apache.http.entity.ContentType; -import org.apache.http.entity.mime.MultipartEntityBuilder; -import org.apache.http.impl.client.BasicCookieStore; -import org.apache.http.impl.client.CloseableHttpClient; -import org.apache.http.impl.client.HttpClients; -import org.apache.http.message.BasicNameValuePair; -import org.apache.http.protocol.HTTP; -import org.apache.http.util.EntityUtils; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.File; -import java.io.InputStream; -import java.io.UnsupportedEncodingException; -import java.net.URLDecoder; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; -import java.util.Optional; - -/** - * Created by cooperyang on 2019/9/16. - */ -public final class AzkabanProjectService extends AppJointUrlImpl implements SchedulerProjectService { - - private Logger logger = LoggerFactory.getLogger(this.getClass()); - - private String projectUrl; - - @Override - public void setBaseUrl(String baseUrl) { - this.projectUrl = baseUrl + "/manager"; - } - - @Override - public List fetchProjects(Session session) { - return null; - } - - /** - * parameters: - * name = value - * description=value - * - * @param project - * @param session - * @throws AppJointErrorException - */ - @Override - public Project createProject(Project project, Session session) throws AppJointErrorException { - List params = new ArrayList<>(); - params.add(new BasicNameValuePair("action", "create")); - params.add(new BasicNameValuePair("name", project.getName())); - params.add(new BasicNameValuePair("description", project.getDescription())); - HttpPost httpPost = new HttpPost(projectUrl); - httpPost.addHeader(HTTP.CONTENT_ENCODING, HTTP.IDENTITY_CODING); - CookieStore cookieStore = new BasicCookieStore(); - cookieStore.addCookie(session.getCookies()[0]); - HttpEntity entity = EntityBuilder.create() - .setContentType(ContentType.create("application/x-www-form-urlencoded", Consts.UTF_8)) - .setParameters(params).build(); - httpPost.setEntity(entity); - CloseableHttpClient httpClient = null; - CloseableHttpResponse response = null; - try { - httpClient = HttpClients.custom().setDefaultCookieStore(cookieStore).build(); - response = httpClient.execute(httpPost); - HttpEntity ent = response.getEntity(); - String entStr = IOUtils.toString(ent.getContent(), "utf-8"); - logger.error("新建工程 {}, azkaban 返回的信息是 {}", project.getName(), entStr); - String message = AzkabanUtils.handleAzkabanEntity(entStr); - if (!"success".equals(message)) { - throw new AppJointErrorException(90008, "新建工程失败, 原因:" + message); - } - } catch (Exception e) { - logger.error("创建工程失败:", e); - throw new AppJointErrorException(90009, e.getMessage(), e); - } finally { - IOUtils.closeQuietly(response); - IOUtils.closeQuietly(httpClient); - } - return null; - } - - /** - * delete=boolean - * project=projectName - * - * @param project - * @param session - */ - @Override - public void deleteProject(Project project, Session session) throws AppJointErrorException { - List params = new ArrayList<>(); - params.add(new BasicNameValuePair("delete", "true")); - params.add(new BasicNameValuePair("project", project.getName())); - CookieStore cookieStore = new BasicCookieStore(); - cookieStore.addCookie(session.getCookies()[0]); - HttpClientContext context = HttpClientContext.create(); - CloseableHttpResponse response = null; - CloseableHttpClient httpClient = null; - try { - String finalUrl = projectUrl + "?" + EntityUtils.toString(new UrlEncodedFormEntity(params)); - HttpGet httpGet = new HttpGet(finalUrl); - httpGet.addHeader(HTTP.CONTENT_ENCODING, "UTF-8"); - httpClient = HttpClients.custom().setDefaultCookieStore(cookieStore).build(); - response = httpClient.execute(httpGet, context); - Header[] allHeaders = context.getRequest().getAllHeaders(); - Optional
header = Arrays.stream(allHeaders).filter(f -> "Cookie".equals(f.getName())).findFirst(); - header.ifPresent(DSSExceptionUtils.handling(this::parseCookie)); - } catch (Exception e) { - logger.error("delete scheduler project failed,reason:",e); - throw new AppJointErrorException(90010, e.getMessage(), e); - } finally { - IOUtils.closeQuietly(response); - IOUtils.closeQuietly(httpClient); - } - } - - private void parseCookie(Header header) { - String[] arr = header.getValue().split(";"); - Arrays.stream(arr).filter(f -> f.toLowerCase().trim().contains("azkaban.failure.message=")) - .forEach(DSSExceptionUtils.handling(this::parseHeader)); - } - - private void parseHeader(String headerValue) throws UnsupportedEncodingException, AppJointErrorException { - String errorMsg = URLDecoder.decode(headerValue.split("=")[1], "UTF-8"); - if (StringUtils.isNotEmpty(errorMsg)) { - throw new AppJointErrorException(90011, errorMsg); - } - } - - @Override - public void publishProject(Project project, Session session) throws AppJointErrorException { - String tmpSavePath; - try { - tmpSavePath = zipProject(project); - //upload zip to Azkaban - uploadProject(tmpSavePath,project,session.getCookies()[0]); - } catch (Exception e) { - logger.error("upload sheduler failed:reason",e); - throw new AppJointErrorException(90012, e.getMessage(), e); - } - } - - private void uploadProject(String tmpSavePath, Project project, Cookie cookie) throws AppJointErrorException { - String projectName = project.getName(); - HttpPost httpPost = new HttpPost(projectUrl + "?project=" + projectName); - httpPost.addHeader(HTTP.CONTENT_ENCODING, "UTF-8"); - CloseableHttpResponse response = null; - File file = new File(tmpSavePath); - CookieStore cookieStore = new BasicCookieStore(); - cookieStore.addCookie(cookie); - CloseableHttpClient httpClient = null; - InputStream inputStream = null; - try { - httpClient = HttpClients.custom().setDefaultCookieStore(cookieStore).build(); - MultipartEntityBuilder entityBuilder = MultipartEntityBuilder.create(); - entityBuilder.addBinaryBody("file",file); - entityBuilder.addTextBody("ajax", "upload"); - entityBuilder.addTextBody("project", projectName); - httpPost.setEntity(entityBuilder.build()); - response = httpClient.execute(httpPost); - HttpEntity httpEntity = response.getEntity(); - inputStream = httpEntity.getContent(); - String entStr = null; - entStr = IOUtils.toString(inputStream, "utf-8"); - if(response.getStatusLine().getStatusCode() != 200){ - logger.error("调用azkaban上传接口的返回不为200, status code 是 {}", response.getStatusLine().getStatusCode()); - throw new AppJointErrorException(90013, "release project failed, " + entStr); - } - logger.info("upload project:{} success!",projectName); - }catch (Exception e){ - logger.error("upload failed,reason:",e); - throw new AppJointErrorException(90014,e.getMessage(), e); - } - finally { - IOUtils.closeQuietly(inputStream); - IOUtils.closeQuietly(response); - IOUtils.closeQuietly(httpClient); - } - } - - @Override - public void updateProject(Project project, Session session) throws AppJointErrorException { - - } - - public String zipProject(Project project) throws DSSErrorException { - AzkabanSchedulerProject publishProject = (AzkabanSchedulerProject) project; - String projectPath = publishProject.getStorePath(); - String zipPath = ZipHelper.zip(projectPath); - return zipPath; - } - -} diff --git a/dss-azkaban-scheduler-appjoint/src/main/java/com/webank/wedatasphere/dss/appjoint/scheduler/azkaban/service/AzkabanSecurityService.java b/dss-azkaban-scheduler-appjoint/src/main/java/com/webank/wedatasphere/dss/appjoint/scheduler/azkaban/service/AzkabanSecurityService.java deleted file mode 100644 index b24fe94e76..0000000000 --- a/dss-azkaban-scheduler-appjoint/src/main/java/com/webank/wedatasphere/dss/appjoint/scheduler/azkaban/service/AzkabanSecurityService.java +++ /dev/null @@ -1,160 +0,0 @@ -package com.webank.wedatasphere.dss.appjoint.scheduler.azkaban.service; - -import com.webank.wedatasphere.dss.appjoint.exception.AppJointErrorException; -import com.webank.wedatasphere.dss.appjoint.scheduler.azkaban.conf.AzkabanConf; -import com.webank.wedatasphere.dss.appjoint.scheduler.service.SchedulerSecurityService; -import com.webank.wedatasphere.dss.appjoint.service.AppJointUrlImpl; -import com.webank.wedatasphere.dss.appjoint.service.session.Session; -import com.webank.wedatasphere.dss.appjoint.scheduler.azkaban.constant.AzkabanConstant; -import com.webank.wedatasphere.linkis.common.utils.Utils; -import org.apache.commons.io.IOUtils; -import org.apache.http.HttpEntity; -import org.apache.http.NameValuePair; -import org.apache.http.client.CookieStore; -import org.apache.http.client.entity.UrlEncodedFormEntity; -import org.apache.http.client.methods.CloseableHttpResponse; -import org.apache.http.client.methods.HttpPost; -import org.apache.http.client.protocol.HttpClientContext; -import org.apache.http.cookie.Cookie; -import org.apache.http.impl.client.BasicCookieStore; -import org.apache.http.impl.client.CloseableHttpClient; -import org.apache.http.impl.client.HttpClients; -import org.apache.http.message.BasicNameValuePair; -import org.apache.http.util.EntityUtils; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.List; -import java.util.Optional; -import java.util.Properties; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.TimeUnit; - -/** - * Created by cooperyang on 2019/9/16. - */ -public final class AzkabanSecurityService extends AppJointUrlImpl implements SchedulerSecurityService { - - private static Logger LOGGER = LoggerFactory.getLogger(AzkabanSecurityService.class); - - private static final long EXPIRE_TIME = 1000 * 60 * 10L; - private ConcurrentHashMap sessionCache = new ConcurrentHashMap<>(); - private String securityUrl; - private static final String USER_NAME_KEY = "username"; - private static final String USER_TOKEN_KEY = AzkabanConf.AZKABAN_LOGIN_PWD.getValue(); - private static final String SESSION_ID_KEY = "azkaban.browser.session.id"; - private static Properties userToken ; - - static { - Utils.defaultScheduler().scheduleAtFixedRate(()->{ - LOGGER.info("load azkaban-user.xml"); - Properties properties = new Properties(); - try { - properties.load(AzkabanSecurityService.class.getClassLoader().getResourceAsStream(AzkabanConstant.TOKEN_FILE_NAME)); - userToken = properties; - } catch (IOException e) { - LOGGER.error("load error:",e); - } - },0,10, TimeUnit.MINUTES); - } - - public void reloadToken(){ - LOGGER.info("reload azkaban-user.xml"); - Properties properties = new Properties(); - try { - properties.load(AzkabanSecurityService.class.getClassLoader().getResourceAsStream(AzkabanConstant.TOKEN_FILE_NAME)); - userToken = properties; - } catch (IOException e) { - LOGGER.error("reload error:",e); - } - } - - @Override - public void setBaseUrl(String baseUrl) { - this.securityUrl = baseUrl + "/checkin"; - } - - @Override - public Session login(String user) throws AppJointErrorException { - synchronized (user.intern()) { - Session session = sessionCache.get(user); - if (session != null && !isExpire(session)) { - session.updateLastAccessTime(); - return session; - } - Session newSession; - try { - newSession = getSession(user, getUserToken(user)); - } catch (Exception e) { - LOGGER.error("获取session失败:", e); - throw new AppJointErrorException(90019, e.getMessage(), e); - } - ((AzkabanSession) newSession).setUser(user); - sessionCache.put(user, newSession); - return newSession; - } - } - - private Session getSession(String user, String token) throws IOException, AppJointErrorException { - HttpPost httpPost = new HttpPost(securityUrl); - List params = new ArrayList<>(); - params.add(new BasicNameValuePair(USER_NAME_KEY, user)); - params.add(new BasicNameValuePair(USER_TOKEN_KEY, token)); - params.add(new BasicNameValuePair("action", "login")); - httpPost.setEntity(new UrlEncodedFormEntity(params)); - CookieStore cookieStore = new BasicCookieStore(); - CloseableHttpClient httpClient = null; - CloseableHttpResponse response = null; - HttpClientContext context; - String responseContent; - try { - httpClient = HttpClients.custom().setDefaultCookieStore(cookieStore).build(); - context = HttpClientContext.create(); - response = httpClient.execute(httpPost, context); - HttpEntity entity = response.getEntity(); - responseContent = EntityUtils.toString(entity,"utf-8"); - LOGGER.info("Get azkaban response code is "+ response.getStatusLine().getStatusCode()+",response: "+responseContent); - if(response.getStatusLine().getStatusCode() != 200){ - throw new AppJointErrorException(90041, responseContent); - } - } finally { - IOUtils.closeQuietly(response); - IOUtils.closeQuietly(httpClient); - } - List cookies = context.getCookieStore().getCookies(); - Optional session = cookies.stream().filter(this::findSessionId).map(this::cookieToSession).findFirst(); - return session.orElseThrow(() -> new AppJointErrorException(90041,"Get azkaban session is null : "+ responseContent)); - } - - private boolean findSessionId(Cookie cookie) { - return SESSION_ID_KEY.equals(cookie.getName()); - } - - private Session cookieToSession(Cookie cookie) { - AzkabanSession session = new AzkabanSession(); - session.setCookies(new Cookie[]{cookie}); - return session; - } - - private String getUserToken(String user) throws AppJointErrorException { - //直接从配置文件中读取，有需求可以自己实现 - Object token = userToken.get(user); - if (token == null) throw new AppJointErrorException(90020,"用户token为空"); - return token.toString(); - } - - @Override - public void logout(String user) { - synchronized (user.intern()) { - sessionCache.remove(user); - } - } - - private boolean isExpire(Session session) { - return System.currentTimeMillis() - session.getLastAccessTime() > EXPIRE_TIME; - } - - -} diff --git a/dss-azkaban-scheduler-appjoint/src/main/java/com/webank/wedatasphere/dss/appjoint/scheduler/azkaban/service/AzkabanSession.java b/dss-azkaban-scheduler-appjoint/src/main/java/com/webank/wedatasphere/dss/appjoint/scheduler/azkaban/service/AzkabanSession.java deleted file mode 100644 index 77219cecf9..0000000000 --- a/dss-azkaban-scheduler-appjoint/src/main/java/com/webank/wedatasphere/dss/appjoint/scheduler/azkaban/service/AzkabanSession.java +++ /dev/null @@ -1,59 +0,0 @@ -package com.webank.wedatasphere.dss.appjoint.scheduler.azkaban.service; - -import com.webank.wedatasphere.dss.appjoint.service.session.Session; -import org.apache.http.Header; -import org.apache.http.cookie.Cookie; - -import java.util.HashMap; -import java.util.Map; - -/** - * Created by cooperyang on 2019/9/16. - */ -public final class AzkabanSession implements Session { - - private long lastAccessTime; - private Cookie[] cookies; - private Header[] headers; - private String user; - private Map parameters = new HashMap<>(); - - @Override - public String getUser() { - return user; - } - - public void setUser(String user) { - this.user = user; - } - - @Override - public Cookie[] getCookies() { - return this.cookies; - } - - @Override - public Header[] getHeaders() { - return this.headers; - } - - @Override - public Map getParameters() { - return this.parameters; - } - - @Override - public long getLastAccessTime() { - return this.lastAccessTime; - } - - public void setCookies(Cookie[] cookies){ - this.cookies = cookies; - } - - @Override - public void updateLastAccessTime() { - lastAccessTime = System.currentTimeMillis(); - } - -} diff --git a/dss-azkaban-scheduler-appjoint/src/main/java/com/webank/wedatasphere/dss/appjoint/scheduler/azkaban/tuning/AzkabanProjectTuning.java b/dss-azkaban-scheduler-appjoint/src/main/java/com/webank/wedatasphere/dss/appjoint/scheduler/azkaban/tuning/AzkabanProjectTuning.java deleted file mode 100644 index b4cec7a03e..0000000000 --- a/dss-azkaban-scheduler-appjoint/src/main/java/com/webank/wedatasphere/dss/appjoint/scheduler/azkaban/tuning/AzkabanProjectTuning.java +++ /dev/null @@ -1,69 +0,0 @@ -package com.webank.wedatasphere.dss.appjoint.scheduler.azkaban.tuning; - -import com.webank.wedatasphere.dss.appjoint.scheduler.azkaban.conf.AzkabanConf; -import com.webank.wedatasphere.dss.appjoint.scheduler.azkaban.entity.AzkabanSchedulerFlow; -import com.webank.wedatasphere.dss.appjoint.scheduler.azkaban.entity.AzkabanSchedulerProject; -import com.webank.wedatasphere.dss.appjoint.scheduler.entity.SchedulerFlow; -import com.webank.wedatasphere.dss.appjoint.scheduler.entity.SchedulerProject; -import com.webank.wedatasphere.dss.appjoint.scheduler.tuning.AbstractProjectTuning; -import com.webank.wedatasphere.dss.appjoint.scheduler.tuning.FlowTuning; -import java.io.File; -import java.text.SimpleDateFormat; -import java.util.ArrayList; -import java.util.Date; -import java.util.List; - -/** - * Created by cooperyang on 2019/9/26. - */ - -public class AzkabanProjectTuning extends AbstractProjectTuning { - - public AzkabanProjectTuning(){ - ArrayList list = new ArrayList<>(); - list.add(new LinkisAzkabanFlowTuning()); - list.add(new LinkisShareNodeFlowTuning()); - FlowTuning[] flowTunings =new FlowTuning[list.size()]; - setFlowTunings(list.toArray(flowTunings)); - } - - @Override - public SchedulerProject tuningSchedulerProject(SchedulerProject schedulerProject) { - if(ifProjectCanTuning(schedulerProject)){ - AzkabanSchedulerProject project = (AzkabanSchedulerProject) schedulerProject; - //1.给AzkabanSchedulerProject的storepath赋值 - assignStorePath(project); - //2.给他的rootflow的storepath也赋值 - List schedulerFlows = project.getSchedulerFlows(); - schedulerFlows.forEach(flow ->setRootFlowStorePath(project.getStorePath(),flow)); - } - return super.tuningSchedulerProject(schedulerProject); - } - - public void setRootFlowStorePath(String projectStorePath,SchedulerFlow schedulerFlow){ - AzkabanSchedulerFlow azkabanSchedulerFlow = (AzkabanSchedulerFlow) schedulerFlow; - azkabanSchedulerFlow.setStorePath(projectStorePath + File.separator + azkabanSchedulerFlow.getName()); - } - - - @Override - public void setFlowTunings(FlowTuning[] flowTunings) { - super.setFlowTunings(flowTunings); - } - - - public boolean ifProjectCanTuning(SchedulerProject schedulerProject){ - return schedulerProject instanceof AzkabanSchedulerProject; - } - - private void assignStorePath(AzkabanSchedulerProject azkabanSchedulerProject) { - SimpleDateFormat dateFormat = new SimpleDateFormat(AzkabanSchedulerProject.DATE_FORMAT); - Date date = new Date(); - String dataStr = dateFormat.format(date); - String userName = azkabanSchedulerProject.getDssProject().getUserName(); - String name = azkabanSchedulerProject.getName(); - String storePath = AzkabanConf.DEFAULT_STORE_PATH.getValue() + File.separator + userName - + File.separator + dataStr + File.separator +name; - azkabanSchedulerProject.setStorePath(storePath); - } -} diff --git a/dss-azkaban-scheduler-appjoint/src/main/java/com/webank/wedatasphere/dss/appjoint/scheduler/azkaban/tuning/LinkisAzkabanFlowTuning.java b/dss-azkaban-scheduler-appjoint/src/main/java/com/webank/wedatasphere/dss/appjoint/scheduler/azkaban/tuning/LinkisAzkabanFlowTuning.java deleted file mode 100644 index 42c9962f13..0000000000 --- a/dss-azkaban-scheduler-appjoint/src/main/java/com/webank/wedatasphere/dss/appjoint/scheduler/azkaban/tuning/LinkisAzkabanFlowTuning.java +++ /dev/null @@ -1,98 +0,0 @@ -package com.webank.wedatasphere.dss.appjoint.scheduler.azkaban.tuning; - - -import com.webank.wedatasphere.dss.appjoint.scheduler.azkaban.entity.AzkabanSchedulerFlow; -import com.webank.wedatasphere.dss.appjoint.scheduler.azkaban.entity.AzkabanSchedulerNode; -import com.webank.wedatasphere.dss.appjoint.scheduler.azkaban.entity.LinkisAzkabanSchedulerNode; -import com.webank.wedatasphere.dss.appjoint.scheduler.entity.SchedulerEdge; -import com.webank.wedatasphere.dss.appjoint.scheduler.entity.SchedulerFlow; -import com.webank.wedatasphere.dss.appjoint.scheduler.entity.SchedulerNode; -import com.webank.wedatasphere.dss.appjoint.scheduler.tuning.AbstractFlowTuning; -import com.webank.wedatasphere.dss.appjoint.scheduler.tuning.NodeTuning; -import com.webank.wedatasphere.dss.common.entity.node.DSSNodeDefault; -import java.io.File; -import java.util.ArrayList; -import java.util.List; -import java.util.Map; - -//DefaultFlowTuning修改为AzkabanFlowTuning -public class LinkisAzkabanFlowTuning extends AbstractFlowTuning { - - - - @Override - public SchedulerFlow tuningSchedulerFlow(SchedulerFlow schedulerFlow) { - //1.尾部添加依赖 - //2.设置子flow和node存储路径 - SchedulerFlow addEndNodeSchedulerFlow = addEndNodeForFlowName(schedulerFlow); - String flowStorePath = ((AzkabanSchedulerFlow) addEndNodeSchedulerFlow).getStorePath(); - if (addEndNodeSchedulerFlow.getChildren() != null) { - addEndNodeSchedulerFlow.getChildren().forEach(flow -> setFlowStorePath(flowStorePath, flow)); - } - - if (addEndNodeSchedulerFlow.getSchedulerNodes() != null) { - addEndNodeSchedulerFlow.getSchedulerNodes().forEach(node -> setNodeStorePath(flowStorePath, node)); - } - return super.tuningSchedulerFlow(schedulerFlow); - } - - private void setFlowStorePath(String flowStorePath, SchedulerFlow schedulerFlow) { - AzkabanSchedulerFlow azkabanSchedulerFlow = (AzkabanSchedulerFlow) schedulerFlow; - azkabanSchedulerFlow.setStorePath(flowStorePath + File.separator + "subFlows" + File.separator + azkabanSchedulerFlow.getName()); - } - - private void setNodeStorePath(String flowStorePath, SchedulerNode schedulerNode) { - AzkabanSchedulerNode azkabanSchedulerNode = (AzkabanSchedulerNode) schedulerNode; - azkabanSchedulerNode.setStorePath(flowStorePath + File.separator + "jobs" + File.separator + schedulerNode.getName()); - } - - @Override - public NodeTuning[] getNodeTunings() { - return new NodeTuning[0]; - } - - @Override - public Boolean ifFlowCanTuning(SchedulerFlow schedulerFlow) { - // TODO: 2019/9/30 应该加上node 是linkisazkabanNode的 instance判断，因为加到尾部节点的就是一个linkisazkabanNode - return schedulerFlow instanceof AzkabanSchedulerFlow; - } - - private SchedulerFlow addEndNodeForFlowName(SchedulerFlow flow) { - DSSNodeDefault endNode = new DSSNodeDefault(); - List endNodeList = getFlowEndJobList(flow); - endNode.setId(flow.getName() + "_"); - endNode.setName(flow.getName() + "_"); - endNode.setNodeType("linkis.control.empty"); - Map jobContentMap = new java.util.HashMap(); - /*jobContentMap.put("script", "");*/ - endNode.setJobContent(jobContentMap); - if (endNodeList.size() > 0) { - endNodeList.forEach(tmpNode -> endNode.addDependency(tmpNode.getName())); - } - LinkisAzkabanSchedulerNode azkabanSchedulerNode = new LinkisAzkabanSchedulerNode(); - azkabanSchedulerNode.setDssNode(endNode); - flow.getSchedulerNodes().add((azkabanSchedulerNode)); - return flow; - } - - private List getFlowEndJobList(SchedulerFlow flow) { - List res = new ArrayList<>(); - for (SchedulerNode job : flow.getSchedulerNodes()) { - int flag = 0; - for (SchedulerEdge link : flow.getSchedulerEdges()) { - if (job.getId().equals(link.getDssEdge().getSource())) { - flag = 1; - } - } - if (flag == 0) { - res.add(job); - } - } - return res; - } - - @Override - public int getOrder() { - return 1; - } -} diff --git a/dss-azkaban-scheduler-appjoint/src/main/java/com/webank/wedatasphere/dss/appjoint/scheduler/azkaban/tuning/LinkisShareNodeFlowTuning.java b/dss-azkaban-scheduler-appjoint/src/main/java/com/webank/wedatasphere/dss/appjoint/scheduler/azkaban/tuning/LinkisShareNodeFlowTuning.java deleted file mode 100644 index 2e01c25821..0000000000 --- a/dss-azkaban-scheduler-appjoint/src/main/java/com/webank/wedatasphere/dss/appjoint/scheduler/azkaban/tuning/LinkisShareNodeFlowTuning.java +++ /dev/null @@ -1,34 +0,0 @@ -package com.webank.wedatasphere.dss.appjoint.scheduler.azkaban.tuning; - - -import com.webank.wedatasphere.dss.appjoint.scheduler.azkaban.entity.AzkabanSchedulerFlow; -import com.webank.wedatasphere.dss.appjoint.scheduler.azkaban.entity.LinkisAzkabanShareNode; -import com.webank.wedatasphere.dss.appjoint.scheduler.entity.SchedulerFlow; -import com.webank.wedatasphere.dss.appjoint.scheduler.entity.ShareNode; -import com.webank.wedatasphere.dss.appjoint.scheduler.tuning.AbstractShareNodeFlowTuning; -import com.webank.wedatasphere.dss.appjoint.scheduler.tuning.NodeTuning; - - - -public class LinkisShareNodeFlowTuning extends AbstractShareNodeFlowTuning { - - @Override - public Boolean ifFlowCanTuning(SchedulerFlow schedulerFlow) { - return schedulerFlow instanceof AzkabanSchedulerFlow; - } - - @Override - public int getOrder() { - return 2; - } - - @Override - protected ShareNode createShareNode() { - return new LinkisAzkabanShareNode(); - } - - @Override - public NodeTuning[] getNodeTunings() { - return new NodeTuning[0]; - } -} diff --git a/dss-azkaban-scheduler-appjoint/src/main/java/com/webank/wedatasphere/dss/appjoint/scheduler/azkaban/util/AzkabanUtils.java b/dss-azkaban-scheduler-appjoint/src/main/java/com/webank/wedatasphere/dss/appjoint/scheduler/azkaban/util/AzkabanUtils.java deleted file mode 100644 index 3fb7801b35..0000000000 --- a/dss-azkaban-scheduler-appjoint/src/main/java/com/webank/wedatasphere/dss/appjoint/scheduler/azkaban/util/AzkabanUtils.java +++ /dev/null @@ -1,34 +0,0 @@ -package com.webank.wedatasphere.dss.appjoint.scheduler.azkaban.util; - -import com.google.gson.Gson; -import org.apache.commons.lang.StringUtils; - -import java.io.IOException; -import java.util.Map; - -/** - * Created by cooperyang on 2019/9/18. - */ -public class AzkabanUtils { - public static String handleAzkabanEntity(String entityString) throws IOException { - Gson gson = new Gson(); - Object object = gson.fromJson(entityString, Object.class); - String status = null; - String message = null; - if (object instanceof Map) { - Map map = (Map) object; - if (map.get("status") != null) { - status = map.get("status").toString(); - } - if (StringUtils.isNotEmpty(status)) { - if (null != map.get("message")) { - message = map.get("message").toString(); - } - } - if ("error".equalsIgnoreCase(status)) { - return message; - } - } - return "success"; - } -} diff --git a/dss-azkaban-scheduler-appjoint/src/main/java/com/webank/wedatasphere/dss/appjoint/scheduler/azkaban/util/AzkabanUtilsScala.scala b/dss-azkaban-scheduler-appjoint/src/main/java/com/webank/wedatasphere/dss/appjoint/scheduler/azkaban/util/AzkabanUtilsScala.scala deleted file mode 100644 index 579f133b4f..0000000000 --- a/dss-azkaban-scheduler-appjoint/src/main/java/com/webank/wedatasphere/dss/appjoint/scheduler/azkaban/util/AzkabanUtilsScala.scala +++ /dev/null @@ -1,17 +0,0 @@ -package com.webank.wedatasphere.dss.appjoint.scheduler.azkaban.util - -import java.util -import scala.collection.JavaConversions._ -import scala.collection.JavaConverters._ - -/** - * Created by cooperyang on 2019/9/27. - */ -object AzkabanUtilsScala { - - def getRepeatNodeName(nodeList:java.util.List[String]):util.List[String]={ - val res = nodeList.map(x=>(x,1)).groupBy(x => x._1).map(x =>(x._1,x._2.size)).filter(x=>x._2 >1).map(x=>x._1).toList - res.asJava - } - -} diff --git a/dss-azkaban-scheduler-appjoint/src/main/resources/appjoint.properties b/dss-azkaban-scheduler-appjoint/src/main/resources/appjoint.properties deleted file mode 100644 index e5983a4595..0000000000 --- a/dss-azkaban-scheduler-appjoint/src/main/resources/appjoint.properties +++ /dev/null @@ -1,21 +0,0 @@ -# -# Copyright 2019 WeBank -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# - - - - - diff --git a/dss-azkaban-scheduler-appjoint/src/main/resources/log4j.properties b/dss-azkaban-scheduler-appjoint/src/main/resources/log4j.properties deleted file mode 100644 index 0807e60877..0000000000 --- a/dss-azkaban-scheduler-appjoint/src/main/resources/log4j.properties +++ /dev/null @@ -1,37 +0,0 @@ -# -# Copyright 2019 WeBank -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# - -### set log levels ### - -log4j.rootCategory=INFO,console - -log4j.appender.console=org.apache.log4j.ConsoleAppender -log4j.appender.console.Threshold=INFO -log4j.appender.console.layout=org.apache.log4j.PatternLayout -#log4j.appender.console.layout.ConversionPattern= %d{ISO8601} %-5p (%t) [%F:%M(%L)] - %m%n -log4j.appender.console.layout.ConversionPattern= %d{ISO8601} %-5p (%t) %p %c{1} - %m%n - - -log4j.appender.com.webank.bdp.ide.core=org.apache.log4j.DailyRollingFileAppender -log4j.appender.com.webank.bdp.ide.core.Threshold=INFO -log4j.additivity.com.webank.bdp.ide.core=false -log4j.appender.com.webank.bdp.ide.core.layout=org.apache.log4j.PatternLayout -log4j.appender.com.webank.bdp.ide.core.Append=true -log4j.appender.com.webank.bdp.ide.core.File=logs/linkis.log -log4j.appender.com.webank.bdp.ide.core.layout.ConversionPattern= %d{ISO8601} %-5p (%t) [%F:%M(%L)] - %m%n - -log4j.logger.org.springframework=INFO diff --git a/dss-azkaban-scheduler-appjoint/src/main/resources/log4j2.xml b/dss-azkaban-scheduler-appjoint/src/main/resources/log4j2.xml deleted file mode 100644 index 3923cd9f39..0000000000 --- a/dss-azkaban-scheduler-appjoint/src/main/resources/log4j2.xml +++ /dev/null @@ -1,39 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - diff --git a/dss-common/src/main/java/com/webank/wedatasphere/dss/common/entity/Resource.java b/dss-common/src/main/java/com/webank/wedatasphere/dss/common/entity/Resource.java deleted file mode 100644 index d4066e401a..0000000000 --- a/dss-common/src/main/java/com/webank/wedatasphere/dss/common/entity/Resource.java +++ /dev/null @@ -1,62 +0,0 @@ -/* - * Copyright 2019 WeBank - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package com.webank.wedatasphere.dss.common.entity; - -import java.io.Serializable; - -/** - * Created by enjoyyin on 2019/9/24. - */ -public class Resource implements Serializable { - private String fileName; - private String resourceId; - private String version; - - public String getFileName() { - return fileName; - } - - public void setFileName(String fileName) { - this.fileName = fileName; - } - - public String getResourceId() { - return resourceId; - } - - public void setResourceId(String resourceId) { - this.resourceId = resourceId; - } - - public String getVersion() { - return version; - } - - public void setVersion(String version) { - this.version = version; - } - -/* @Override - public String toString() { - return "{" + - "fileName='" + fileName + '\'' + - ", resourceId='" + resourceId + '\'' + - ", version='" + version + '\'' + - '}'; - }*/ -} diff --git a/dss-common/src/main/java/com/webank/wedatasphere/dss/common/entity/flow/DSSFlow.java b/dss-common/src/main/java/com/webank/wedatasphere/dss/common/entity/flow/DSSFlow.java deleted file mode 100644 index ff768f791e..0000000000 --- a/dss-common/src/main/java/com/webank/wedatasphere/dss/common/entity/flow/DSSFlow.java +++ /dev/null @@ -1,194 +0,0 @@ -/* - * Copyright 2019 WeBank - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package com.webank.wedatasphere.dss.common.entity.flow; - -import java.util.Date; -import java.util.List; -import java.util.stream.Collectors; - -/** - * Created by enjoyyin on 2019/5/14. - */ -public class DSSFlow implements Flow { - private Long id; - private String name; - private Boolean state; //0,1代表发布过和未发布过 - private String source; - private String description; - private Date createTime; - private Long creatorID; - private Boolean isRootFlow; - private Integer rank; - private Long projectID; - private Boolean hasSaved;//0disable 1 enable 0表示工作流从来没存过，发布的时候忽略 - private String uses; - - private List versions; //为了前台不做修改，还是使用versions 而不使用flowVersions的变量名 - private List children; - private String flowType; - - private DSSFlowVersion latestVersion; - - - public Integer getRank() { - return rank; - } - - public void setRank(Integer rank) { - this.rank = rank; - } - - @Override - public Long getId() { - return id; - } - - @Override - public void setId(Long id) { - this.id = id; - } - - @Override - public String getName() { - return name; - } - - @Override - public void setName(String name) { - this.name = name; - } - - - @Override - public String getDescription() { - return description; - } - - @Override - public void setDescription(String description) { - this.description = description; - } - - @Override - public void addFlowVersion(FlowVersion flowVersion) { - this.versions.add((DSSFlowVersion) flowVersion); - } - - @Override - public List getChildren() { - return children; - } - - @Override - public void setChildren(List children) { - this.children = children.stream().map(f ->(DSSFlow)f).collect(Collectors.toList()); - } - - @Override - public List getFlowVersions() { - return this.versions; - } - - @Override - public void setFlowVersions(List flowVersions) { - this.versions = flowVersions.stream().map(f ->(DSSFlowVersion)f).collect(Collectors.toList()); - } - - @Override - public String getFlowType() { - return flowType; - } - - @Override - public void setFlowType(String flowType) { - this.flowType = flowType; - } - - @Override - public Boolean getRootFlow() { - return isRootFlow; - } - - public void setRootFlow(Boolean rootFlow) { - isRootFlow = rootFlow; - } - - public Boolean getState() { - return state; - } - - public void setState(Boolean state) { - this.state = state; - } - - public String getSource() { - return source; - } - - public void setSource(String source) { - this.source = source; - } - - public Date getCreateTime() { - return createTime; - } - - public void setCreateTime(Date createTime) { - this.createTime = createTime; - } - - public Long getCreatorID() { - return creatorID; - } - - public void setCreatorID(Long creatorID) { - this.creatorID = creatorID; - } - - public Long getProjectID() { - return projectID; - } - - public void setProjectID(Long projectID) { - this.projectID = projectID; - } - - public Boolean getHasSaved() { - return hasSaved; - } - - public void setHasSaved(Boolean hasSaved) { - this.hasSaved = hasSaved; - } - - public DSSFlowVersion getLatestVersion() { - return latestVersion; - } - - public void setLatestVersion(DSSFlowVersion latestVersion) { - this.latestVersion = latestVersion; - } - - public String getUses() { - return uses; - } - - public void setUses(String uses) { - this.uses = uses; - } -} diff --git a/dss-common/src/main/java/com/webank/wedatasphere/dss/common/entity/flow/DSSFlowPublishHistory.java b/dss-common/src/main/java/com/webank/wedatasphere/dss/common/entity/flow/DSSFlowPublishHistory.java deleted file mode 100644 index 7fe3e9ad42..0000000000 --- a/dss-common/src/main/java/com/webank/wedatasphere/dss/common/entity/flow/DSSFlowPublishHistory.java +++ /dev/null @@ -1,24 +0,0 @@ -/* - * Copyright 2019 WeBank - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package com.webank.wedatasphere.dss.common.entity.flow; - -/** - * Created by enjoyyin on 2019/9/19. - */ -public class DSSFlowPublishHistory { -} diff --git a/dss-common/src/main/java/com/webank/wedatasphere/dss/common/entity/flow/DSSFlowVersion.java b/dss-common/src/main/java/com/webank/wedatasphere/dss/common/entity/flow/DSSFlowVersion.java deleted file mode 100644 index ce8559097a..0000000000 --- a/dss-common/src/main/java/com/webank/wedatasphere/dss/common/entity/flow/DSSFlowVersion.java +++ /dev/null @@ -1,163 +0,0 @@ -/* - * Copyright 2019 WeBank - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package com.webank.wedatasphere.dss.common.entity.flow; - -import java.util.Date; - -/** - * Created by enjoyyin on 2019/9/19. - */ -public class DSSFlowVersion implements FlowVersion, Comparable { - private Long id; - private Long flowID; - private String source; - private String jsonPath; - private String comment; - private Date updateTime; - private Long updatorID; - private String version; - private DSSFlowPublishHistory publishHistory; - private String json; - private String updator; - private Boolean isNotPublished; //true 未发过版，false已经过版 - private Long projectVersionID; - private Long oldFlowID; - - public Long getOldFlowID() { - return oldFlowID; - } - - public void setOldFlowID(Long oldFlowID) { - this.oldFlowID = oldFlowID; - } - - public Long getProjectVersionID() { - return projectVersionID; - } - - public void setProjectVersionID(Long projectVersionID) { - this.projectVersionID = projectVersionID; - } - - public Boolean getNotPublished() { - return isNotPublished; - } - - public void setNotPublished(Boolean notPublished) { - isNotPublished = notPublished; - } - - public String getUpdator() { - return updator; - } - - public void setUpdator(String updator) { - this.updator = updator; - } - - public String getJson() { - return json; - } - - public void setJson(String json) { - this.json = json; - } - - @Override - public String getVersion() { - return version; - } - - @Override - public void setVersion(String version) { - this.version = version; - } - - public Long getId() { - return id; - } - - public void setId(Long id) { - this.id = id; - } - - public Long getFlowID() { - return flowID; - } - - public void setFlowID(Long flowID) { - this.flowID = flowID; - } - - public String getSource() { - return source; - } - - public void setSource(String source) { - this.source = source; - } - - public String getJsonPath() { - return jsonPath; - } - - public void setJsonPath(String jsonPath) { - this.jsonPath = jsonPath; - } - - @Override - public String getComment() { - return comment; - } - - @Override - public void setComment(String comment) { - this.comment = comment; - } - - public Date getUpdateTime() { - return updateTime; - } - - public void setUpdateTime(Date updateTime) { - this.updateTime = updateTime; - } - - public Long getUpdatorID() { - return updatorID; - } - - public void setUpdatorID(Long updatorID) { - this.updatorID = updatorID; - } - - public DSSFlowPublishHistory getPublishHistory() { - return publishHistory; - } - - public void setPublishHistory(DSSFlowPublishHistory publishHistory) { - this.publishHistory = publishHistory; - } - - @Override - public int compareTo(DSSFlowVersion o) { - Integer v1 = Integer.valueOf(this.version.substring(1, version.length())); - Integer v2 = Integer.valueOf(o.version.substring(1,o.version.length())); - return v2 - v1; - } -} diff --git a/dss-common/src/main/java/com/webank/wedatasphere/dss/common/entity/flow/DSSJSONFlow.java b/dss-common/src/main/java/com/webank/wedatasphere/dss/common/entity/flow/DSSJSONFlow.java deleted file mode 100644 index ae9b986318..0000000000 --- a/dss-common/src/main/java/com/webank/wedatasphere/dss/common/entity/flow/DSSJSONFlow.java +++ /dev/null @@ -1,48 +0,0 @@ -/* - * Copyright 2019 WeBank - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package com.webank.wedatasphere.dss.common.entity.flow; - -import java.util.List; -import java.util.stream.Collectors; - -/** - * Created by enjoyyin on 2019/5/14. - */ -public class DSSJSONFlow extends DSSFlow { - private String json; - - private List children; - - public String getJson() { - return json; - } - - public void setJson(String json) { - this.json = json; - } - - @Override - public void setChildren(List children) { - this.children = children.stream().map(f ->(DSSJSONFlow)f).collect(Collectors.toList()); - } - - @Override - public List getChildren() { - return children; - } -} diff --git a/dss-common/src/main/java/com/webank/wedatasphere/dss/common/entity/flow/FileFlowVersion.java b/dss-common/src/main/java/com/webank/wedatasphere/dss/common/entity/flow/FileFlowVersion.java deleted file mode 100644 index a249d030ae..0000000000 --- a/dss-common/src/main/java/com/webank/wedatasphere/dss/common/entity/flow/FileFlowVersion.java +++ /dev/null @@ -1,27 +0,0 @@ -/* - * Copyright 2019 WeBank - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package com.webank.wedatasphere.dss.common.entity.flow; - -/** - * Created by enjoyyin on 2019/5/14. - */ -public interface FileFlowVersion extends FlowVersion { - Object getFlowFile(); - - void setFlowFile(Object flowFile); -} diff --git a/dss-common/src/main/java/com/webank/wedatasphere/dss/common/entity/flow/Flow.java b/dss-common/src/main/java/com/webank/wedatasphere/dss/common/entity/flow/Flow.java deleted file mode 100644 index d986c025d9..0000000000 --- a/dss-common/src/main/java/com/webank/wedatasphere/dss/common/entity/flow/Flow.java +++ /dev/null @@ -1,53 +0,0 @@ -/* - * Copyright 2019 WeBank - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package com.webank.wedatasphere.dss.common.entity.flow; - -import java.util.List; - -/** - * Created by enjoyyin on 2019/5/14. - */ -public interface Flow { - Long getId(); - - void setId(Long id); - - String getName(); - - void setName(String name); - - String getFlowType(); - - void setFlowType(String flowType); - - String getDescription(); - - void setDescription(String description); - - Boolean getRootFlow(); - - List getChildren(); - - void setChildren(List children); - - List getFlowVersions(); - - void setFlowVersions(List flowVersions); - - void addFlowVersion(FlowVersion flowVersion); -} diff --git a/dss-common/src/main/java/com/webank/wedatasphere/dss/common/entity/flow/FlowVersion.java b/dss-common/src/main/java/com/webank/wedatasphere/dss/common/entity/flow/FlowVersion.java deleted file mode 100644 index 747bab71ce..0000000000 --- a/dss-common/src/main/java/com/webank/wedatasphere/dss/common/entity/flow/FlowVersion.java +++ /dev/null @@ -1,31 +0,0 @@ -/* - * Copyright 2019 WeBank - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package com.webank.wedatasphere.dss.common.entity.flow; - -/** - * Created by enjoyyin on 2019/5/14. - */ -public interface FlowVersion { - String getVersion(); - - void setVersion(String version); - - String getComment(); - - void setComment(String comment); -} diff --git a/dss-common/src/main/java/com/webank/wedatasphere/dss/common/entity/flow/FlowVersionForNodes.java b/dss-common/src/main/java/com/webank/wedatasphere/dss/common/entity/flow/FlowVersionForNodes.java deleted file mode 100644 index f61c596ea7..0000000000 --- a/dss-common/src/main/java/com/webank/wedatasphere/dss/common/entity/flow/FlowVersionForNodes.java +++ /dev/null @@ -1,35 +0,0 @@ -/* - * Copyright 2019 WeBank - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package com.webank.wedatasphere.dss.common.entity.flow; - -import com.webank.wedatasphere.dss.common.entity.node.Node; - -import java.util.List; - -/** - * Created by enjoyyin on 2019/5/14. - */ -public interface FlowVersionForNodes { - List getNodes(); - - void setNodes(List nodes); - - void addNode(Node node); - - void removeNode(Node node); -} diff --git a/dss-common/src/main/java/com/webank/wedatasphere/dss/common/entity/flow/JSONFlowVersion.java b/dss-common/src/main/java/com/webank/wedatasphere/dss/common/entity/flow/JSONFlowVersion.java deleted file mode 100644 index 22be80d81b..0000000000 --- a/dss-common/src/main/java/com/webank/wedatasphere/dss/common/entity/flow/JSONFlowVersion.java +++ /dev/null @@ -1,27 +0,0 @@ -/* - * Copyright 2019 WeBank - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package com.webank.wedatasphere.dss.common.entity.flow; - -/** - * Created by enjoyyin on 2019/5/14. - */ -public interface JSONFlowVersion extends FlowVersion { - Object getFlowJSON(); - - void setFlowJSON(Object flowJSON); -} diff --git a/dss-common/src/main/java/com/webank/wedatasphere/dss/common/entity/node/DSSEdge.java b/dss-common/src/main/java/com/webank/wedatasphere/dss/common/entity/node/DSSEdge.java deleted file mode 100644 index c4b30593af..0000000000 --- a/dss-common/src/main/java/com/webank/wedatasphere/dss/common/entity/node/DSSEdge.java +++ /dev/null @@ -1,40 +0,0 @@ -/* - * Copyright 2019 WeBank - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package com.webank.wedatasphere.dss.common.entity.node; - -/** - * Created by enjoyyin on 2019/5/14. - */ -public interface DSSEdge { - - String getSource(); - - void setSource(); - - String getTarget(); - - void setTarget(String target); - - String getSourceLocation(); - - void setSourceLocation(String sourceLocation); - - String getTargetLocation(); - - void setTargetLocation(String targetLocation); -} diff --git a/dss-common/src/main/java/com/webank/wedatasphere/dss/common/entity/node/DSSEdgeDefault.java b/dss-common/src/main/java/com/webank/wedatasphere/dss/common/entity/node/DSSEdgeDefault.java deleted file mode 100644 index 3e6bc8b139..0000000000 --- a/dss-common/src/main/java/com/webank/wedatasphere/dss/common/entity/node/DSSEdgeDefault.java +++ /dev/null @@ -1,78 +0,0 @@ -/* - * Copyright 2019 WeBank - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package com.webank.wedatasphere.dss.common.entity.node; - -/** - * Created by enjoyyin on 2019/5/14. - */ -public class DSSEdgeDefault implements DSSEdge { - private String source; - private String target; - private String sourceLocation; - private String targetLocation; - - @Override - public String getSource() { - return source; - } - - @Override - public void setSource() { - this.source = source; - } - - @Override - public String getTarget() { - return target; - } - - @Override - public void setTarget(String target) { - this.target = target; - } - - @Override - public String getSourceLocation() { - return sourceLocation; - } - - @Override - public void setSourceLocation(String sourceLocation) { - this.sourceLocation = sourceLocation; - } - - @Override - public String getTargetLocation() { - return targetLocation; - } - - @Override - public void setTargetLocation(String targetLocation) { - this.targetLocation = targetLocation; - } - - @Override - public String toString() { - return "DSSEdge{" + - "source='" + source + '\'' + - ", target='" + target + '\'' + - ", sourceLocation='" + sourceLocation + '\'' + - ", targetLocation='" + targetLocation + '\'' + - '}'; - } -} diff --git a/dss-common/src/main/java/com/webank/wedatasphere/dss/common/entity/node/DSSNode.java b/dss-common/src/main/java/com/webank/wedatasphere/dss/common/entity/node/DSSNode.java deleted file mode 100644 index 8f6575c836..0000000000 --- a/dss-common/src/main/java/com/webank/wedatasphere/dss/common/entity/node/DSSNode.java +++ /dev/null @@ -1,49 +0,0 @@ -/* - * Copyright 2019 WeBank - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package com.webank.wedatasphere.dss.common.entity.node; - -import com.webank.wedatasphere.dss.common.entity.Resource; - -import java.util.List; -import java.util.Map; - -/** - * Created by enjoyyin on 2019/5/14. - */ -public interface DSSNode extends Node { - - Layout getLayout(); - - void setLayout(Layout layout); - - Map getParams(); - - void setParams(Map params); - - List getResources(); - - void setResources(List resources); - - Map getJobContent(); - - void setJobContent(Map jobContent); - - String getUserProxy(); - - void setUserProxy(String userProxy); -} diff --git a/dss-common/src/main/java/com/webank/wedatasphere/dss/common/entity/node/DSSNodeDefault.java b/dss-common/src/main/java/com/webank/wedatasphere/dss/common/entity/node/DSSNodeDefault.java deleted file mode 100644 index 2d416dfb02..0000000000 --- a/dss-common/src/main/java/com/webank/wedatasphere/dss/common/entity/node/DSSNodeDefault.java +++ /dev/null @@ -1,162 +0,0 @@ -/* - * Copyright 2019 WeBank - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package com.webank.wedatasphere.dss.common.entity.node; - -import com.webank.wedatasphere.dss.common.entity.Resource; - -import java.util.ArrayList; -import java.util.List; -import java.util.Map; - -/** - * Created by enjoyyin on 2019/5/14. - */ -public class DSSNodeDefault implements DSSNode { - private Layout layout; - private String id; - private String jobType; - private String lastUpdateTime; - private Map params; - private List resources; - private String title; - private String desc; - private String createTime; - private String userProxy; - /** - * dependencys 是该Node的依赖节点 - */ - private List dependencys = new ArrayList(); - /** - * jobContent 是node的内容 - */ - private Map jobContent; - - - @Override - public Layout getLayout() { - return layout; - } - - - @Override - public void setLayout(Layout layout) { - this.layout = layout; - } - - @Override - public Map getParams() { - return params; - } - - @Override - public void setParams(Map params) { - this.params = params; - } - - @Override - public List getResources() { - return resources; - } - - @Override - public void setResources(List resources) { - this.resources = resources; - } - - @Override - public Map getJobContent() { - return jobContent; - } - - @Override - public void setJobContent(Map jobContent) { - this.jobContent = jobContent; - } - - @Override - public String getUserProxy() { - return userProxy; - } - - @Override - public void setUserProxy(String userProxy) { - this.userProxy = userProxy; - } - - @Override - public String getId() { - return id; - } - - @Override - public void setId(String id) { - this.id = id; - } - - @Override - public String getNodeType() { - return jobType; - } - - @Override - public void setNodeType(String nodeType) { - this.jobType = nodeType; - } - - @Override - public String getName() { - return title; - } - - @Override - public void setName(String name) { - this.title = name; - } - - @Override - public void addDependency(String nodeName) { - dependencys.add(nodeName); - } - - @Override - public void setDependency(List dependency) { - this.dependencys = dependency; - } - - @Override - public void removeDependency(String nodeName) { - dependencys.remove(nodeName); - } - - @Override - public List getDependencys() { - return dependencys; - } - - @Override - public String toString() { - return "dwsNode{" + - "layout=" + layout + - ", id='" + id + '\'' + - ", jobType='" + jobType + '\'' + - ", lastUpdateTime=" + lastUpdateTime + - ", params=" + params + - ", resources=" + resources + - '}'; - } -} diff --git a/dss-common/src/main/java/com/webank/wedatasphere/dss/common/entity/node/Layout.java b/dss-common/src/main/java/com/webank/wedatasphere/dss/common/entity/node/Layout.java deleted file mode 100644 index d9c4d3b14f..0000000000 --- a/dss-common/src/main/java/com/webank/wedatasphere/dss/common/entity/node/Layout.java +++ /dev/null @@ -1,70 +0,0 @@ -/* - * Copyright 2019 WeBank - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package com.webank.wedatasphere.dss.common.entity.node; - -/** - * Created by enjoyyin on 2019/11/12. - */ -public class Layout { - private float x; - private float y; - private float width; - private float height; - - public float getX() { - return x; - } - - public void setX(float x) { - this.x = x; - } - - public float getY() { - return y; - } - - public void setY(float y) { - this.y = y; - } - - public float getWidth() { - return width; - } - - public void setWidth(float width) { - this.width = width; - } - - public float getHeight() { - return height; - } - - public void setHeight(float height) { - this.height = height; - } - - @Override - public String toString() { - return "Layout{" + - "x=" + x + - ", y=" + y + - ", width=" + width + - ", height=" + height + - '}'; - } -} diff --git a/dss-common/src/main/java/com/webank/wedatasphere/dss/common/entity/node/Node.java b/dss-common/src/main/java/com/webank/wedatasphere/dss/common/entity/node/Node.java deleted file mode 100644 index a802d75157..0000000000 --- a/dss-common/src/main/java/com/webank/wedatasphere/dss/common/entity/node/Node.java +++ /dev/null @@ -1,46 +0,0 @@ -/* - * Copyright 2019 WeBank - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package com.webank.wedatasphere.dss.common.entity.node; - - -import java.util.List; - -/** - * Created by enjoyyin on 2019/5/14. - */ -public interface Node { - String getId(); - - void setId(String id); - - String getNodeType(); - - void setNodeType(String nodeType); - - String getName(); - - void setName(String name); - - void addDependency(String nodeName); - - void setDependency(List dependency); - - void removeDependency(String nodeName); - - List getDependencys(); -} diff --git a/dss-common/src/main/java/com/webank/wedatasphere/dss/common/entity/project/DSSJSONProject.java b/dss-common/src/main/java/com/webank/wedatasphere/dss/common/entity/project/DSSJSONProject.java deleted file mode 100644 index ffd9026fc3..0000000000 --- a/dss-common/src/main/java/com/webank/wedatasphere/dss/common/entity/project/DSSJSONProject.java +++ /dev/null @@ -1,41 +0,0 @@ -/* - * Copyright 2019 WeBank - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package com.webank.wedatasphere.dss.common.entity.project; - -import com.webank.wedatasphere.dss.common.entity.flow.DSSFlow; -import com.webank.wedatasphere.dss.common.entity.flow.DSSJSONFlow; - -import java.util.List; -import java.util.stream.Collectors; - -/** - * Created by allenlliu on 2019/9/16. - */ -public class DSSJSONProject extends DSSProject { - private List flows; - - @Override - public List getFlows() { - return this.flows; - } - - @Override - public void setFlows(List flows) { - this.flows = flows.stream().map(f ->(DSSJSONFlow)f).collect(Collectors.toList()); - } -} diff --git a/dss-common/src/main/java/com/webank/wedatasphere/dss/common/entity/project/DSSProject.java b/dss-common/src/main/java/com/webank/wedatasphere/dss/common/entity/project/DSSProject.java deleted file mode 100644 index 4454362d8e..0000000000 --- a/dss-common/src/main/java/com/webank/wedatasphere/dss/common/entity/project/DSSProject.java +++ /dev/null @@ -1,294 +0,0 @@ -/* - * Copyright 2019 WeBank - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package com.webank.wedatasphere.dss.common.entity.project; - -import com.webank.wedatasphere.dss.common.entity.Resource; -import com.webank.wedatasphere.dss.common.entity.flow.DSSFlow; - -import java.util.Date; -import java.util.List; -import java.util.stream.Collectors; - -/** - * Created by enjoyyin on 2019/9/16. - */ -public class DSSProject implements Project { - - private Long id; - private String name; - private String description; - private String source; - private Long userID; - private Date createTime; - private Long createBy; - private Date updateTime; - private Long updateBy; - private Long orgID; - private Boolean visibility; - private Boolean isTransfer; - private Boolean isArchive; - private Long initialOrgID; - private String pic; - private Long starNum; - private String product; - private Integer applicationArea; - private String business; - private Long workspaceId; - - private DSSProjectVersion latestVersion; - private Boolean isNotPublish; - private String userName; - - private String projectGroup; - private List projectVersions; - private List flows; - private List projectResources; - - public List getProjectResources() { - return projectResources; - } - - public void setProjectResources(List projectResources) { - this.projectResources = projectResources; - } - - public List getFlows() { - return flows; - } - - public void setFlows(List flows) { - this.flows = flows.stream().map(f -> (DSSFlow) f).collect(Collectors.toList()); - } - - public String getUserName() { - return userName; - } - - public void setUserName(String userName) { - this.userName = userName; - } - - public Boolean getNotPublish() { - return isNotPublish; - } - - public void setNotPublish(Boolean notPublish) { - isNotPublish = notPublish; - } - - public Long getOrgID() { - return orgID; - } - - public void setOrgID(Long orgID) { - this.orgID = orgID; - } - - public Boolean getVisibility() { - return visibility; - } - - public void setVisibility(Boolean visibility) { - this.visibility = visibility; - } - - public Boolean getTransfer() { - return isTransfer; - } - - public void setTransfer(Boolean transfer) { - isTransfer = transfer; - } - - public Long getInitialOrgID() { - return initialOrgID; - } - - public void setInitialOrgID(Long initialOrgID) { - this.initialOrgID = initialOrgID; - } - - public DSSProjectVersion getLatestVersion() { - return latestVersion; - } - - public void setLatestVersion(DSSProjectVersion latestVersion) { - this.latestVersion = latestVersion; - } - - public String getSource() { - return source; - } - - public void setSource(String source) { - this.source = source; - } - - public Date getCreateTime() { - return createTime; - } - - public void setCreateTime(Date createTime) { - this.createTime = createTime; - } - - - @Override - public String getDescription() { - return description; - } - - @Override - public void setDescription(String description) { - this.description = description; - } - - @Override - public String getPorjectGroup() { - return projectGroup; - } - - @Override - public void setProjectGroup(String projectGroup) { - this.projectGroup = projectGroup; - } - - @Override - public List getProjectVersions() { - return projectVersions; - } - - @Override - public void setProjectVersions(List projectVersions) { - this.projectVersions = projectVersions.stream().map(f -> (DSSProjectVersion) f).collect(Collectors.toList()); - } - - - @Override - public void addProjectVersion(ProjectVersion projectVersion) { - this.projectVersions.add((DSSProjectVersion) projectVersion); - } - - @Override - public Long getId() { - return id; - } - - @Override - public void setId(Long id) { - this.id = id; - } - - @Override - public String getName() { - return name; - } - - @Override - public void setName(String name) { - this.name = name; - } - - public Long getUserID() { - return userID; - } - - public void setUserID(Long userID) { - this.userID = userID; - } - - public Long getCreateBy() { - return createBy; - } - - public void setCreateBy(Long createBy) { - this.createBy = createBy; - } - - public Date getUpdateTime() { - return updateTime; - } - - public void setUpdateTime(Date updateTime) { - this.updateTime = updateTime; - } - - public Long getUpdateBy() { - return updateBy; - } - - public void setUpdateBy(Long updateBy) { - this.updateBy = updateBy; - } - - public Boolean getArchive() { - return isArchive; - } - - public void setArchive(Boolean archive) { - isArchive = archive; - } - - public String getPic() { - return pic; - } - - public void setPic(String pic) { - this.pic = pic; - } - - public Long getStarNum() { - return starNum; - } - - public void setStarNum(Long starNum) { - this.starNum = starNum; - } - - public String getProduct() { - return product; - } - - public void setProduct(String product) { - this.product = product; - } - - public Integer getApplicationArea() { - return applicationArea; - } - - public void setApplicationArea(Integer applicationArea) { - this.applicationArea = applicationArea; - } - - public String getBusiness() { - return business; - } - - public void setBusiness(String business) { - this.business = business; - } - - public Long getWorkspaceId() { - return workspaceId; - } - - public void setWorkspaceId(Long workspaceId) { - this.workspaceId = workspaceId; - } -} diff --git a/dss-common/src/main/java/com/webank/wedatasphere/dss/common/entity/project/DSSProjectPublishHistory.java b/dss-common/src/main/java/com/webank/wedatasphere/dss/common/entity/project/DSSProjectPublishHistory.java deleted file mode 100644 index dca93dd487..0000000000 --- a/dss-common/src/main/java/com/webank/wedatasphere/dss/common/entity/project/DSSProjectPublishHistory.java +++ /dev/null @@ -1,116 +0,0 @@ -/* - * Copyright 2019 WeBank - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package com.webank.wedatasphere.dss.common.entity.project; - -import java.util.Date; - -/** - * Created by enjoyyin on 2019/5/14. - */ -public class DSSProjectPublishHistory { - private Long id; - private Long projectVersionID; - private Date createTime; - private Long createID; - private String creator; - private Date updateTime; - private String comment; - private Integer state; - private String versionPath; - private Date expireTime; - - public Long getId() { - return id; - } - - public void setId(Long id) { - this.id = id; - } - - public Long getProjectVersionID() { - return projectVersionID; - } - - public void setProjectVersionID(Long projectVersionID) { - this.projectVersionID = projectVersionID; - } - - public Date getCreateTime() { - return createTime; - } - - public void setCreateTime(Date createTime) { - this.createTime = createTime; - } - - public Long getCreateID() { - return createID; - } - - public void setCreateID(Long createID) { - this.createID = createID; - } - - public String getCreator() { - return creator; - } - - public void setCreator(String creator) { - this.creator = creator; - } - - public Date getUpdateTime() { - return updateTime; - } - - public void setUpdateTime(Date updateTime) { - this.updateTime = updateTime; - } - - public String getComment() { - return comment; - } - - public void setComment(String comment) { - this.comment = comment; - } - - public Integer getState() { - return state; - } - - public void setState(Integer state) { - this.state = state; - } - - public String getVersionPath() { - return versionPath; - } - - public void setVersionPath(String versionPath) { - this.versionPath = versionPath; - } - - public Date getExpireTime() { - return expireTime; - } - - public void setExpireTime(Date expireTime) { - this.expireTime = expireTime; - } -} diff --git a/dss-common/src/main/java/com/webank/wedatasphere/dss/common/entity/project/DSSProjectVersion.java b/dss-common/src/main/java/com/webank/wedatasphere/dss/common/entity/project/DSSProjectVersion.java deleted file mode 100644 index 5731ecf8e3..0000000000 --- a/dss-common/src/main/java/com/webank/wedatasphere/dss/common/entity/project/DSSProjectVersion.java +++ /dev/null @@ -1,126 +0,0 @@ -/* - * Copyright 2019 WeBank - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package com.webank.wedatasphere.dss.common.entity.project; - -import java.util.Date; - -/** - * Created by enjoyyin on 2019/9/18. - */ -public class DSSProjectVersion implements ProjectVersion { - - private Long id; - private Long projectID; - private String version; - private String comment; - private Date updateTime; - private Long updatorID; - private Integer lock; - private String updator; - private Boolean isNotPublish; - private DSSProjectPublishHistory publishHistory; - - @Override - public String getVersion() { - return this.version; - } - - @Override - public void setVersion() { - this.version = version; - } - - @Override - public String getComment() { - return this.comment; - } - - @Override - public void setComment(String comment) { - this.comment = comment; - } - - - public Long getId() { - return id; - } - - public void setId(Long id) { - this.id = id; - } - - public Long getProjectID() { - return projectID; - } - - public void setProjectID(Long projectID) { - this.projectID = projectID; - } - - public void setVersion(String version) { - this.version = version; - } - - public Date getUpdateTime() { - return updateTime; - } - - public void setUpdateTime(Date updateTime) { - this.updateTime = updateTime; - } - - public Long getUpdatorID() { - return updatorID; - } - - public void setUpdatorID(Long updatorID) { - this.updatorID = updatorID; - } - - public Integer getLock() { - return lock; - } - - public void setLock(Integer lock) { - this.lock = lock; - } - - public String getUpdator() { - return updator; - } - - public void setUpdator(String updator) { - this.updator = updator; - } - - public Boolean getNotPublish() { - return isNotPublish; - } - - public void setNotPublish(Boolean notPublish) { - isNotPublish = notPublish; - } - - public DSSProjectPublishHistory getPublishHistory() { - return publishHistory; - } - - public void setPublishHistory(DSSProjectPublishHistory publishHistory) { - this.publishHistory = publishHistory; - } -} diff --git a/dss-common/src/main/java/com/webank/wedatasphere/dss/common/entity/project/Project.java b/dss-common/src/main/java/com/webank/wedatasphere/dss/common/entity/project/Project.java deleted file mode 100644 index 492d5d601a..0000000000 --- a/dss-common/src/main/java/com/webank/wedatasphere/dss/common/entity/project/Project.java +++ /dev/null @@ -1,47 +0,0 @@ -/* - * Copyright 2019 WeBank - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package com.webank.wedatasphere.dss.common.entity.project; - -import java.util.List; - -/** - * Created by enjoyyin on 2019/9/16. - */ -public interface Project { - Long getId(); - - void setId(Long id); - - String getName(); - - void setName(String name); - - String getDescription(); - - void setDescription(String description); - - String getPorjectGroup(); - - void setProjectGroup(String projectGroup); - - List getProjectVersions(); - - void setProjectVersions(List projectVersions); - - void addProjectVersion(ProjectVersion projectVersion); -} diff --git a/dss-common/src/main/java/com/webank/wedatasphere/dss/common/entity/project/ProjectVersion.java b/dss-common/src/main/java/com/webank/wedatasphere/dss/common/entity/project/ProjectVersion.java deleted file mode 100644 index b9f3c26345..0000000000 --- a/dss-common/src/main/java/com/webank/wedatasphere/dss/common/entity/project/ProjectVersion.java +++ /dev/null @@ -1,31 +0,0 @@ -/* - * Copyright 2019 WeBank - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package com.webank.wedatasphere.dss.common.entity.project; - -/** - * Created by enjoyyin on 2019/9/16. - */ -public interface ProjectVersion { - String getVersion(); - - void setVersion(); - - String getComment(); - - void setComment(String comment); -} diff --git a/dss-common/src/main/java/com/webank/wedatasphere/dss/common/entity/project/ProjectVersionForFlows.java b/dss-common/src/main/java/com/webank/wedatasphere/dss/common/entity/project/ProjectVersionForFlows.java deleted file mode 100644 index 773a22cc6a..0000000000 --- a/dss-common/src/main/java/com/webank/wedatasphere/dss/common/entity/project/ProjectVersionForFlows.java +++ /dev/null @@ -1,36 +0,0 @@ -/* - * Copyright 2019 WeBank - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package com.webank.wedatasphere.dss.common.entity.project; - -import com.webank.wedatasphere.dss.common.entity.flow.Flow; - -import java.util.List; - -/** - * Created by enjoyyin on 2019/9/26. - */ -public interface ProjectVersionForFlows extends ProjectVersion { - - List getFlows(); - - void setFlows(List flows); - - void addFlow(Flow flow); - - void removeFlow(Flow flow); -} diff --git a/dss-common/src/main/java/com/webank/wedatasphere/dss/common/exception/DSSErrorException.java b/dss-common/src/main/java/com/webank/wedatasphere/dss/common/exception/DSSErrorException.java deleted file mode 100644 index 2fa78edaf2..0000000000 --- a/dss-common/src/main/java/com/webank/wedatasphere/dss/common/exception/DSSErrorException.java +++ /dev/null @@ -1,34 +0,0 @@ -/* - * Copyright 2019 WeBank - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package com.webank.wedatasphere.dss.common.exception; - -import com.webank.wedatasphere.linkis.common.exception.ErrorException; - -/** - * Created by enjoyyin on 2019/11/6. - */ -public class DSSErrorException extends ErrorException { - - public DSSErrorException(int errCode, String desc) { - super(errCode, desc); - } - - public DSSErrorException(int errCode, String desc, String ip, int port, String serviceKind) { - super(errCode, desc, ip, port, serviceKind); - } -} diff --git a/dss-common/src/main/java/com/webank/wedatasphere/dss/common/exception/DSSRuntimeException.java b/dss-common/src/main/java/com/webank/wedatasphere/dss/common/exception/DSSRuntimeException.java deleted file mode 100644 index 057a2e3738..0000000000 --- a/dss-common/src/main/java/com/webank/wedatasphere/dss/common/exception/DSSRuntimeException.java +++ /dev/null @@ -1,30 +0,0 @@ -/* - * Copyright 2019 WeBank - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package com.webank.wedatasphere.dss.common.exception; - -import com.webank.wedatasphere.linkis.common.exception.WarnException; - -/** - * Created by enjoyyin on 2019/9/24. - */ -public class DSSRuntimeException extends WarnException { - - public DSSRuntimeException(String msg){ - super(100000, msg); - } -} diff --git a/dss-common/src/main/java/com/webank/wedatasphere/dss/common/exception/ThrowingConsumer.java b/dss-common/src/main/java/com/webank/wedatasphere/dss/common/exception/ThrowingConsumer.java deleted file mode 100644 index 54848a970b..0000000000 --- a/dss-common/src/main/java/com/webank/wedatasphere/dss/common/exception/ThrowingConsumer.java +++ /dev/null @@ -1,26 +0,0 @@ -/* - * Copyright 2019 WeBank - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package com.webank.wedatasphere.dss.common.exception; - -/** - * Created by enjoyyin on 2019/9/20. - */ -@FunctionalInterface -public interface ThrowingConsumer { - void accept(T t) throws E; -} diff --git a/dss-common/src/main/java/com/webank/wedatasphere/dss/common/exception/ThrowingFunction.java b/dss-common/src/main/java/com/webank/wedatasphere/dss/common/exception/ThrowingFunction.java deleted file mode 100644 index f261c11ee0..0000000000 --- a/dss-common/src/main/java/com/webank/wedatasphere/dss/common/exception/ThrowingFunction.java +++ /dev/null @@ -1,26 +0,0 @@ -/* - * Copyright 2019 WeBank - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package com.webank.wedatasphere.dss.common.exception; - -/** - * Created by enjoyyin on 2019/11/13. - */ -@FunctionalInterface -public interface ThrowingFunction { - R accept(T t) throws E; -} diff --git a/dss-common/src/main/java/com/webank/wedatasphere/dss/common/utils/DSSExceptionUtils.java b/dss-common/src/main/java/com/webank/wedatasphere/dss/common/utils/DSSExceptionUtils.java deleted file mode 100644 index 5009109333..0000000000 --- a/dss-common/src/main/java/com/webank/wedatasphere/dss/common/utils/DSSExceptionUtils.java +++ /dev/null @@ -1,59 +0,0 @@ -/* - * Copyright 2019 WeBank - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package com.webank.wedatasphere.dss.common.utils; - -import com.webank.wedatasphere.dss.common.exception.DSSRuntimeException; -import com.webank.wedatasphere.dss.common.exception.ThrowingConsumer; -import com.webank.wedatasphere.dss.common.exception.ThrowingFunction; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.util.function.Consumer; -import java.util.function.Function; - -/** - * Created by enjoyyin on 2019/9/24. - */ -public class DSSExceptionUtils { - - private static final Logger LOGGER = LoggerFactory.getLogger(DSSExceptionUtils.class); - - public static Consumer handling( - ThrowingConsumer throwingConsumer) { - return i -> { - try { - throwingConsumer.accept(i); - } catch (Exception e) { - LOGGER.error("execute failed,reason:",e); - throw new DSSRuntimeException(e.getMessage()); - } - }; - } - - public static Function map( - ThrowingFunction throwingFunction) { - return i -> { - try { - return throwingFunction.accept(i); - } catch (Exception e) { - LOGGER.error("execute failed,reason:",e); - throw new DSSRuntimeException(e.getMessage()); - } - }; - } -} diff --git a/dss-common/src/main/java/com/webank/wedatasphere/dss/common/utils/FileHelper.java b/dss-common/src/main/java/com/webank/wedatasphere/dss/common/utils/FileHelper.java deleted file mode 100644 index ea9e61469f..0000000000 --- a/dss-common/src/main/java/com/webank/wedatasphere/dss/common/utils/FileHelper.java +++ /dev/null @@ -1,33 +0,0 @@ -/* - * Copyright 2019 WeBank - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package com.webank.wedatasphere.dss.common.utils; - -import java.io.File; - -/** - * created by enjoyyin on 2019/6/14 - * Description: FileHelper 是 linux 文件系统的帮助类，检查目录是否存在等 - */ -public class FileHelper { - - public static boolean checkDirExists(String dir){ - File file = new File(dir); - return file.exists() && file.isDirectory(); - } - -} diff --git a/dss-common/src/main/java/com/webank/wedatasphere/dss/common/utils/ZipHelper.java b/dss-common/src/main/java/com/webank/wedatasphere/dss/common/utils/ZipHelper.java deleted file mode 100644 index 594d5c922c..0000000000 --- a/dss-common/src/main/java/com/webank/wedatasphere/dss/common/utils/ZipHelper.java +++ /dev/null @@ -1,126 +0,0 @@ -/* - * Copyright 2019 WeBank - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package com.webank.wedatasphere.dss.common.utils; - - - -import com.webank.wedatasphere.dss.common.exception.DSSErrorException; -import org.apache.commons.io.IOUtils; -import org.apache.commons.lang.StringUtils; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.BufferedReader; -import java.io.File; -import java.io.InputStreamReader; -import java.util.ArrayList; -import java.util.List; - -/** - * created by enjoyyin on 2019/6/13 - * Description: - */ -public class ZipHelper { - - private static final Logger logger = LoggerFactory.getLogger(ZipHelper.class); - - private static final String ZIP_CMD = "zip"; - private static final String RECURSIVE = "-r"; - private static final String ZIP_TYPE = ".zip"; - /** - * ZipHelper可以将传入的path进行打包 - * @param dirPath 需要打包的project路径,绝对路径 - * @return 打包之后的zip包全路径 - */ - public static String zip(String dirPath)throws DSSErrorException { - if(!FileHelper.checkDirExists(dirPath)){ - logger.error("{} 不存在, 不能创建zip文件", dirPath); - throw new DSSErrorException(90001,dirPath + " does not exist, can not zip"); - } - //先用简单的方法，调用新进程进行压缩 - String[] strArr = dirPath.split(File.separator); - String shortPath = strArr[strArr.length - 1]; - String workPath = dirPath.substring(0, dirPath.length() - shortPath.length() - 1); - List list = new ArrayList<>(); - list.add(ZIP_CMD); - list.add(RECURSIVE); - String zipFilePath = shortPath + ZIP_TYPE; - String longZipFilePath = dirPath + ZIP_TYPE; - list.add(zipFilePath); - list.add(shortPath); - ProcessBuilder processBuilder = new ProcessBuilder(list); - processBuilder.redirectErrorStream(true); - processBuilder.directory(new File(workPath)); - BufferedReader infoReader = null; - BufferedReader errorReader = null; - try{ - Process process = processBuilder.start(); - infoReader = new BufferedReader(new InputStreamReader(process.getInputStream())); - errorReader = new BufferedReader(new InputStreamReader(process.getErrorStream())); - String infoLine = null; - while((infoLine = infoReader.readLine()) != null){ - logger.info("process output: {} ", infoLine); - } - String errorLine = null; - StringBuilder errMsg = new StringBuilder(); - while((errorLine = errorReader.readLine()) != null){ - if (StringUtils.isNotEmpty(errorLine)){ - errMsg.append(errorLine).append("\n"); - } - logger.error("process error: {} ", errorLine); - } - int exitCode = process.waitFor(); - if (exitCode != 0){ - throw new DSSErrorException(90002,errMsg.toString()); - } - }catch(final Exception e){ - logger.error("{} 压缩成 zip 文件失败, reason: ", e); - DSSErrorException exception = new DSSErrorException(90003,dirPath + " to zip file failed"); - exception.initCause(e); - throw exception; - } finally { - //删掉整个目录 - File file = new File(dirPath); - logger.info("开始删除目录 {}", dirPath); - if (deleteDir(file)){ - logger.info("结束删除目录 {} 成功", dirPath); - }else{ - logger.info("删除目录 {} 失败", dirPath); - } - IOUtils.closeQuietly(infoReader); - IOUtils.closeQuietly(errorReader); - } - return longZipFilePath; - } - - private static boolean deleteDir(File dir) { - if (dir.isDirectory()) { - String[] children = dir.list(); - if (children != null && children.length > 0){ - for (String s : children) { - boolean success = deleteDir(new File(dir, s)); - if (!success) { - return false; - } - } - } - } - // 目录此时为空，可以删除 - return dir.delete(); - } -} diff --git a/dss-common/src/main/scala/com/webank/wedatasphere/dss/common/protocol/RequestDSSProject.scala b/dss-common/src/main/scala/com/webank/wedatasphere/dss/common/protocol/RequestDSSProject.scala deleted file mode 100644 index b9b3da46af..0000000000 --- a/dss-common/src/main/scala/com/webank/wedatasphere/dss/common/protocol/RequestDSSProject.scala +++ /dev/null @@ -1,25 +0,0 @@ -/* - * Copyright 2019 WeBank - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package com.webank.wedatasphere.dss.common.protocol - -/** - * Created by enjoyyin on 2019/11/8. - */ -case class RequestDSSProject(flowId:Long, version:String, projectVersionId:Long) - -case class RequestDSSApplication(name:String) \ No newline at end of file diff --git a/dss-flow-execution-entrance/bin/start-dss-flow-execution-entrance.sh b/dss-flow-execution-entrance/bin/start-dss-flow-execution-entrance.sh deleted file mode 100644 index 9bd4a006cb..0000000000 --- a/dss-flow-execution-entrance/bin/start-dss-flow-execution-entrance.sh +++ /dev/null @@ -1,49 +0,0 @@ -#!/bin/bash -cd `dirname $0` -cd .. -HOME=`pwd` - -export SERVER_PID=$HOME/bin/linkis.pid -export SERVER_LOG_PATH=$HOME/logs -export SERVER_CLASS=com.webank.wedatasphere.linkis.DataWorkCloudApplication - -if test -z "$SERVER_HEAP_SIZE" -then - export SERVER_HEAP_SIZE="512M" -fi - -if test -z "$SERVER_JAVA_OPTS" -then - export SERVER_JAVA_OPTS=" -Xmx$SERVER_HEAP_SIZE -XX:+UseG1GC -Xloggc:$HOME/logs/linkis-gc.log" -fi - -if [[ -f "${SERVER_PID}" ]]; then - pid=$(cat ${SERVER_PID}) - if kill -0 ${pid} >/dev/null 2>&1; then - echo "Server is already running." - exit 1 - fi -fi - -nohup java $SERVER_JAVA_OPTS -cp $HOME/conf:$HOME/lib/* $SERVER_CLASS 2>&1 > $SERVER_LOG_PATH/linkis.out & -pid=$! -if [[ -z "${pid}" ]]; then - echo "server $SERVER_NAME start failed!" - exit 1 -else - echo "server $SERVER_NAME start succeeded!" - echo $pid > $SERVER_PID - sleep 1 -fi - - - - - - - - - - - - diff --git a/dss-flow-execution-entrance/bin/stop-dss-flow-execution-entrance.sh b/dss-flow-execution-entrance/bin/stop-dss-flow-execution-entrance.sh deleted file mode 100644 index ce9cb2ee26..0000000000 --- a/dss-flow-execution-entrance/bin/stop-dss-flow-execution-entrance.sh +++ /dev/null @@ -1,47 +0,0 @@ -#!/bin/bash - -cd `dirname $0` -cd .. -HOME=`pwd` - -export FLOW_ENTRANCE_HOME_PID=$HOME/bin/linkis.pid - -function wait_for_FLOW_ENTRANCE_HOME_to_die() { - local pid - local count - pid=$1 - timeout=$2 - count=0 - timeoutTime=$(date "+%s") - let "timeoutTime+=$timeout" - currentTime=$(date "+%s") - forceKill=1 - - while [[ $currentTime -lt $timeoutTime ]]; do - $(kill ${pid} > /dev/null 2> /dev/null) - if kill -0 ${pid} > /dev/null 2>&1; then - sleep 3 - else - forceKill=0 - break - fi - currentTime=$(date "+%s") - done - - if [[ forceKill -ne 0 ]]; then - $(kill -9 ${pid} > /dev/null 2> /dev/null) - fi -} - -if [[ ! -f "${FLOW_ENTRANCE_HOME_PID}" ]]; then - echo "FlowExecution Entrance is not running" -else - pid=$(cat ${FLOW_ENTRANCE_HOME_PID}) - if [[ -z "${pid}" ]]; then - echo "FlowExecution Entrance is not running" - else - wait_for_FLOW_ENTRANCE_HOME_to_die $pid 40 - $(rm -f ${FLOW_ENTRANCE_HOME_PID}) - echo "FlowExecution Entrance is stopped." - fi -fi diff --git a/dss-flow-execution-entrance/pom.xml b/dss-flow-execution-entrance/pom.xml deleted file mode 100644 index 669ca30cd5..0000000000 --- a/dss-flow-execution-entrance/pom.xml +++ /dev/null @@ -1,126 +0,0 @@ - - - - - - dss - com.webank.wedatasphere.dss - 0.9.1 - - 4.0.0 - - dss-flow-execution-entrance - - - - com.webank.wedatasphere.linkis - linkis-ujes-entrance - ${linkis.version} - - - org.apache.poi - ooxml-schemas - - - - - com.webank.wedatasphere.linkis - linkis-cloudRPC - ${linkis.version} - - - com.webank.wedatasphere.dss - dss-linkis-node-execution - ${dss.version} - - - com.ibm.icu - icu4j - - - - - - com.webank.wedatasphere.dss - dss-scheduler-appjoint-core - ${dss.version} - - - - - - - - org.apache.maven.plugins - maven-deploy-plugin - - - - net.alchim31.maven - scala-maven-plugin - - - org.apache.maven.plugins - maven-jar-plugin - - - org.apache.maven.plugins - maven-assembly-plugin - 2.3 - false - - - make-assembly - package - - single - - - - src/main/assembly/distribution.xml - - - - - - false - dss-flow-execution-entrance - false - false - - src/main/assembly/distribution.xml - - - - - - - ${basedir}/src/main/resources - - **/*.properties - **/*.xml - **/*.yml - - - - ${project.artifactId}-${project.version} - - - \ No newline at end of file diff --git a/dss-flow-execution-entrance/src/main/assembly/distribution.xml b/dss-flow-execution-entrance/src/main/assembly/distribution.xml deleted file mode 100644 index bb09aad22b..0000000000 --- a/dss-flow-execution-entrance/src/main/assembly/distribution.xml +++ /dev/null @@ -1,303 +0,0 @@ - - - - dss-flow-execution-entrance - - zip - - true - dss-flow-execution-entrance - - - - - - lib - true - true - false - true - true - - - antlr:antlr:jar - aopalliance:aopalliance:jar - asm:asm:jar - cglib:cglib:jar - - - - - - - com.fasterxml.jackson.core:jackson-annotations:jar - com.fasterxml.jackson.core:jackson-core:jar - com.fasterxml.jackson.core:jackson-databind:jar - - com.fasterxml.jackson.datatype:jackson-datatype-jdk8:jar - com.fasterxml.jackson.datatype:jackson-datatype-jsr310:jar - com.fasterxml.jackson.jaxrs:jackson-jaxrs-base:jar - com.fasterxml.jackson.jaxrs:jackson-jaxrs-json-provider:jar - com.fasterxml.jackson.module:jackson-module-jaxb-annotations:jar - com.fasterxml.jackson.module:jackson-module-parameter-names:jar - com.fasterxml.jackson.module:jackson-module-paranamer:jar - com.fasterxml.jackson.module:jackson-module-scala_2.11:jar - - - com.google.code.findbugs:annotations:jar - - com.google.code.gson:gson:jar - com.google.guava:guava:jar - com.google.inject:guice:jar - com.netflix.archaius:archaius-core:jar - com.netflix.eureka:eureka-client:jar - com.netflix.eureka:eureka-core:jar - com.netflix.hystrix:hystrix-core:jar - com.netflix.netflix-commons:netflix-commons-util:jar - com.netflix.netflix-commons:netflix-eventbus:jar - com.netflix.netflix-commons:netflix-infix:jar - com.netflix.netflix-commons:netflix-statistics:jar - com.netflix.ribbon:ribbon:jar - com.netflix.ribbon:ribbon-core:jar - com.netflix.ribbon:ribbon-eureka:jar - com.netflix.ribbon:ribbon-httpclient:jar - com.netflix.ribbon:ribbon-loadbalancer:jar - com.netflix.ribbon:ribbon-transport:jar - com.netflix.servo:servo-core:jar - com.sun.jersey.contribs:jersey-apache-client4:jar - com.sun.jersey:jersey-client:jar - com.sun.jersey:jersey-core:jar - com.sun.jersey:jersey-json:jar - com.sun.jersey:jersey-server:jar - com.sun.jersey:jersey-servlet:jar - com.sun.xml.bind:jaxb-impl:jar - com.thoughtworks.paranamer:paranamer:jar - com.thoughtworks.xstream:xstream:jar - com.webank.wedatasphere.linkis:linkis-common:jar - com.webank.wedatasphere.linkis:linkis-module:jar - commons-beanutils:commons-beanutils:jar - commons-codec:commons-codec:jar - commons-collections:commons-collections:jar - commons-configuration:commons-configuration:jar - commons-dbcp:commons-dbcp:jar - commons-io:commons-io:jar - commons-jxpath:commons-jxpath:jar - commons-lang:commons-lang:jar - commons-logging:commons-logging:jar - commons-net:commons-net:jar - commons-pool:commons-pool:jar - io.micrometer:micrometer-core:jar - io.netty:netty-all:jar - io.netty:netty-buffer:jar - io.netty:netty-codec:jar - io.netty:netty-codec-http:jar - io.netty:netty-common:jar - io.netty:netty-handler:jar - io.netty:netty-transport:jar - io.netty:netty-transport-native-epoll:jar - io.reactivex:rxjava:jar - io.reactivex:rxnetty:jar - io.reactivex:rxnetty-contexts:jar - io.reactivex:rxnetty-servo:jar - javax.activation:activation:jar - javax.annotation:javax.annotation-api:jar - javax.inject:javax.inject:jar - javax.servlet:javax.servlet-api:jar - javax.validation:validation-api:jar - javax.websocket:javax.websocket-api:jar - javax.ws.rs:javax.ws.rs-api:jar - javax.xml.bind:jaxb-api:jar - javax.xml.stream:stax-api:jar - joda-time:joda-time:jar - mysql:mysql-connector-java:jar - org.antlr:antlr-runtime:jar - org.antlr:stringtemplate:jar - org.apache.commons:commons-math:jar - org.apache.httpcomponents:httpclient:jar - org.apache.logging.log4j:log4j-api:jar - org.apache.logging.log4j:log4j-core:jar - org.apache.logging.log4j:log4j-jul:jar - org.apache.logging.log4j:log4j-slf4j-impl:jar - org.aspectj:aspectjweaver:jar - org.bouncycastle:bcpkix-jdk15on:jar - org.bouncycastle:bcprov-jdk15on:jar - org.codehaus.jackson:jackson-jaxrs:jar - org.codehaus.jackson:jackson-xc:jar - org.codehaus.jettison:jettison:jar - org.codehaus.woodstox:stax2-api:jar - org.codehaus.woodstox:woodstox-core-asl:jar - org.eclipse.jetty:jetty-annotations:jar - org.eclipse.jetty:jetty-client:jar - org.eclipse.jetty:jetty-continuation:jar - org.eclipse.jetty:jetty-http:jar - org.eclipse.jetty:jetty-io:jar - org.eclipse.jetty:jetty-plus:jar - org.eclipse.jetty:jetty-security:jar - org.eclipse.jetty:jetty-server:jar - org.eclipse.jetty:jetty-servlet:jar - org.eclipse.jetty:jetty-servlets:jar - org.eclipse.jetty:jetty-util:jar - org.eclipse.jetty:jetty-webapp:jar - org.eclipse.jetty:jetty-xml:jar - org.eclipse.jetty.websocket:javax-websocket-client-impl:jar - org.eclipse.jetty.websocket:javax-websocket-server-impl:jar - org.eclipse.jetty.websocket:websocket-api:jar - org.eclipse.jetty.websocket:websocket-client:jar - org.eclipse.jetty.websocket:websocket-common:jar - org.eclipse.jetty.websocket:websocket-server:jar - org.eclipse.jetty.websocket:websocket-servlet:jar - org.glassfish.hk2:class-model:jar - org.glassfish.hk2:config-types:jar - org.glassfish.hk2.external:aopalliance-repackaged:jar - org.glassfish.hk2.external:asm-all-repackaged:jar - org.glassfish.hk2.external:bean-validator:jar - org.glassfish.hk2.external:javax.inject:jar - org.glassfish.hk2:hk2:jar - org.glassfish.hk2:hk2-api:jar - org.glassfish.hk2:hk2-config:jar - org.glassfish.hk2:hk2-core:jar - org.glassfish.hk2:hk2-locator:jar - org.glassfish.hk2:hk2-runlevel:jar - org.glassfish.hk2:hk2-utils:jar - org.glassfish.hk2:osgi-resource-locator:jar - org.glassfish.hk2:spring-bridge:jar - org.glassfish.jersey.bundles:jaxrs-ri:jar - org.glassfish.jersey.bundles.repackaged:jersey-guava:jar - org.glassfish.jersey.containers:jersey-container-servlet:jar - org.glassfish.jersey.containers:jersey-container-servlet-core:jar - org.glassfish.jersey.core:jersey-client:jar - org.glassfish.jersey.core:jersey-common:jar - org.glassfish.jersey.core:jersey-server:jar - org.glassfish.jersey.ext:jersey-entity-filtering:jar - org.glassfish.jersey.ext:jersey-spring3:jar - org.glassfish.jersey.media:jersey-media-jaxb:jar - org.glassfish.jersey.media:jersey-media-json-jackson:jar - org.glassfish.jersey.media:jersey-media-multipart:jar - org.hdrhistogram:HdrHistogram:jar - org.javassist:javassist:jar - org.json4s:json4s-ast_2.11:jar - org.json4s:json4s-core_2.11:jar - org.json4s:json4s-jackson_2.11:jar - org.jvnet.mimepull:mimepull:jar - org.jvnet:tiger-types:jar - org.latencyutils:LatencyUtils:jar - org.mortbay.jasper:apache-el:jar - org.ow2.asm:asm-analysis:jar - org.ow2.asm:asm-commons:jar - org.ow2.asm:asm-tree:jar - org.reflections:reflections:jar - org.scala-lang.modules:scala-parser-combinators_2.11:jar - org.scala-lang.modules:scala-xml_2.11:jar - org.scala-lang:scala-compiler:jar - org.scala-lang:scala-library:jar - org.scala-lang:scala-reflect:jar - org.scala-lang:scalap:jar - org.slf4j:jul-to-slf4j:jar - org.slf4j:slf4j-api:jar - org.springframework.boot:spring-boot:jar - org.springframework.boot:spring-boot-actuator:jar - org.springframework.boot:spring-boot-actuator-autoconfigure:jar - org.springframework.boot:spring-boot-autoconfigure:jar - org.springframework.boot:spring-boot-starter:jar - org.springframework.boot:spring-boot-starter-actuator:jar - org.springframework.boot:spring-boot-starter-aop:jar - org.springframework.boot:spring-boot-starter-jetty:jar - org.springframework.boot:spring-boot-starter-json:jar - org.springframework.boot:spring-boot-starter-log4j2:jar - org.springframework.boot:spring-boot-starter-web:jar - org.springframework.cloud:spring-cloud-commons:jar - org.springframework.cloud:spring-cloud-config-client:jar - org.springframework.cloud:spring-cloud-context:jar - - org.springframework.cloud:spring-cloud-netflix-core:jar - org.springframework.cloud:spring-cloud-netflix-eureka-client:jar - - org.springframework.cloud:spring-cloud-starter:jar - org.springframework.cloud:spring-cloud-starter-config:jar - org.springframework.cloud:spring-cloud-starter-eureka:jar - org.springframework.cloud:spring-cloud-starter-netflix-archaius:jar - org.springframework.cloud:spring-cloud-starter-netflix-eureka-client:jar - org.springframework.cloud:spring-cloud-starter-netflix-ribbon:jar - org.springframework.security:spring-security-crypto:jar - org.springframework.security:spring-security-rsa:jar - org.springframework:spring-aop:jar - org.springframework:spring-beans:jar - org.springframework:spring-context:jar - org.springframework:spring-core:jar - org.springframework:spring-expression:jar - org.springframework:spring-jcl:jar - org.springframework:spring-web:jar - org.springframework:spring-webmvc:jar - org.yaml:snakeyaml:jar - - xmlpull:xmlpull:jar - xpp3:xpp3_min:jar - - - - - - - ${basedir}/src/main/resources - - * - - 0777 - conf - unix - - - ${basedir}/bin - - * - - 0777 - bin - unix - - - . - - */** - - logs - - - - - diff --git a/dss-flow-execution-entrance/src/main/java/com/webank/wedatasphere/dss/flow/execution/entrance/entity/FlowExecutionCode.java b/dss-flow-execution-entrance/src/main/java/com/webank/wedatasphere/dss/flow/execution/entrance/entity/FlowExecutionCode.java deleted file mode 100644 index 417da3972e..0000000000 --- a/dss-flow-execution-entrance/src/main/java/com/webank/wedatasphere/dss/flow/execution/entrance/entity/FlowExecutionCode.java +++ /dev/null @@ -1,70 +0,0 @@ -/* - * Copyright 2019 WeBank - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package com.webank.wedatasphere.dss.flow.execution.entrance.entity; - -/** - * Created by peacewong on 2019/11/7. - */ -public class FlowExecutionCode { - - private String version; - - /*private String flowName;*/ - - private Long flowId; - - private Long projectVersionId; - - public String getVersion() { - return version; - } - - public void setVersion(String version) { - this.version = version; - } - - - - - - - - public Long getFlowId() { - return flowId; - } - - public void setFlowId(Long flowId) { - this.flowId = flowId; - } - - public Long getProjectVersionId() { - return projectVersionId; - } - - public void setProjectVersionId(Long projectVersionId) { - this.projectVersionId = projectVersionId; - } - - /* public String getFlowName() { - return flowName; - } - - public void setFlowName(String flowName) { - this.flowName = flowName; - }*/ -} diff --git a/dss-flow-execution-entrance/src/main/java/com/webank/wedatasphere/dss/flow/execution/entrance/entity/FlowExecutionNode.java b/dss-flow-execution-entrance/src/main/java/com/webank/wedatasphere/dss/flow/execution/entrance/entity/FlowExecutionNode.java deleted file mode 100644 index 73f6b21a46..0000000000 --- a/dss-flow-execution-entrance/src/main/java/com/webank/wedatasphere/dss/flow/execution/entrance/entity/FlowExecutionNode.java +++ /dev/null @@ -1,27 +0,0 @@ -/* - * Copyright 2019 WeBank - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package com.webank.wedatasphere.dss.flow.execution.entrance.entity; - - -import com.webank.wedatasphere.dss.appjoint.scheduler.entity.AbstractSchedulerNode; - -/** - * Created by peacewong on 2019/11/5. - */ -public class FlowExecutionNode extends AbstractSchedulerNode { -} diff --git a/dss-flow-execution-entrance/src/main/java/com/webank/wedatasphere/dss/flow/execution/entrance/entity/FlowExecutonReadNode.java b/dss-flow-execution-entrance/src/main/java/com/webank/wedatasphere/dss/flow/execution/entrance/entity/FlowExecutonReadNode.java deleted file mode 100644 index 386a0b2db2..0000000000 --- a/dss-flow-execution-entrance/src/main/java/com/webank/wedatasphere/dss/flow/execution/entrance/entity/FlowExecutonReadNode.java +++ /dev/null @@ -1,51 +0,0 @@ -/* - * Copyright 2019 WeBank - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package com.webank.wedatasphere.dss.flow.execution.entrance.entity; - - -import com.webank.wedatasphere.dss.appjoint.scheduler.entity.ReadNode; -import com.webank.wedatasphere.dss.appjoint.scheduler.entity.SchedulerNode; - -/** - * Created by peacewong on 2019/11/6. - */ -public class FlowExecutonReadNode extends FlowExecutionNode implements ReadNode { - - private SchedulerNode schedulerNode; - private String[] nodeIds; - - @Override - public String[] getShareNodeIds() { - return this.nodeIds; - } - - @Override - public void setShareNodeIds(String[] strings) { - this.nodeIds = strings; - } - - @Override - public SchedulerNode getSchedulerNode() { - return schedulerNode; - } - - @Override - public void setSchedulerNode(SchedulerNode schedulerNode) { - this.schedulerNode = schedulerNode; - } -} diff --git a/dss-flow-execution-entrance/src/main/java/com/webank/wedatasphere/dss/flow/execution/entrance/entity/FlowExecutonSharedNode.java b/dss-flow-execution-entrance/src/main/java/com/webank/wedatasphere/dss/flow/execution/entrance/entity/FlowExecutonSharedNode.java deleted file mode 100644 index 52dfcdcc8b..0000000000 --- a/dss-flow-execution-entrance/src/main/java/com/webank/wedatasphere/dss/flow/execution/entrance/entity/FlowExecutonSharedNode.java +++ /dev/null @@ -1,51 +0,0 @@ -/* - * Copyright 2019 WeBank - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package com.webank.wedatasphere.dss.flow.execution.entrance.entity; - - -import com.webank.wedatasphere.dss.appjoint.scheduler.entity.SchedulerNode; -import com.webank.wedatasphere.dss.appjoint.scheduler.entity.ShareNode; - -/** - * Created by peacewong on 2019/11/6. - */ -public class FlowExecutonSharedNode extends FlowExecutionNode implements ShareNode { - - private SchedulerNode schedulerNode; - private int shareTimes; - - @Override - public int getShareTimes() { - return this.shareTimes; - } - - @Override - public void setShareTimes(int i) { - this.shareTimes = i; - } - - @Override - public SchedulerNode getSchedulerNode() { - return this.schedulerNode; - } - - @Override - public void setSchedulerNode(SchedulerNode schedulerNode) { - this.schedulerNode = schedulerNode; - } -} diff --git a/dss-flow-execution-entrance/src/main/java/com/webank/wedatasphere/dss/flow/execution/entrance/job/FlowExecutionAppJointLinkisJob.java b/dss-flow-execution-entrance/src/main/java/com/webank/wedatasphere/dss/flow/execution/entrance/job/FlowExecutionAppJointLinkisJob.java deleted file mode 100644 index 6a82646ddc..0000000000 --- a/dss-flow-execution-entrance/src/main/java/com/webank/wedatasphere/dss/flow/execution/entrance/job/FlowExecutionAppJointLinkisJob.java +++ /dev/null @@ -1,48 +0,0 @@ -/* - * Copyright 2019 WeBank - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package com.webank.wedatasphere.dss.flow.execution.entrance.job; - -import com.webank.wedatasphere.dss.flow.execution.entrance.conf.FlowExecutionEntranceConfiguration; -import com.webank.wedatasphere.dss.linkis.node.execution.job.AbstractAppJointLinkisJob; - - -/** - * Created by peacewong on 2019/11/3. - */ -public class FlowExecutionAppJointLinkisJob extends AbstractAppJointLinkisJob { - - - @Override - public String getSubmitUser() { - return getJobProps().get(FlowExecutionEntranceConfiguration.FLOW_SUBMIT_USER()); - } - - - - @Override - public String getUser() { - return getJobProps().get(FlowExecutionEntranceConfiguration.PROXY_USER()); - } - - @Override - public String getJobName() { - return getJobProps().get(FlowExecutionEntranceConfiguration.JOB_ID()); - } - - -} diff --git a/dss-flow-execution-entrance/src/main/java/com/webank/wedatasphere/dss/flow/execution/entrance/job/FlowExecutionAppJointLinkisReadJob.java b/dss-flow-execution-entrance/src/main/java/com/webank/wedatasphere/dss/flow/execution/entrance/job/FlowExecutionAppJointLinkisReadJob.java deleted file mode 100644 index f1ca1da847..0000000000 --- a/dss-flow-execution-entrance/src/main/java/com/webank/wedatasphere/dss/flow/execution/entrance/job/FlowExecutionAppJointLinkisReadJob.java +++ /dev/null @@ -1,62 +0,0 @@ -/* - * Copyright 2019 WeBank - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package com.webank.wedatasphere.dss.flow.execution.entrance.job; - -import com.webank.wedatasphere.dss.appjoint.scheduler.entity.ReadNode; -import com.webank.wedatasphere.dss.flow.execution.entrance.conf.FlowExecutionEntranceConfiguration; -import com.webank.wedatasphere.dss.flow.execution.entrance.conf.FlowExecutionEntranceConfiguration$; -import com.webank.wedatasphere.dss.linkis.node.execution.job.ReadJob; - -import java.util.Map; - -/** - * Created by peacewong on 2019/11/3. - */ -public class FlowExecutionAppJointLinkisReadJob extends FlowExecutionAppJointLinkisJob implements ReadJob { - - private Map sharedNodesInfo; - - private ReadNode readNode; - - public void setReadNode(ReadNode readNode){ - this.readNode = readNode; - } - - @Override - public Map getSharedNodesInfo() { - return this.sharedNodesInfo; - } - - @Override - public String getSharedKey(String value) { - String projectId = getJobProps().get(FlowExecutionEntranceConfiguration.PROJECT_NAME()); - String flowId = getJobProps().get(FlowExecutionEntranceConfiguration.FLOW_NAME()); - String flowExecId = getJobProps().get(FlowExecutionEntranceConfiguration.FLOW_EXEC_ID()); - return projectId + "." + flowId + "." + flowExecId + "." + value; - } - - @Override - public void setSharedNodesInfo(Map sharedNodesInfo) { - this.sharedNodesInfo = sharedNodesInfo; - } - - @Override - public String[] getShareNodeIds() { - return this.readNode.getShareNodeIds(); - } -} diff --git a/dss-flow-execution-entrance/src/main/java/com/webank/wedatasphere/dss/flow/execution/entrance/job/FlowExecutionAppJointLinkisSharedJob.java b/dss-flow-execution-entrance/src/main/java/com/webank/wedatasphere/dss/flow/execution/entrance/job/FlowExecutionAppJointLinkisSharedJob.java deleted file mode 100644 index 1cb7256817..0000000000 --- a/dss-flow-execution-entrance/src/main/java/com/webank/wedatasphere/dss/flow/execution/entrance/job/FlowExecutionAppJointLinkisSharedJob.java +++ /dev/null @@ -1,49 +0,0 @@ -/* - * Copyright 2019 WeBank - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package com.webank.wedatasphere.dss.flow.execution.entrance.job; - -import com.webank.wedatasphere.dss.flow.execution.entrance.conf.FlowExecutionEntranceConfiguration; -import com.webank.wedatasphere.dss.linkis.node.execution.job.SharedJob; - -/** - * Created by peacewong on 2019/11/3. - */ -public class FlowExecutionAppJointLinkisSharedJob extends FlowExecutionAppJointLinkisJob implements SharedJob { - - private int sharedNum; - - @Override - public int getSharedNum() { - return this.sharedNum; - } - - @Override - public String getSharedKey() { - - String projectId = getJobProps().get(FlowExecutionEntranceConfiguration.PROJECT_NAME()); - String flowId = getJobProps().get(FlowExecutionEntranceConfiguration.FLOW_NAME()); - String flowExecId = getJobProps().get(FlowExecutionEntranceConfiguration.FLOW_EXEC_ID()); - String nodeId = getJobProps().get(FlowExecutionEntranceConfiguration.JOB_ID()); - return projectId + "." + flowId + "." + flowExecId + "." + nodeId; - - } - - public void setSharedNum(int sharedNum) { - this.sharedNum = sharedNum; - } -} diff --git a/dss-flow-execution-entrance/src/main/java/com/webank/wedatasphere/dss/flow/execution/entrance/job/FlowExecutionAppJointSignalSharedJob.java b/dss-flow-execution-entrance/src/main/java/com/webank/wedatasphere/dss/flow/execution/entrance/job/FlowExecutionAppJointSignalSharedJob.java deleted file mode 100644 index d0d556aadf..0000000000 --- a/dss-flow-execution-entrance/src/main/java/com/webank/wedatasphere/dss/flow/execution/entrance/job/FlowExecutionAppJointSignalSharedJob.java +++ /dev/null @@ -1,56 +0,0 @@ -/* - * Copyright 2019 WeBank - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package com.webank.wedatasphere.dss.flow.execution.entrance.job; - -import com.webank.wedatasphere.dss.flow.execution.entrance.conf.FlowExecutionEntranceConfiguration; -import com.webank.wedatasphere.dss.linkis.node.execution.job.JobSignalKeyCreator; -import com.webank.wedatasphere.dss.linkis.node.execution.job.SignalSharedJob; - -import java.util.Map; - -/** - * Created by johnnwang on 2019/11/14. - */ -public class FlowExecutionAppJointSignalSharedJob extends FlowExecutionAppJointLinkisJob implements SignalSharedJob { - - private JobSignalKeyCreator signalKeyCreator; - - @Override - public JobSignalKeyCreator getSignalKeyCreator() { - return this.signalKeyCreator; - } - - @Override - public void setSignalKeyCreator(JobSignalKeyCreator signalKeyCreator) { - this.signalKeyCreator = signalKeyCreator; - } - - @Override - public String getMsgSaveKey() { - Map configuration = this.getConfiguration(); - Object map = configuration.get("runtime"); - if (map != null && map instanceof Map) { - Map runtime = (Map) map; - Object msgValue = runtime.get(FlowExecutionEntranceConfiguration.EVENT_CHECK_MSG_KEY().getValue()); - if (null != msgValue) { - return msgValue.toString(); - } - } - return null; - } -} diff --git a/dss-flow-execution-entrance/src/main/java/com/webank/wedatasphere/dss/flow/execution/entrance/job/FlowExecutionCommonLinkisJob.java b/dss-flow-execution-entrance/src/main/java/com/webank/wedatasphere/dss/flow/execution/entrance/job/FlowExecutionCommonLinkisJob.java deleted file mode 100644 index bd27ae43c1..0000000000 --- a/dss-flow-execution-entrance/src/main/java/com/webank/wedatasphere/dss/flow/execution/entrance/job/FlowExecutionCommonLinkisJob.java +++ /dev/null @@ -1,54 +0,0 @@ -/* - * Copyright 2019 WeBank - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package com.webank.wedatasphere.dss.flow.execution.entrance.job; - -import com.webank.wedatasphere.dss.flow.execution.entrance.conf.FlowExecutionEntranceConfiguration; -import com.webank.wedatasphere.dss.linkis.node.execution.entity.BMLResource; -import com.webank.wedatasphere.dss.linkis.node.execution.job.AbstractCommonLinkisJob; -import com.webank.wedatasphere.dss.linkis.node.execution.job.CommonLinkisJob; -import com.webank.wedatasphere.dss.linkis.node.execution.job.JobTypeEnum; -import com.webank.wedatasphere.dss.linkis.node.execution.log.LinkisJobExecutionLog; -import com.webank.wedatasphere.linkis.ujes.client.response.JobExecuteResult; - -import java.util.ArrayList; -import java.util.Map; - -/** - * Created by peacewong on 2019/11/3. - */ -public class FlowExecutionCommonLinkisJob extends AbstractCommonLinkisJob { - - - @Override - public String getSubmitUser() { - return getJobProps().get(FlowExecutionEntranceConfiguration.FLOW_SUBMIT_USER()); - } - - - @Override - public String getUser() { - return getJobProps().get(FlowExecutionEntranceConfiguration.PROXY_USER()); - } - - @Override - public String getJobName() { - return getJobProps().get(FlowExecutionEntranceConfiguration.JOB_ID()); - } - - -} diff --git a/dss-flow-execution-entrance/src/main/java/com/webank/wedatasphere/dss/flow/execution/entrance/job/FlowExecutionCommonLinkisReadJob.java b/dss-flow-execution-entrance/src/main/java/com/webank/wedatasphere/dss/flow/execution/entrance/job/FlowExecutionCommonLinkisReadJob.java deleted file mode 100644 index fa094ab373..0000000000 --- a/dss-flow-execution-entrance/src/main/java/com/webank/wedatasphere/dss/flow/execution/entrance/job/FlowExecutionCommonLinkisReadJob.java +++ /dev/null @@ -1,62 +0,0 @@ -/* - * Copyright 2019 WeBank - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package com.webank.wedatasphere.dss.flow.execution.entrance.job; - -import com.webank.wedatasphere.dss.appjoint.scheduler.entity.ReadNode; -import com.webank.wedatasphere.dss.flow.execution.entrance.conf.FlowExecutionEntranceConfiguration; -import com.webank.wedatasphere.dss.linkis.node.execution.job.ReadJob; - -import java.util.Map; - -/** - * Created by peacewong on 2019/11/3. - */ -public class FlowExecutionCommonLinkisReadJob extends FlowExecutionCommonLinkisJob implements ReadJob { - - private Map sharedNodesInfo; - - - private ReadNode readNode; - - public void setReadNode(ReadNode readNode){ - this.readNode = readNode; - } - - @Override - public Map getSharedNodesInfo() { - return this.sharedNodesInfo; - } - - @Override - public String getSharedKey(String value) { - String projectId = getJobProps().get(FlowExecutionEntranceConfiguration.PROJECT_NAME()); - String flowId = getJobProps().get(FlowExecutionEntranceConfiguration.FLOW_NAME()); - String flowExecId = getJobProps().get(FlowExecutionEntranceConfiguration.FLOW_EXEC_ID()); - return projectId + "." + flowId + "." + flowExecId + "." + value; - } - - @Override - public void setSharedNodesInfo(Map sharedNodesInfo) { - this.sharedNodesInfo = sharedNodesInfo; - } - - @Override - public String[] getShareNodeIds() { - return this.readNode.getShareNodeIds(); - } -} diff --git a/dss-flow-execution-entrance/src/main/java/com/webank/wedatasphere/dss/flow/execution/entrance/job/FlowExecutionCommonLinkisSharedJob.java b/dss-flow-execution-entrance/src/main/java/com/webank/wedatasphere/dss/flow/execution/entrance/job/FlowExecutionCommonLinkisSharedJob.java deleted file mode 100644 index 81a07d17aa..0000000000 --- a/dss-flow-execution-entrance/src/main/java/com/webank/wedatasphere/dss/flow/execution/entrance/job/FlowExecutionCommonLinkisSharedJob.java +++ /dev/null @@ -1,47 +0,0 @@ -/* - * Copyright 2019 WeBank - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package com.webank.wedatasphere.dss.flow.execution.entrance.job; - -import com.webank.wedatasphere.dss.flow.execution.entrance.conf.FlowExecutionEntranceConfiguration; -import com.webank.wedatasphere.dss.linkis.node.execution.job.SharedJob; - -/** - * Created by peacewong on 2019/11/3. - */ -public class FlowExecutionCommonLinkisSharedJob extends FlowExecutionCommonLinkisJob implements SharedJob { - - private int sharedNum; - - @Override - public int getSharedNum() { - return this.sharedNum; - } - - @Override - public String getSharedKey() { - String projectId = getJobProps().get(FlowExecutionEntranceConfiguration.PROJECT_NAME()); - String flowId = getJobProps().get(FlowExecutionEntranceConfiguration.FLOW_NAME()); - String flowExecId = getJobProps().get(FlowExecutionEntranceConfiguration.FLOW_EXEC_ID()); - String nodeId = getJobProps().get(FlowExecutionEntranceConfiguration.JOB_ID()); - return projectId + "." + flowId + "." + flowExecId + "." + nodeId; - } - - public void setSharedNum(int sharedNum) { - this.sharedNum = sharedNum; - } -} diff --git a/dss-flow-execution-entrance/src/main/java/com/webank/wedatasphere/dss/flow/execution/entrance/job/FlowExecutionJobSignalKeyCreator.java b/dss-flow-execution-entrance/src/main/java/com/webank/wedatasphere/dss/flow/execution/entrance/job/FlowExecutionJobSignalKeyCreator.java deleted file mode 100644 index e284b44d8e..0000000000 --- a/dss-flow-execution-entrance/src/main/java/com/webank/wedatasphere/dss/flow/execution/entrance/job/FlowExecutionJobSignalKeyCreator.java +++ /dev/null @@ -1,39 +0,0 @@ -/* - * Copyright 2019 WeBank - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package com.webank.wedatasphere.dss.flow.execution.entrance.job; - -import com.webank.wedatasphere.dss.flow.execution.entrance.conf.FlowExecutionEntranceConfiguration; -import com.webank.wedatasphere.dss.linkis.node.execution.job.Job; -import com.webank.wedatasphere.dss.linkis.node.execution.job.JobSignalKeyCreator; -import com.webank.wedatasphere.dss.linkis.node.execution.job.SignalSharedJob; - -public class FlowExecutionJobSignalKeyCreator implements JobSignalKeyCreator { - - @Override - public String getSignalKeyByJob(Job job) { - String projectId = job.getJobProps().get(FlowExecutionEntranceConfiguration.PROJECT_NAME()); - String flowId = job.getJobProps().get(FlowExecutionEntranceConfiguration.FLOW_NAME()); - String flowExecId = job.getJobProps().get(FlowExecutionEntranceConfiguration.FLOW_EXEC_ID()); - return projectId + "." + flowId + "." + flowExecId; - } - - @Override - public String getSignalKeyBySignalSharedJob(SignalSharedJob job) { - return getSignalKeyByJob((Job)job); - } -} diff --git a/dss-flow-execution-entrance/src/main/java/com/webank/wedatasphere/dss/flow/execution/entrance/parser/FlowExecutionFlowParser.java b/dss-flow-execution-entrance/src/main/java/com/webank/wedatasphere/dss/flow/execution/entrance/parser/FlowExecutionFlowParser.java deleted file mode 100644 index fbd8bdd762..0000000000 --- a/dss-flow-execution-entrance/src/main/java/com/webank/wedatasphere/dss/flow/execution/entrance/parser/FlowExecutionFlowParser.java +++ /dev/null @@ -1,44 +0,0 @@ -/* - * Copyright 2019 WeBank - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package com.webank.wedatasphere.dss.flow.execution.entrance.parser; - -import com.webank.wedatasphere.dss.appjoint.scheduler.parser.AbstractFlowParser; -import com.webank.wedatasphere.dss.appjoint.scheduler.parser.NodeParser; -import com.webank.wedatasphere.dss.common.entity.flow.DSSJSONFlow; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.stereotype.Component; - -/** - * Created by peacewong on 2019/11/5. - */ -@Component -public class FlowExecutionFlowParser extends AbstractFlowParser { - - @Autowired - @Override - public void setNodeParsers(NodeParser[] nodeParsers) { - super.setNodeParsers(nodeParsers); - } - - @Override - public Boolean ifFlowCanParse(DSSJSONFlow flow) { - return true; - } - - -} diff --git a/dss-flow-execution-entrance/src/main/java/com/webank/wedatasphere/dss/flow/execution/entrance/parser/FlowExecutionNodeParser.java b/dss-flow-execution-entrance/src/main/java/com/webank/wedatasphere/dss/flow/execution/entrance/parser/FlowExecutionNodeParser.java deleted file mode 100644 index 97c5824b98..0000000000 --- a/dss-flow-execution-entrance/src/main/java/com/webank/wedatasphere/dss/flow/execution/entrance/parser/FlowExecutionNodeParser.java +++ /dev/null @@ -1,50 +0,0 @@ -/* - * Copyright 2019 WeBank - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package com.webank.wedatasphere.dss.flow.execution.entrance.parser; - -import com.webank.wedatasphere.dss.appjoint.scheduler.entity.SchedulerNode; -import com.webank.wedatasphere.dss.appjoint.scheduler.parser.AbstractNodeParser; -import com.webank.wedatasphere.dss.common.entity.node.DSSNode; -import com.webank.wedatasphere.dss.flow.execution.entrance.entity.FlowExecutionNode; -import org.springframework.stereotype.Component; - - -/** - * Created by peacewong on 2019/11/5. - */ -@Component -public class FlowExecutionNodeParser extends AbstractNodeParser { - - @Override - public SchedulerNode parseNode(DSSNode dssNode) { - FlowExecutionNode node = new FlowExecutionNode(); - node.setDssNode(dssNode); - return node; - } - - @Override - public Boolean ifNodeCanParse(DSSNode dssNode) { - return true; - } - - @Override - public int getOrder() { - return 1; - } - -} diff --git a/dss-flow-execution-entrance/src/main/java/com/webank/wedatasphere/dss/flow/execution/entrance/parser/FlowExecutionProjectParser.java b/dss-flow-execution-entrance/src/main/java/com/webank/wedatasphere/dss/flow/execution/entrance/parser/FlowExecutionProjectParser.java deleted file mode 100644 index 0654c20aa2..0000000000 --- a/dss-flow-execution-entrance/src/main/java/com/webank/wedatasphere/dss/flow/execution/entrance/parser/FlowExecutionProjectParser.java +++ /dev/null @@ -1,43 +0,0 @@ -/* - * Copyright 2019 WeBank - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package com.webank.wedatasphere.dss.flow.execution.entrance.parser; - - -import com.webank.wedatasphere.dss.appjoint.scheduler.entity.AbstractSchedulerProject; -import com.webank.wedatasphere.dss.appjoint.scheduler.parser.AbstractProjectParser; -import com.webank.wedatasphere.dss.appjoint.scheduler.parser.FlowParser; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.stereotype.Component; - -/** - * Created by peacewong on 2019/11/7. - */ -@Component -public class FlowExecutionProjectParser extends AbstractProjectParser { - - @Autowired - @Override - public void setFlowParsers(FlowParser[] flowParsers) { - super.setFlowParsers(flowParsers); - } - - @Override - protected AbstractSchedulerProject createSchedulerProject() { - return new AbstractSchedulerProject() {}; - } -} diff --git a/dss-flow-execution-entrance/src/main/java/com/webank/wedatasphere/dss/flow/execution/entrance/parser/FlowExecutionReadNodeParser.java b/dss-flow-execution-entrance/src/main/java/com/webank/wedatasphere/dss/flow/execution/entrance/parser/FlowExecutionReadNodeParser.java deleted file mode 100644 index 1c04fdfd2a..0000000000 --- a/dss-flow-execution-entrance/src/main/java/com/webank/wedatasphere/dss/flow/execution/entrance/parser/FlowExecutionReadNodeParser.java +++ /dev/null @@ -1,55 +0,0 @@ -/* - * Copyright 2019 WeBank - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package com.webank.wedatasphere.dss.flow.execution.entrance.parser; - -import com.webank.wedatasphere.dss.appjoint.scheduler.entity.ReadNode; -import com.webank.wedatasphere.dss.appjoint.scheduler.entity.SchedulerNode; -import com.webank.wedatasphere.dss.appjoint.scheduler.parser.AbstractReadNodeParser; -import com.webank.wedatasphere.dss.common.entity.node.DSSNode; -import com.webank.wedatasphere.dss.flow.execution.entrance.entity.FlowExecutionNode; -import com.webank.wedatasphere.dss.flow.execution.entrance.entity.FlowExecutonReadNode; -import com.webank.wedatasphere.dss.flow.execution.entrance.utils.FlowExecutionUtils; - - -import org.springframework.stereotype.Component; - -/** - * Created by peacewong on 2019/11/7. - */ -@Component -public class FlowExecutionReadNodeParser extends AbstractReadNodeParser { - @Override - public int getOrder() { - return 2; - } - - @Override - public Boolean ifNodeCanParse(DSSNode dssNode) { - return FlowExecutionUtils.isReadNode(dssNode.getNodeType()); - } - - @Override - protected ReadNode createReadNode() { - return new FlowExecutonReadNode(); - } - - @Override - protected SchedulerNode createSchedulerNode() { - return new FlowExecutionNode(); - } -} diff --git a/dss-flow-execution-entrance/src/main/java/com/webank/wedatasphere/dss/flow/execution/entrance/restful/FlowExecutionRestfulApi.java b/dss-flow-execution-entrance/src/main/java/com/webank/wedatasphere/dss/flow/execution/entrance/restful/FlowExecutionRestfulApi.java deleted file mode 100644 index 8ad6dcb51f..0000000000 --- a/dss-flow-execution-entrance/src/main/java/com/webank/wedatasphere/dss/flow/execution/entrance/restful/FlowExecutionRestfulApi.java +++ /dev/null @@ -1,95 +0,0 @@ -/* - * Copyright 2019 WeBank - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package com.webank.wedatasphere.dss.flow.execution.entrance.restful; - -import com.webank.wedatasphere.dss.flow.execution.entrance.FlowContext; -import com.webank.wedatasphere.dss.flow.execution.entrance.FlowContext$; -import com.webank.wedatasphere.dss.flow.execution.entrance.job.FlowEntranceJob; -import com.webank.wedatasphere.linkis.entrance.EntranceServer; -import com.webank.wedatasphere.linkis.entrance.annotation.EntranceServerBeanAnnotation; -import com.webank.wedatasphere.linkis.protocol.utils.ZuulEntranceUtils; -import com.webank.wedatasphere.linkis.scheduler.queue.Job; -import com.webank.wedatasphere.linkis.server.Message; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.springframework.context.annotation.ComponentScan; -import org.springframework.stereotype.Component; -import scala.Option; - -import javax.ws.rs.*; -import javax.ws.rs.core.MediaType; -import javax.ws.rs.core.Response; -import java.util.List; -import java.util.Map; - -/** - * Created by peacewong on 2019/11/5. - */ - -@Path("/entrance") -@Component -@Produces(MediaType.APPLICATION_JSON) -@Consumes(MediaType.APPLICATION_JSON) -public class FlowExecutionRestfulApi { - - private EntranceServer entranceServer; - - private static final Logger logger = LoggerFactory.getLogger(FlowExecutionRestfulApi.class); - - @EntranceServerBeanAnnotation.EntranceServerAutowiredAnnotation - public void setEntranceServer(EntranceServer entranceServer){ - this.entranceServer = entranceServer; - } - - @GET - @Path("/{id}/execution") - public Response execution(@PathParam("id") String id) { - Message message = null; - String realId = ZuulEntranceUtils.parseExecID(id)[3]; - Option job = entranceServer.getJob(realId); - try { - if (job.isDefined() && job.get() instanceof FlowEntranceJob){ - logger.info("Start to get job {} execution info", job.get().getId()); - FlowEntranceJob flowEntranceJob = (FlowEntranceJob) job.get(); - - message = Message.ok("Successfully get job execution info"); - message.setMethod("/api/entrance/" + id + "/execution"); - message.setStatus(0); - long nowTime = System.currentTimeMillis(); - flowEntranceJob.getFlowContext().getRunningNodes().forEach((k, v) -> { - if (v != null) { - v.setNowTime(nowTime); - } - }); - message.data("runningJobs", FlowContext$.MODULE$.convertView(flowEntranceJob.getFlowContext().getRunningNodes())); - List> pendingList = FlowContext$.MODULE$.convertView(flowEntranceJob.getFlowContext().getPendingNodes()); - pendingList.addAll(FlowContext$.MODULE$.convertView(flowEntranceJob.getFlowContext().getSkippedNodes())); - message.data("pendingJobs", pendingList); - message.data("succeedJobs", FlowContext$.MODULE$.convertView(flowEntranceJob.getFlowContext().getSucceedNodes())); - message.data("failedJobs", FlowContext$.MODULE$.convertView(flowEntranceJob.getFlowContext().getFailedNodes())); - message.data("skippedJobs", FlowContext$.MODULE$.convertView(flowEntranceJob.getFlowContext().getSkippedNodes())); - }else{ - message = Message.error("ID The corresponding job is empty and cannot obtain the corresponding task status.(ID 对应的job为空，不能获取相应的任务状态)"); - } - } catch (Exception e) { - message = Message.error("Failed to get job execution info"); - } - return Message.messageToResponse(message); - } - -} diff --git a/dss-flow-execution-entrance/src/main/java/com/webank/wedatasphere/dss/flow/execution/entrance/tuning/FlowExecutionFlowTuning.java b/dss-flow-execution-entrance/src/main/java/com/webank/wedatasphere/dss/flow/execution/entrance/tuning/FlowExecutionFlowTuning.java deleted file mode 100644 index 0361e0ded9..0000000000 --- a/dss-flow-execution-entrance/src/main/java/com/webank/wedatasphere/dss/flow/execution/entrance/tuning/FlowExecutionFlowTuning.java +++ /dev/null @@ -1,58 +0,0 @@ -/* - * Copyright 2019 WeBank - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package com.webank.wedatasphere.dss.flow.execution.entrance.tuning; - -import com.webank.wedatasphere.dss.appjoint.scheduler.entity.SchedulerFlow; -import com.webank.wedatasphere.dss.appjoint.scheduler.entity.ShareNode; -import com.webank.wedatasphere.dss.appjoint.scheduler.tuning.AbstractShareNodeFlowTuning; -import com.webank.wedatasphere.dss.appjoint.scheduler.tuning.NodeTuning; -import com.webank.wedatasphere.dss.flow.execution.entrance.entity.FlowExecutonSharedNode; - -import org.springframework.stereotype.Component; - -/** - * Created by peacewong on 2019/11/7. - */ -@Component -public class FlowExecutionFlowTuning extends AbstractShareNodeFlowTuning { - @Override - public int getOrder() { - return 0; - } - - @Override - public SchedulerFlow tuningSchedulerFlow(SchedulerFlow schedulerFlow) { - schedulerFlow = super.tuningSchedulerFlow(schedulerFlow); - return schedulerFlow; - } - - @Override - public Boolean ifFlowCanTuning(SchedulerFlow schedulerFlow) { - return true ; - } - - @Override - protected ShareNode createShareNode() { - return new FlowExecutonSharedNode(); - } - - @Override - public NodeTuning[] getNodeTunings() { - return new NodeTuning[0]; - } -} diff --git a/dss-flow-execution-entrance/src/main/java/com/webank/wedatasphere/dss/flow/execution/entrance/tuning/FlowExecutionProjectTuning.java b/dss-flow-execution-entrance/src/main/java/com/webank/wedatasphere/dss/flow/execution/entrance/tuning/FlowExecutionProjectTuning.java deleted file mode 100644 index 1ac0ee1575..0000000000 --- a/dss-flow-execution-entrance/src/main/java/com/webank/wedatasphere/dss/flow/execution/entrance/tuning/FlowExecutionProjectTuning.java +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright 2019 WeBank - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package com.webank.wedatasphere.dss.flow.execution.entrance.tuning; - - -import com.webank.wedatasphere.dss.appjoint.scheduler.tuning.AbstractProjectTuning; -import com.webank.wedatasphere.dss.appjoint.scheduler.tuning.FlowTuning; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.stereotype.Component; - -/** - * Created by peacewong on 2019/11/7. - */ -@Component -public class FlowExecutionProjectTuning extends AbstractProjectTuning { - - @Autowired - @Override - public void setFlowTunings(FlowTuning[] flowTunings) { - super.setFlowTunings(flowTunings); - } -} diff --git a/dss-flow-execution-entrance/src/main/resources/application.yml b/dss-flow-execution-entrance/src/main/resources/application.yml deleted file mode 100644 index c4bf9270ac..0000000000 --- a/dss-flow-execution-entrance/src/main/resources/application.yml +++ /dev/null @@ -1,23 +0,0 @@ -server: - port: 9005 -spring: - application: - name: flowExecutionEntrance - - -eureka: - client: - serviceUrl: - defaultZone: http://127.0.0.1:20303/eureka/ - instance: - metadata-map: - test: wedatasphere - -management: - endpoints: - web: - exposure: - include: refresh,info -logging: - config: classpath:log4j2.xml - diff --git a/dss-flow-execution-entrance/src/main/resources/linkis.properties b/dss-flow-execution-entrance/src/main/resources/linkis.properties deleted file mode 100644 index 09e5c93176..0000000000 --- a/dss-flow-execution-entrance/src/main/resources/linkis.properties +++ /dev/null @@ -1,42 +0,0 @@ -# -# Copyright 2019 WeBank -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# - - -wds.linkis.server.restful.scan.packages=com.webank.wedatasphere.linkis.entrance.restful,com.webank.wedatasphere.dss.flow.execution.entrance.restful - -#wds.linkis.server.component.exclude.classes=com.webank.wedatasphere.linkis.DataWorkCloudApplication - -wds.linkis.engine.application.name=flowExecutionEngine -wds.linkis.enginemanager.application.name=flowExecution - -wds.linkis.query.application.name=cloud-publicservice - -wds.linkis.console.config.application.name=cloud-publicservice -wds.linkis.engine.creation.wait.time.max=20m -wds.linkis.server.version=v1 -#hadoop config dir -#hadoop.config.dir=/appcom/config/hadoop-config -wds.linkis.entrance.config.logPath=file:///tmp/linkis/ - -wds.linkis.resultSet.store.path=hdfs:///tmp/linkis - -wds.linkis.server.socket.mode=true - -##linkis gateway conf -wds.linkis.gateway.url=http://127.0.0.1:9001 -wds.linkis.client.flow.adminuser=ws -wds.linkis.client.flow.author.user.token=WS-AUTH diff --git a/dss-flow-execution-entrance/src/main/resources/log4j.properties b/dss-flow-execution-entrance/src/main/resources/log4j.properties deleted file mode 100644 index 6c64396931..0000000000 --- a/dss-flow-execution-entrance/src/main/resources/log4j.properties +++ /dev/null @@ -1,37 +0,0 @@ -# -# Copyright 2019 WeBank -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# - -### set log levels ### - -log4j.rootCategory=INFO,console - -log4j.appender.console=org.apache.log4j.ConsoleAppender -log4j.appender.console.Threshold=INFO -log4j.appender.console.layout=org.apache.log4j.PatternLayout -#log4j.appender.console.layout.ConversionPattern= %d{ISO8601} %-5p (%t) [%F:%M(%L)] - %m%n -log4j.appender.console.layout.ConversionPattern= %d{ISO8601} %-5p (%t) %p %c{1} - %m%n - - -log4j.appender.com.webank.bdp.ide.core=org.apache.log4j.DailyRollingFileAppender -log4j.appender.com.webank.bdp.ide.core.Threshold=INFO -log4j.additivity.com.webank.bdp.ide.core=false -log4j.appender.com.webank.bdp.ide.core.layout=org.apache.log4j.PatternLayout -log4j.appender.com.webank.bdp.ide.core.Append=true -log4j.appender.com.webank.bdp.ide.core.File=logs/linkis.log -log4j.appender.com.webank.bdp.ide.core.layout.ConversionPattern= %d{ISO8601} %-5p (%t) [%F:%M(%L)] - %m%n - -log4j.logger.org.springframework=INFO \ No newline at end of file diff --git a/dss-flow-execution-entrance/src/main/resources/log4j2.xml b/dss-flow-execution-entrance/src/main/resources/log4j2.xml deleted file mode 100644 index 3923cd9f39..0000000000 --- a/dss-flow-execution-entrance/src/main/resources/log4j2.xml +++ /dev/null @@ -1,39 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - diff --git a/dss-flow-execution-entrance/src/main/scala/com/webank/wedatasphere/dss/flow/execution/entrance/FlowContext.scala b/dss-flow-execution-entrance/src/main/scala/com/webank/wedatasphere/dss/flow/execution/entrance/FlowContext.scala deleted file mode 100644 index f03081a5e1..0000000000 --- a/dss-flow-execution-entrance/src/main/scala/com/webank/wedatasphere/dss/flow/execution/entrance/FlowContext.scala +++ /dev/null @@ -1,86 +0,0 @@ -/* - * Copyright 2019 WeBank - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package com.webank.wedatasphere.dss.flow.execution.entrance - -import java.util - -import com.webank.wedatasphere.dss.appjoint.scheduler.entity.SchedulerNode -import com.webank.wedatasphere.dss.flow.execution.entrance.node.NodeRunner -import com.webank.wedatasphere.linkis.scheduler.queue.SchedulerEventState.SchedulerEventState - -/** - * Created by peacewong on 2019/11/5. - */ -trait FlowContext { - - def getRunningNodes: util.Map[String, NodeRunner] - - def getSucceedNodes: util.Map[String, NodeRunner] - - def getPendingNodes: util.Map[String, NodeRunner] - - def getFailedNodes: util.Map[String, NodeRunner] - - def getSkippedNodes: util.Map[String, NodeRunner] - - def getScheduledNodes: util.Map[String, NodeRunner] - - def getFlowStatus: SchedulerEventState - - def isNodeCompleted(nodeName: String): Boolean -} - -object FlowContext { - - def isNodeRunning(nodeName: String, flowContext: FlowContext):Boolean = { - flowContext.getRunningNodes.containsKey(nodeName) || flowContext.getScheduledNodes.containsKey(nodeName) - } - - def changedNodeState(fromMap: util.Map[String, NodeRunner], - toMap: util.Map[String, NodeRunner], node: SchedulerNode,info:String): Unit = { - val nodeName = node.getDssNode.getName - if (fromMap.containsKey(nodeName)) { - val runner = fromMap.get(nodeName) - runner.setNodeExecutedInfo(info) - toMap.put(nodeName, runner) - fromMap.remove(nodeName) - } - } - - def convertView(nodeRunnerMap: java.util.Map[String, NodeRunner]): java.util.List[java.util.Map[String, Any]] = { - val nodes = new util.ArrayList[java.util.Map[String, Any]]() - val iterator = nodeRunnerMap.entrySet().iterator() - while (iterator.hasNext) { - val nodeRunner = iterator.next().getValue - if (nodeRunner != null && nodeRunner.getNode != null) { - val nodeView = new java.util.HashMap[String, Any]() - val linkisJob = nodeRunner.getLinkisJob - val node = nodeRunner.getNode - if (linkisJob != null && linkisJob.getJobExecuteResult != null) { - nodeView.put("execID", linkisJob.getJobExecuteResult.getExecID) - nodeView.put("taskID", linkisJob.getJobExecuteResult.getTaskID()) - } - nodeView.put("nodeID", node.getId) - nodeView.put("info", nodeRunner.getNodeExecutedInfo()) - nodeView.put("startTime", nodeRunner.getStartTime()) - nodes.add(nodeView) - } - } - nodes - } -} \ No newline at end of file diff --git a/dss-flow-execution-entrance/src/main/scala/com/webank/wedatasphere/dss/flow/execution/entrance/FlowContextImpl.scala b/dss-flow-execution-entrance/src/main/scala/com/webank/wedatasphere/dss/flow/execution/entrance/FlowContextImpl.scala deleted file mode 100644 index 84f6d7caac..0000000000 --- a/dss-flow-execution-entrance/src/main/scala/com/webank/wedatasphere/dss/flow/execution/entrance/FlowContextImpl.scala +++ /dev/null @@ -1,65 +0,0 @@ -/* - * Copyright 2019 WeBank - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package com.webank.wedatasphere.dss.flow.execution.entrance - -import java.util - -import com.webank.wedatasphere.dss.flow.execution.entrance.node.NodeRunner -import com.webank.wedatasphere.linkis.common.utils.Logging -import com.webank.wedatasphere.linkis.scheduler.queue.SchedulerEventState -import com.webank.wedatasphere.linkis.scheduler.queue.SchedulerEventState.SchedulerEventState - -/** - * created by chaogefeng on 2019/11/5 15:59 - * Description: - */ -class FlowContextImpl extends FlowContext with Logging { - - val runningNodes: util.Map[String, NodeRunner] = new util.HashMap[String, NodeRunner] - - val succeedNodes: util.Map[String, NodeRunner] = new util.HashMap[String, NodeRunner] - - val pendingNodes: util.Map[String, NodeRunner] = new util.HashMap[String, NodeRunner] - - val failedNodes: util.Map[String, NodeRunner] = new util.HashMap[String, NodeRunner] - - val skippedNodes: util.Map[String, NodeRunner] = new util.HashMap[String, NodeRunner] - - val scheduledNodes: util.Map[String, NodeRunner] = new util.HashMap[String, NodeRunner] - - val flowStatus:SchedulerEventState = SchedulerEventState.Inited - - override def getRunningNodes: util.Map[String, NodeRunner] = this.runningNodes - - override def getSucceedNodes: util.Map[String, NodeRunner] = this.succeedNodes - - override def getPendingNodes: util.Map[String, NodeRunner] =this.pendingNodes - - override def getFailedNodes: util.Map[String, NodeRunner] = this.failedNodes - - override def getSkippedNodes: util.Map[String, NodeRunner] = this.skippedNodes - - override def getScheduledNodes: util.Map[String, NodeRunner] = this.scheduledNodes - - override def getFlowStatus: SchedulerEventState = this.flowStatus - - override def isNodeCompleted(nodeName: String): Boolean = { - getSkippedNodes.containsKey(nodeName) || getSucceedNodes.containsKey(nodeName) || getFailedNodes.containsKey(nodeName) - } - -} \ No newline at end of file diff --git a/dss-flow-execution-entrance/src/main/scala/com/webank/wedatasphere/dss/flow/execution/entrance/conf/FlowExecutionEntranceConfiguration.scala b/dss-flow-execution-entrance/src/main/scala/com/webank/wedatasphere/dss/flow/execution/entrance/conf/FlowExecutionEntranceConfiguration.scala deleted file mode 100644 index 245a23687f..0000000000 --- a/dss-flow-execution-entrance/src/main/scala/com/webank/wedatasphere/dss/flow/execution/entrance/conf/FlowExecutionEntranceConfiguration.scala +++ /dev/null @@ -1,95 +0,0 @@ -/* - * Copyright 2019 WeBank - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package com.webank.wedatasphere.dss.flow.execution.entrance.conf - -import com.webank.wedatasphere.linkis.common.conf.{ByteType, CommonVars} - -/** - * created by chaogefeng on 2019/11/4 16:35 - * Description: - */ -object FlowExecutionEntranceConfiguration { - - val APPLICATION_SPRING_APPLICATION = CommonVars("wds.linkis.application.spring.name", "cloud-application") - - val PUBLIC_SERVICE_SPRING_APPLICATION = CommonVars("wds.linkis.publicservice.spring.name", "cloud-publicserivce") - - val GATEWAY_SPRING_APPLICATION = CommonVars("wds.linkis.gateway.spring.name", "dataworkcloud-gateway") - - val QUERY_APPLICATION_NAME = CommonVars("wds.linkis.flow.execution.query.name", "cloud-query") - - val SCHEDULER_APPLICATION = CommonVars("wds.linkis.scheduler.spring.name", "dss-server") - - - val EVENT_CHECK_MSG_KEY = CommonVars("wds.dss.flow.datacheck.msg.key", "msg.savekey") - - val FLOW_EXECUTION_POOL_SIZE = CommonVars("wds.linkis.flow.execution.pool.size", 1000) - - val NODE_PRINT_FLAG = CommonVars("wds.linkis.flow.node.log.flag", false) - - val NODE_STATUS_POLLER_THREAD_SIZE = CommonVars("wds.dds.flow.node.status.poller.thread.size", 20) - - val NODE_STATUS_POLLER_SCHEDULER_TIME = CommonVars("wds.dds.flow.node.status.poller.scheduler.time", 5) - val FLOW_EXECUTION_SCHEDULER_POOL_SIZE = CommonVars("wds.linkis.flow.execution.pool.size", 30) - - val NODE_STATUS_INTERVAL = CommonVars("wds.dds.flow.node.status.poller.interval.time", 3000) - - val COMMAND = "command" - - val JOB_ID = "job.id" - - val FLOW_NAME = "dss.flow.flowid" - - - val PROJECT_NAME = "dss.flow.projectname" - - val FLOW_EXEC_ID = "dss.flow.execid" - - - - val PROXY_USER = "user.to.proxy" - - - val FLOW_SUBMIT_USER = "flow.submituser" - - val READ_NODE_TOKEN = "read.nodes" - - val SHARED_NODE_TOKEN = "share.num" - - - - val SKIP_NODES = CommonVars("wds.dss.flow.skip.nodes", "workflow.subflow") - val EMAIL_TYPE = "linkis.appjoint.sendemail" - - val SIGNAL_NODES = CommonVars("wds.dss.flow.signal.nodes", "linkis.appjoint.eventchecker.eventreceiver") - - val PROPS_MAP = "props_map" - - val FLOW_VAR_MAP = "flow_var_map" - - - val NODE_CONFIGURATION_KEY = "configuration" - - val PROJECT_RESOURCES = "project_resources" - - val FLOW_RESOURCES = "flow_resources" - - - - -} diff --git a/dss-flow-execution-entrance/src/main/scala/com/webank/wedatasphere/dss/flow/execution/entrance/conf/FlowExecutionEntranceSpringConfiguration.scala b/dss-flow-execution-entrance/src/main/scala/com/webank/wedatasphere/dss/flow/execution/entrance/conf/FlowExecutionEntranceSpringConfiguration.scala deleted file mode 100644 index 666c7a6e92..0000000000 --- a/dss-flow-execution-entrance/src/main/scala/com/webank/wedatasphere/dss/flow/execution/entrance/conf/FlowExecutionEntranceSpringConfiguration.scala +++ /dev/null @@ -1,60 +0,0 @@ -/* - * Copyright 2019 WeBank - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package com.webank.wedatasphere.dss.flow.execution.entrance.conf - -import com.webank.wedatasphere.dss.flow.execution.entrance.engine.{FlowEntranceEngine, FlowExecutionExecutorManagerImpl} -import com.webank.wedatasphere.dss.flow.execution.entrance.entranceparser.FlowExecutionParser -import com.webank.wedatasphere.linkis.entrance.EntranceParser -import com.webank.wedatasphere.linkis.entrance.annotation.{EngineBuilderBeanAnnotation, EngineManagerBeanAnnotation, EngineRequesterBeanAnnotation, EngineSelectorBeanAnnotation, EntranceExecutorManagerBeanAnnotation, EntranceParserBeanAnnotation, GroupFactoryBeanAnnotation} -import com.webank.wedatasphere.linkis.entrance.execute.{EngineBuilder, EngineManager, EngineRequester, EngineSelector, EntranceExecutorManager, EntranceExecutorRuler} -import com.webank.wedatasphere.linkis.scheduler.queue.GroupFactory -import org.slf4j.LoggerFactory -import org.springframework.beans.factory.annotation.Autowired -import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean -import org.springframework.context.annotation.Configuration - -/** - * created by chaogefeng on 2019/11/4 16:37 - * Description: - */ -@Configuration -class FlowExecutionEntranceSpringConfiguration { - private val logger = LoggerFactory.getLogger(classOf[FlowExecutionEntranceSpringConfiguration]) - - @EntranceExecutorManagerBeanAnnotation - def generateExternalEntranceExecutorManager(@GroupFactoryBeanAnnotation.GroupFactoryAutowiredAnnotation groupFactory: GroupFactory, - @EngineBuilderBeanAnnotation.EngineBuilderAutowiredAnnotation engineBuilder: EngineBuilder, - @EngineRequesterBeanAnnotation.EngineRequesterAutowiredAnnotation engineRequester: EngineRequester, - @EngineSelectorBeanAnnotation.EngineSelectorAutowiredAnnotation engineSelector: EngineSelector, - @EngineManagerBeanAnnotation.EngineManagerAutowiredAnnotation engineManager: EngineManager, - @Autowired entranceExecutorRulers: Array[EntranceExecutorRuler], - @Autowired flowEntranceEngine:FlowEntranceEngine): EntranceExecutorManager = { - logger.info("begin to get FlowExecution Entrance EntranceExecutorManager") - new FlowExecutionExecutorManagerImpl(groupFactory, engineBuilder, engineRequester, engineSelector, engineManager, entranceExecutorRulers, flowEntranceEngine) - } - - - - @EntranceParserBeanAnnotation - def generateEntranceParser(): EntranceParser = { - logger.info("begin to get FlowExecution Entrance parser") - new FlowExecutionParser() - } - - -} diff --git a/dss-flow-execution-entrance/src/main/scala/com/webank/wedatasphere/dss/flow/execution/entrance/engine/FlowEntranceEngine.scala b/dss-flow-execution-entrance/src/main/scala/com/webank/wedatasphere/dss/flow/execution/entrance/engine/FlowEntranceEngine.scala deleted file mode 100644 index 38042fc016..0000000000 --- a/dss-flow-execution-entrance/src/main/scala/com/webank/wedatasphere/dss/flow/execution/entrance/engine/FlowEntranceEngine.scala +++ /dev/null @@ -1,123 +0,0 @@ -/* - * Copyright 2019 WeBank - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package com.webank.wedatasphere.dss.flow.execution.entrance.engine - -import java.util - -import com.webank.wedatasphere.dss.flow.execution.entrance.exception.FlowExecutionErrorException -import com.webank.wedatasphere.dss.flow.execution.entrance.execution.FlowExecution -import com.webank.wedatasphere.dss.flow.execution.entrance.job.{FlowEntranceJob, FlowExecutionRequest} -import com.webank.wedatasphere.dss.flow.execution.entrance.job.parser.FlowEntranceJobParser -import com.webank.wedatasphere.dss.flow.execution.entrance.resolver.FlowDependencyResolver -import com.webank.wedatasphere.linkis.common.utils.{Logging, Utils} -import com.webank.wedatasphere.linkis.entrance.EntranceServer -import com.webank.wedatasphere.linkis.entrance.annotation.EntranceServerBeanAnnotation -import com.webank.wedatasphere.linkis.entrance.execute.{EngineExecuteAsynReturn, EntranceEngine, StorePathExecuteRequest} -import com.webank.wedatasphere.linkis.entrance.persistence.EntranceResultSetEngine -import com.webank.wedatasphere.linkis.protocol.engine.{JobProgressInfo, RequestTask} -import com.webank.wedatasphere.linkis.scheduler.executer._ -import org.apache.commons.lang.StringUtils -import org.springframework.beans.factory.annotation.Autowired -import org.springframework.stereotype.Component - -import scala.collection.JavaConversions._ - - -@Component -class FlowEntranceEngine extends EntranceEngine(id = 0) with ConcurrentTaskOperateSupport with Logging { - - - @Autowired - private var flowParsers: util.List[FlowEntranceJobParser] = _ - - @Autowired - private var flowDependencyResolvers: util.List[FlowDependencyResolver] = _ - - @Autowired - private var flowExecution: FlowExecution = _ - - - protected def executeLine(code: String, storePath: String, alias: String): ExecuteResponse = { - val realCode = code.trim() - logger.info(s"flowExecutionEngine begins to run code:\n ${realCode.trim}") - - SuccessExecuteResponse() - } - - override def execute(executeRequest: ExecuteRequest): ExecuteResponse = { - executeRequest match { - case flowExecutionRequest: FlowExecutionRequest => - flowExecutionRequest.job match { - case job: FlowEntranceJob => - if (job.isCompleted) { - info(s"Flow(${job.getId}) isCompleted status is ${job.getState}") - return SuccessExecuteResponse() - } - Utils.tryCatch { - if (null == job.getFlow) { - Utils.tryCatch{ - for (flowParser <- flowParsers) { - flowParser.parse(job) - } - }{ t => - throw new FlowExecutionErrorException(90101, s"Failed to parser flow of job(${job.getId})", t) - } - } - for (flowDependencyResolver <- flowDependencyResolvers) { - flowDependencyResolver.resolvedFlow(job) - } - flowExecution.runJob(job) - } { t => - job.kill() - error(s"Failed to execute job: ${job.getId}", t) - } - } - } - SuccessExecuteResponse() - } - - override protected def callExecute(request: RequestTask): EngineExecuteAsynReturn = null - - override def kill(jobId: String): Boolean = { - true - } - - override def killAll(): Boolean = { - true - } - - override def pause(jobId: String): Boolean = { - true - } - - override def pauseAll(): Boolean = { - true - } - - override def resume(jobId: String): Boolean = { - true - } - - override def resumeAll(): Boolean = { - true - } - - override def close(): Unit = { - - } -} diff --git a/dss-flow-execution-entrance/src/main/scala/com/webank/wedatasphere/dss/flow/execution/entrance/engine/FlowExecutionExecutorManagerImpl.scala b/dss-flow-execution-entrance/src/main/scala/com/webank/wedatasphere/dss/flow/execution/entrance/engine/FlowExecutionExecutorManagerImpl.scala deleted file mode 100644 index a3cd00fe75..0000000000 --- a/dss-flow-execution-entrance/src/main/scala/com/webank/wedatasphere/dss/flow/execution/entrance/engine/FlowExecutionExecutorManagerImpl.scala +++ /dev/null @@ -1,78 +0,0 @@ -/* - * Copyright 2019 WeBank - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package com.webank.wedatasphere.dss.flow.execution.entrance.engine - -import com.webank.wedatasphere.dss.flow.execution.entrance.job.FlowEntranceJob -import com.webank.wedatasphere.linkis.common.utils.Logging -import com.webank.wedatasphere.linkis.entrance.execute.{EngineBuilder, EngineManager, EngineRequester, EngineSelector, EntranceEngine, EntranceExecutorRuler} -import com.webank.wedatasphere.linkis.entrance.execute.impl.EntranceExecutorManagerImpl -import com.webank.wedatasphere.linkis.rpc.Sender -import com.webank.wedatasphere.linkis.scheduler.executer.Executor -import com.webank.wedatasphere.linkis.scheduler.listener.ExecutorListener -import com.webank.wedatasphere.linkis.scheduler.queue.{GroupFactory, Job, SchedulerEvent} - -import scala.concurrent.duration.Duration - -/** - * created by chaogefeng on 2019/11/4 17:25 - * Description: - */ -class FlowExecutionExecutorManagerImpl(groupFactory: GroupFactory, - engineBuilder: EngineBuilder, - engineRequester: EngineRequester, - engineSelector: EngineSelector, - engineManager: EngineManager, - entranceExecutorRulers: Array[EntranceExecutorRuler], - flowEntranceEngine: FlowEntranceEngine - ) - extends EntranceExecutorManagerImpl(groupFactory, engineBuilder, engineRequester, engineSelector, engineManager, entranceExecutorRulers) with Logging{ - - this.flowEntranceEngine.setServiceInstance(Sender.getThisServiceInstance.getApplicationName, Sender.getThisServiceInstance.getInstance) - - override def setExecutorListener(executorListener: ExecutorListener): Unit = super.setExecutorListener(executorListener) - - override def initialEntranceEngine(engine: EntranceEngine): Unit = super.initialEntranceEngine(engine) - - protected override def createExecutor(schedulerEvent: SchedulerEvent): EntranceEngine = schedulerEvent match { - case job: FlowEntranceJob => - - this.flowEntranceEngine - case _ => null - } - - override protected def findExecutors(job: Job): Array[EntranceEngine] = super.findExecutors(job) - - override def askExecutor(schedulerEvent: SchedulerEvent): Option[Executor] = schedulerEvent match { - case job: FlowEntranceJob => - Some(createExecutor(schedulerEvent)) - } - - override def askExecutor(schedulerEvent: SchedulerEvent, wait: Duration): Option[Executor] = schedulerEvent match { - case job: FlowEntranceJob => - Some(createExecutor(schedulerEvent)) - } - - - override def getById(id: Long): Option[Executor] = super.getById(id) - - override def getByGroup(groupName: String): Array[Executor] = super.getByGroup(groupName) - - protected override def delete(executor: Executor): Unit = super.delete(executor) - - override def shutdown(): Unit = super.shutdown() -} diff --git a/dss-flow-execution-entrance/src/main/scala/com/webank/wedatasphere/dss/flow/execution/entrance/entranceparser/FlowExecutionParser.scala b/dss-flow-execution-entrance/src/main/scala/com/webank/wedatasphere/dss/flow/execution/entrance/entranceparser/FlowExecutionParser.scala deleted file mode 100644 index 2145029544..0000000000 --- a/dss-flow-execution-entrance/src/main/scala/com/webank/wedatasphere/dss/flow/execution/entrance/entranceparser/FlowExecutionParser.scala +++ /dev/null @@ -1,50 +0,0 @@ -/* - * Copyright 2019 WeBank - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package com.webank.wedatasphere.dss.flow.execution.entrance.entranceparser - - -import com.webank.wedatasphere.dss.flow.execution.entrance.job.FlowEntranceJob -import com.webank.wedatasphere.linkis.common.utils.Logging -import com.webank.wedatasphere.linkis.entrance.parser.CommonEntranceParser -import com.webank.wedatasphere.linkis.protocol.query.RequestPersistTask -import com.webank.wedatasphere.linkis.protocol.task.Task -import com.webank.wedatasphere.linkis.scheduler.queue.Job -import scala.collection.JavaConversions._ - -/** - * created by chaogefeng on 2019/11/4 17:11 - * Description: - */ -class FlowExecutionParser extends CommonEntranceParser with Logging { - logger.info("FlowExecutionParser Registered") - - override def parseToJob(task: Task): Job = { - val job = new FlowEntranceJob - task match { - case requestPersistTask: RequestPersistTask => - job.setTask(task) - job.setUser(requestPersistTask.getUmUser) - job.setCreator(requestPersistTask.getRequestApplicationName) - job.setParams(requestPersistTask.getParams.toMap[String, Any] ) - job.setEntranceListenerBus(getEntranceContext.getOrCreateEventListenerBus) - job.setListenerEventBus(null) - job.setProgress(0f) - } - job - } -} diff --git a/dss-flow-execution-entrance/src/main/scala/com/webank/wedatasphere/dss/flow/execution/entrance/exception/FlowExecutionErrorException.scala b/dss-flow-execution-entrance/src/main/scala/com/webank/wedatasphere/dss/flow/execution/entrance/exception/FlowExecutionErrorException.scala deleted file mode 100644 index 0ad268be44..0000000000 --- a/dss-flow-execution-entrance/src/main/scala/com/webank/wedatasphere/dss/flow/execution/entrance/exception/FlowExecutionErrorException.scala +++ /dev/null @@ -1,32 +0,0 @@ -/* - * Copyright 2019 WeBank - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package com.webank.wedatasphere.dss.flow.execution.entrance.exception - -import com.webank.wedatasphere.linkis.common.exception.ErrorException - -/** - * Created by peacewong on 2019/11/5. - */ -class FlowExecutionErrorException(errorCode: Int, desc: String) extends ErrorException(errorCode, desc){ - - def this(errorCode: Int, desc: String, t:Throwable){ - this(errorCode, desc) - initCause(t) - } - -} diff --git a/dss-flow-execution-entrance/src/main/scala/com/webank/wedatasphere/dss/flow/execution/entrance/execution/DefaultFlowExecution.scala b/dss-flow-execution-entrance/src/main/scala/com/webank/wedatasphere/dss/flow/execution/entrance/execution/DefaultFlowExecution.scala deleted file mode 100644 index c1164c9d3d..0000000000 --- a/dss-flow-execution-entrance/src/main/scala/com/webank/wedatasphere/dss/flow/execution/entrance/execution/DefaultFlowExecution.scala +++ /dev/null @@ -1,85 +0,0 @@ -/* - * Copyright 2019 WeBank - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package com.webank.wedatasphere.dss.flow.execution.entrance.execution - -import java.util.concurrent.{Executors, LinkedBlockingQueue, TimeUnit} - -import com.webank.wedatasphere.dss.flow.execution.entrance.conf.FlowExecutionEntranceConfiguration -import com.webank.wedatasphere.dss.flow.execution.entrance.job.FlowEntranceJob -import com.webank.wedatasphere.dss.flow.execution.entrance.node.{NodeExecutionState, NodeRunner} -import com.webank.wedatasphere.dss.flow.execution.entrance.utils.FlowExecutionUtils -import com.webank.wedatasphere.linkis.common.utils.{Logging, Utils} -import org.springframework.stereotype.Service - -import scala.collection.JavaConversions._ -import scala.collection.mutable.ArrayBuffer - - -/** - * Created by johnnwang on 2019/11/5. - */ -@Service -class DefaultFlowExecution extends FlowExecution with Logging { - - - private val nodeRunnerQueue: LinkedBlockingQueue[NodeRunner] = new LinkedBlockingQueue[NodeRunner]() - - private val scheduledThreadPool = Executors.newScheduledThreadPool(FlowExecutionEntranceConfiguration.FLOW_EXECUTION_SCHEDULER_POOL_SIZE.getValue) - - private var pollerCount = 0 - - override def runJob(flowEntranceJob: FlowEntranceJob): Unit = { - - info(s"${flowEntranceJob.getId} Start to run executable node") - val scheduledNodes = flowEntranceJob.getFlowContext.getScheduledNodes - - if( ! scheduledNodes.isEmpty) { - // get executableNodes - flowEntranceJob.getFlowContext synchronized{ - val runners = new ArrayBuffer[NodeRunner]() - runners.addAll(scheduledNodes.values()) - val runningNodes = new ArrayBuffer[NodeRunner]() - runners.foreach{ runner => - if ( ! FlowExecutionUtils.isSkippedNode(runner.getNode)){ - info(s"scheduled node ${runner.getNode.getName} to running") - runner.fromScheduledTunToState(NodeExecutionState.Running) - // submit node runner - runningNodes.add(runner) - } else { - info(s"This node ${runner.getNode.getDssNode.getName} Skipped in execution") - runner.fromScheduledTunToState(NodeExecutionState.Skipped) - } - } - info(s"${flowEntranceJob.getId} Submit nodes(${runningNodes.size}) to running") - runningNodes.foreach{ node => - node.run() - nodeRunnerQueue.put(node) - if (pollerCount < FlowExecutionEntranceConfiguration.NODE_STATUS_POLLER_THREAD_SIZE.getValue){ - scheduledThreadPool.scheduleAtFixedRate(new NodeExecutionStatusPoller(nodeRunnerQueue), 1, - FlowExecutionEntranceConfiguration.NODE_STATUS_POLLER_SCHEDULER_TIME.getValue ,TimeUnit.SECONDS) - pollerCount = pollerCount + 1 - } - } - } - } else { - info(s"${flowEntranceJob.getId} no executable nodes") - flowEntranceJob.tryCompleted - } - info(s"${flowEntranceJob.getId} finished to run node") - } -} diff --git a/dss-flow-execution-entrance/src/main/scala/com/webank/wedatasphere/dss/flow/execution/entrance/execution/FlowExecution.scala b/dss-flow-execution-entrance/src/main/scala/com/webank/wedatasphere/dss/flow/execution/entrance/execution/FlowExecution.scala deleted file mode 100644 index 2160a01ee1..0000000000 --- a/dss-flow-execution-entrance/src/main/scala/com/webank/wedatasphere/dss/flow/execution/entrance/execution/FlowExecution.scala +++ /dev/null @@ -1,29 +0,0 @@ -/* - * Copyright 2019 WeBank - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package com.webank.wedatasphere.dss.flow.execution.entrance.execution - -import com.webank.wedatasphere.dss.flow.execution.entrance.job.FlowEntranceJob - -/** - * Created by peacewong on 2019/11/5. - */ -trait FlowExecution { - - def runJob(flowEntranceJob: FlowEntranceJob) - -} diff --git a/dss-flow-execution-entrance/src/main/scala/com/webank/wedatasphere/dss/flow/execution/entrance/execution/NodeExecutionStatusPoller.scala b/dss-flow-execution-entrance/src/main/scala/com/webank/wedatasphere/dss/flow/execution/entrance/execution/NodeExecutionStatusPoller.scala deleted file mode 100644 index ecaa1bf888..0000000000 --- a/dss-flow-execution-entrance/src/main/scala/com/webank/wedatasphere/dss/flow/execution/entrance/execution/NodeExecutionStatusPoller.scala +++ /dev/null @@ -1,44 +0,0 @@ -/* - * Copyright 2019 WeBank - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package com.webank.wedatasphere.dss.flow.execution.entrance.execution - - -import java.util.concurrent.LinkedBlockingQueue - -import com.webank.wedatasphere.dss.flow.execution.entrance.node.NodeRunner -import com.webank.wedatasphere.linkis.common.utils.Logging - -/** - * Created by peacewong on 2019/11/17. - */ -class NodeExecutionStatusPoller(nodeRunnerQueue: LinkedBlockingQueue[NodeRunner] ) extends Runnable with Logging{ - - - - override def run(): Unit = { - - val runner = nodeRunnerQueue.take() - - if(! runner.isLinkisJobCompleted) { - nodeRunnerQueue.put(runner) - } else { - info(s"node ${runner.getNode.getName} status is completed") - } - } - -} diff --git a/dss-flow-execution-entrance/src/main/scala/com/webank/wedatasphere/dss/flow/execution/entrance/job/FlowEntranceJob.scala b/dss-flow-execution-entrance/src/main/scala/com/webank/wedatasphere/dss/flow/execution/entrance/job/FlowEntranceJob.scala deleted file mode 100644 index 42bf395d46..0000000000 --- a/dss-flow-execution-entrance/src/main/scala/com/webank/wedatasphere/dss/flow/execution/entrance/job/FlowEntranceJob.scala +++ /dev/null @@ -1,179 +0,0 @@ -/* - * Copyright 2019 WeBank - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package com.webank.wedatasphere.dss.flow.execution.entrance.job - -import com.webank.wedatasphere.dss.appjoint.scheduler.entity.{SchedulerFlow, SchedulerNode} -import com.webank.wedatasphere.dss.common.entity.project.DSSProject -import com.webank.wedatasphere.dss.flow.execution.entrance.exception.FlowExecutionErrorException -import com.webank.wedatasphere.dss.flow.execution.entrance.{FlowContext, FlowContextImpl} -import com.webank.wedatasphere.dss.flow.execution.entrance.listener.NodeRunnerListener -import com.webank.wedatasphere.dss.flow.execution.entrance.node.{NodeExecutionState, NodeRunner} -import com.webank.wedatasphere.dss.flow.execution.entrance.node.NodeExecutionState.NodeExecutionState -import com.webank.wedatasphere.linkis.common.log.LogUtils -import com.webank.wedatasphere.linkis.common.utils.Utils -import com.webank.wedatasphere.linkis.entrance.execute.StorePathExecuteRequest -import com.webank.wedatasphere.linkis.entrance.job.EntranceExecutionJob -import com.webank.wedatasphere.linkis.protocol.query.RequestPersistTask -import com.webank.wedatasphere.linkis.scheduler.executer.{ErrorExecuteResponse, ExecuteRequest, SuccessExecuteResponse} -import com.webank.wedatasphere.linkis.scheduler.queue.{Job, SchedulerEventState} -import com.webank.wedatasphere.linkis.scheduler.queue.SchedulerEventState.Running - -import scala.beans.BeanProperty -import scala.collection.JavaConversions._ -import scala.collection.mutable.ArrayBuffer - -class FlowEntranceJob extends EntranceExecutionJob with NodeRunnerListener { - - private var flow: SchedulerFlow = _ - - private val flowContext: FlowContext = new FlowContextImpl - - - @BeanProperty var dwsProject: DSSProject = _ - - def setFlow(flow: SchedulerFlow): Unit = this.flow = flow - - def getFlow: SchedulerFlow = this.flow - - def getFlowContext: FlowContext = this.flowContext - - private val STATUS_CHANGED_LOCK = "STATUS_CHANGED_LOCK".intern() - - - override def jobToExecuteRequest(): ExecuteRequest = { - new ExecuteRequest with StorePathExecuteRequest with FlowExecutionRequest { - override val code: String = FlowEntranceJob.this.getTask match { - case requestPersistTask: RequestPersistTask => requestPersistTask.getExecutionCode - case _ => null - } - override val storePath: String = FlowEntranceJob.this.getTask match { - case requestPersistTask: RequestPersistTask => requestPersistTask.getResultLocation - case _ => "" - } - override val job: Job = FlowEntranceJob.this - } - } - - override def run(): Unit = { - setResultSize(0) - if (!isScheduled) return - startTime = System.currentTimeMillis - Utils.tryAndWarn(transition(Running)) - getExecutor.execute(jobToExecuteRequest()) - } - - - override def onStatusChanged(fromState: NodeExecutionState, toState: NodeExecutionState, node: SchedulerNode): Unit = { - - val nodeName = node.getDssNode.getName - toState match { - case NodeExecutionState.Failed => - printLog(s"Failed to execute node($nodeName),prepare to kill flow job", "ERROR") - if(NodeExecutionState.isRunning(fromState)) - FlowContext.changedNodeState(this.getFlowContext.getRunningNodes, this.getFlowContext.getFailedNodes, node, "node execute fail") - this.kill() - info(s"Succeed to kill flow job") - case NodeExecutionState.Cancelled => - printLog(s"node($nodeName) has cancelled execution", "WARN") - if(NodeExecutionState.isRunning(fromState)) - FlowContext.changedNodeState(this.getFlowContext.getRunningNodes, this.getFlowContext.getFailedNodes, node, "node has cancelled") - if(NodeExecutionState.isScheduled(fromState)) - FlowContext.changedNodeState(this.getFlowContext.getScheduledNodes, this.getFlowContext.getFailedNodes, node, "node has cancelled") - case NodeExecutionState.Skipped => - printLog(s"node($nodeName) has skipped execution from $fromState", "WARN") - FlowContext.changedNodeState(this.getFlowContext.getScheduledNodes, this.getFlowContext.getSkippedNodes, node, "node has skipped") - //Trigger the next execution - this.STATUS_CHANGED_LOCK.synchronized { - getExecutor.execute(jobToExecuteRequest()) - } - case NodeExecutionState.Succeed => - printLog(s"Succeed to execute node($nodeName)", "INFO") - if (NodeExecutionState.isRunning(fromState)) - FlowContext.changedNodeState(this.getFlowContext.getRunningNodes, this.getFlowContext.getSucceedNodes, node, "node execute success") - //Trigger the next execution - this.STATUS_CHANGED_LOCK.synchronized { - getExecutor.execute(jobToExecuteRequest()) - } - case NodeExecutionState.Running => - printLog(s"Start to execute node($nodeName) ", "INFO") - if(NodeExecutionState.isScheduled(fromState)) - FlowContext.changedNodeState(this.getFlowContext.getScheduledNodes, this.getFlowContext.getRunningNodes, node, "node in running") - - case NodeExecutionState.Scheduled => - printLog(s"node($nodeName) from inited to scheduled", "INFO") - if(NodeExecutionState.isInited(fromState)) - FlowContext.changedNodeState(this.getFlowContext.getPendingNodes, this.getFlowContext.getScheduledNodes, node, "node in scheduled") - case _ => - } - tryCompleted - } - - def printLog(log:String, level:String): Unit = level match { - case "INFO" => - info(log) - getLogListener.foreach(_.onLogUpdate(this, LogUtils.generateInfo(log))) - case "WARN" => - warn(log) - getLogListener.foreach(_.onLogUpdate(this, LogUtils.generateWarn(log))) - case "ERROR" => - error(log) - getLogListener.foreach(_.onLogUpdate(this, LogUtils.generateERROR(log))) - case _ => - } - - override def kill(): Unit = if (! SchedulerEventState.isCompleted(this.getState)) this synchronized { - if(! SchedulerEventState.isCompleted(this.getState)){ - Utils.tryAndWarn(this.killNodes) - super.kill() - transitionCompleted(ErrorExecuteResponse(s"execute job(${getId}) failed!", new FlowExecutionErrorException(90101, s"This Flow killed by user") )) - } - } - - override def cancel(): Unit = if (! SchedulerEventState.isCompleted(this.getState)) this synchronized { - if(! SchedulerEventState.isCompleted(this.getState)){ - Utils.tryAndWarn(this.killNodes) - super.cancel() - transitionCompleted(ErrorExecuteResponse(s"cancel job(${getId}) execution!", new FlowExecutionErrorException(90101, s"This Flow killed by user") )) - } - } - - def isFlowCompleted: Boolean = this.getFlowContext.getRunningNodes.isEmpty && this.getFlowContext.getPendingNodes.isEmpty && this.getFlowContext.getScheduledNodes.isEmpty - - - def tryCompleted: Unit = { - if (this.isFlowCompleted) { - info(s"This Flow(${getId}) is Completed") - /*transition(SchedulerEventState.Succeed)*/ - if(! SchedulerEventState.isCompleted(this.getState)) - transitionCompleted(SuccessExecuteResponse()) - } - } - - def killNodes: Unit = { - val runners = new ArrayBuffer[NodeRunner]() - runners.addAll(this.getFlowContext.getRunningNodes.values()) - for (node <- runners) { - node.cancel() - } - - } - - - - -} diff --git a/dss-flow-execution-entrance/src/main/scala/com/webank/wedatasphere/dss/flow/execution/entrance/job/FlowExecutionRequest.scala b/dss-flow-execution-entrance/src/main/scala/com/webank/wedatasphere/dss/flow/execution/entrance/job/FlowExecutionRequest.scala deleted file mode 100644 index b76afe5b4a..0000000000 --- a/dss-flow-execution-entrance/src/main/scala/com/webank/wedatasphere/dss/flow/execution/entrance/job/FlowExecutionRequest.scala +++ /dev/null @@ -1,28 +0,0 @@ -/* - * Copyright 2019 WeBank - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package com.webank.wedatasphere.dss.flow.execution.entrance.job - -import com.webank.wedatasphere.linkis.scheduler.queue.Job - -/** - * created by chaogefeng on 2019/11/4 16:59 - * Description: - */ -trait FlowExecutionRequest { - val job:Job -} diff --git a/dss-flow-execution-entrance/src/main/scala/com/webank/wedatasphere/dss/flow/execution/entrance/job/parser/FlowEntranceJobParser.scala b/dss-flow-execution-entrance/src/main/scala/com/webank/wedatasphere/dss/flow/execution/entrance/job/parser/FlowEntranceJobParser.scala deleted file mode 100644 index 4de0b1bf38..0000000000 --- a/dss-flow-execution-entrance/src/main/scala/com/webank/wedatasphere/dss/flow/execution/entrance/job/parser/FlowEntranceJobParser.scala +++ /dev/null @@ -1,29 +0,0 @@ -/* - * Copyright 2019 WeBank - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package com.webank.wedatasphere.dss.flow.execution.entrance.job.parser - -import com.webank.wedatasphere.dss.flow.execution.entrance.job.FlowEntranceJob - -/** - * Created by peacewong on 2019/11/5. - */ -trait FlowEntranceJobParser { - - def parse(flowEntranceJob: FlowEntranceJob):Unit - -} diff --git a/dss-flow-execution-entrance/src/main/scala/com/webank/wedatasphere/dss/flow/execution/entrance/job/parser/FlowJobFlowParser.scala b/dss-flow-execution-entrance/src/main/scala/com/webank/wedatasphere/dss/flow/execution/entrance/job/parser/FlowJobFlowParser.scala deleted file mode 100644 index a088e146f7..0000000000 --- a/dss-flow-execution-entrance/src/main/scala/com/webank/wedatasphere/dss/flow/execution/entrance/job/parser/FlowJobFlowParser.scala +++ /dev/null @@ -1,84 +0,0 @@ -/* - * Copyright 2019 WeBank - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package com.webank.wedatasphere.dss.flow.execution.entrance.job.parser - -import com.webank.wedatasphere.dss.appjoint.scheduler.entity.{AbstractSchedulerProject, SchedulerFlow} -import com.webank.wedatasphere.dss.common.entity.project.DSSProject -import com.webank.wedatasphere.dss.common.protocol.RequestDSSProject -import com.webank.wedatasphere.dss.flow.execution.entrance.conf.FlowExecutionEntranceConfiguration -import com.webank.wedatasphere.dss.flow.execution.entrance.entity.FlowExecutionCode -import com.webank.wedatasphere.dss.flow.execution.entrance.job.FlowEntranceJob -import com.webank.wedatasphere.dss.flow.execution.entrance.parser.FlowExecutionProjectParser -import com.webank.wedatasphere.dss.flow.execution.entrance.tuning.FlowExecutionProjectTuning -import com.webank.wedatasphere.dss.linkis.node.execution.utils.LinkisJobExecutionUtils -import com.webank.wedatasphere.linkis.common.utils.{Logging, Utils} -import com.webank.wedatasphere.linkis.rpc.Sender -import org.springframework.beans.factory.annotation.Autowired -import org.springframework.core.annotation.Order -import org.springframework.stereotype.Component - -import scala.collection.JavaConversions._ - - -@Order(1) -@Component -class FlowJobFlowParser extends FlowEntranceJobParser with Logging { - - - @Autowired - var flowExecutionProjectParser: FlowExecutionProjectParser = _ - - @Autowired - var flowExecutionProjectTuning: FlowExecutionProjectTuning = _ - - - override def parse(flowEntranceJob: FlowEntranceJob): Unit = { - info(s"${flowEntranceJob.getId} start to parse flow") - val code = flowEntranceJob.jobToExecuteRequest().code - val flowExecutionCode = LinkisJobExecutionUtils.gson.fromJson(code, classOf[FlowExecutionCode]) - - getDSSProjectByCode(flowExecutionCode) match { - case dwsProject: DSSProject => - - val project = this.flowExecutionProjectParser.parseProject(dwsProject) - - this.flowExecutionProjectTuning.tuningSchedulerProject(project) - - val allFlows = project.asInstanceOf[AbstractSchedulerProject].getProjectVersions.head.getFlows - - - var dwsFlow: SchedulerFlow = null - for (flow <- allFlows) { - if (flowExecutionCode.getFlowId == flow.getId) { - dwsFlow = flow - } - } - //save dwsProject - flowEntranceJob.setDwsProject(dwsProject) - //set flow - flowEntranceJob.setFlow(dwsFlow) - } - info(s"${flowEntranceJob.getId} finished to parse flow") - } - - private def getDSSProjectByCode(flowExecutionCode: FlowExecutionCode) = { - val req = new RequestDSSProject(flowExecutionCode.getFlowId, flowExecutionCode.getVersion, flowExecutionCode.getProjectVersionId) - Sender.getSender(FlowExecutionEntranceConfiguration.SCHEDULER_APPLICATION.getValue).ask(req) - } - -} diff --git a/dss-flow-execution-entrance/src/main/scala/com/webank/wedatasphere/dss/flow/execution/entrance/job/parser/FlowJobNodeParser.scala b/dss-flow-execution-entrance/src/main/scala/com/webank/wedatasphere/dss/flow/execution/entrance/job/parser/FlowJobNodeParser.scala deleted file mode 100644 index 2f18486fa3..0000000000 --- a/dss-flow-execution-entrance/src/main/scala/com/webank/wedatasphere/dss/flow/execution/entrance/job/parser/FlowJobNodeParser.scala +++ /dev/null @@ -1,99 +0,0 @@ -/* - * Copyright 2019 WeBank - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package com.webank.wedatasphere.dss.flow.execution.entrance.job.parser - -import java.util - -import com.webank.wedatasphere.dss.flow.execution.entrance.conf.FlowExecutionEntranceConfiguration -import com.webank.wedatasphere.dss.flow.execution.entrance.conf.FlowExecutionEntranceConfiguration._ -import com.webank.wedatasphere.dss.flow.execution.entrance.exception.FlowExecutionErrorException -import com.webank.wedatasphere.dss.flow.execution.entrance.job.FlowEntranceJob -import com.webank.wedatasphere.dss.flow.execution.entrance.node.DefaultNodeRunner -import com.webank.wedatasphere.dss.flow.execution.entrance.utils.FlowExecutionUtils -import com.webank.wedatasphere.dss.linkis.node.execution.conf.LinkisJobExecutionConfiguration -import com.webank.wedatasphere.dss.linkis.node.execution.entity.BMLResource -import com.webank.wedatasphere.dss.linkis.node.execution.utils.LinkisJobExecutionUtils -import com.webank.wedatasphere.linkis.common.utils.Logging - -import scala.collection.JavaConversions._ -import org.springframework.core.annotation.Order -import org.springframework.stereotype.Component - -/** - * Created by johnnwang on 2019/11/6. - */ - -@Order(2) -@Component -class FlowJobNodeParser extends FlowEntranceJobParser with Logging{ - - override def parse(flowEntranceJob: FlowEntranceJob): Unit = { - info(s"${flowEntranceJob.getId} Start to parse node of flow") - val project = flowEntranceJob.getDwsProject - val flow = flowEntranceJob.getFlow - val flowContext = flowEntranceJob.getFlowContext - if(null == flow) throw new FlowExecutionErrorException(90101, "This fow of job is empty ") - val nodes = flow.getSchedulerNodes - for (node <- nodes) { - - val nodeName = node.getName - val propsMap = new util.HashMap[String, String]() - val proxyUser = if (node.getDssNode.getUserProxy == null) flowEntranceJob.getUser else node.getDssNode.getUserProxy - propsMap.put(PROJECT_NAME, project.getName) - propsMap.put(FLOW_NAME, flow.getName) - propsMap.put(JOB_ID, nodeName) - propsMap.put(FLOW_SUBMIT_USER, flowEntranceJob.getUser) - info(s"${flowEntranceJob.getId} nodeName:${nodeName} node:${node.getNodeType}") - propsMap.put(LinkisJobExecutionConfiguration.LINKIS_TYPE, node.getNodeType) - - propsMap.put(PROXY_USER, proxyUser) - propsMap.put(COMMAND, LinkisJobExecutionUtils.gson.toJson(node.getDssNode.getJobContent)) - - var params = node.getDssNode.getParams - if (params == null) { - params = new util.HashMap[String,AnyRef]() - node.getDssNode.setParams(params) - } - val flowVar = new util.HashMap[String, AnyRef]() - val properties = flow.getFlowProperties - if(properties != null) { - for(proper <- properties){ - flowVar.putAll(proper) - } - } - - propsMap.put(FlowExecutionEntranceConfiguration.FLOW_EXEC_ID, flowEntranceJob.getId) - - params.put(PROPS_MAP, propsMap) - params.put(FLOW_VAR_MAP, flowVar) - params.put(PROJECT_RESOURCES, project.getProjectResources) - val flowNameAndResources = new util.HashMap[String, util.List[BMLResource]]() - flowNameAndResources.put("flow." + flow.getName + LinkisJobExecutionConfiguration.RESOURCES_NAME, FlowExecutionUtils.resourcesAdaptation(flow.getFlowResources)) - params.put(FLOW_RESOURCES, flowNameAndResources) - - val pendingNodeMap = flowContext.getPendingNodes - - val nodeRunner = pendingNodeMap.getOrDefault(nodeName, new DefaultNodeRunner) - nodeRunner.setNodeRunnerListener(flowEntranceJob) - nodeRunner.setNode(node) - pendingNodeMap.put(nodeName, nodeRunner) - } - info(s"${flowEntranceJob.getId} finished to parse node of flow") - } - -} diff --git a/dss-flow-execution-entrance/src/main/scala/com/webank/wedatasphere/dss/flow/execution/entrance/listener/NodeRunnerListener.scala b/dss-flow-execution-entrance/src/main/scala/com/webank/wedatasphere/dss/flow/execution/entrance/listener/NodeRunnerListener.scala deleted file mode 100644 index 4d25360f39..0000000000 --- a/dss-flow-execution-entrance/src/main/scala/com/webank/wedatasphere/dss/flow/execution/entrance/listener/NodeRunnerListener.scala +++ /dev/null @@ -1,29 +0,0 @@ -/* - * Copyright 2019 WeBank - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package com.webank.wedatasphere.dss.flow.execution.entrance.listener - -import com.webank.wedatasphere.dss.appjoint.scheduler.entity.SchedulerNode -import com.webank.wedatasphere.dss.flow.execution.entrance.node.NodeExecutionState.NodeExecutionState - -/** - * Created by peacewong on 2019/11/5. - */ -trait NodeRunnerListener { - - def onStatusChanged(fromState: NodeExecutionState, toState: NodeExecutionState, node: SchedulerNode): Unit -} diff --git a/dss-flow-execution-entrance/src/main/scala/com/webank/wedatasphere/dss/flow/execution/entrance/log/FlowExecutionLog.scala b/dss-flow-execution-entrance/src/main/scala/com/webank/wedatasphere/dss/flow/execution/entrance/log/FlowExecutionLog.scala deleted file mode 100644 index 8a317bbae5..0000000000 --- a/dss-flow-execution-entrance/src/main/scala/com/webank/wedatasphere/dss/flow/execution/entrance/log/FlowExecutionLog.scala +++ /dev/null @@ -1,40 +0,0 @@ -/* - * Copyright 2019 WeBank - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package com.webank.wedatasphere.dss.flow.execution.entrance.log - - -import com.webank.wedatasphere.dss.flow.execution.entrance.conf.FlowExecutionEntranceConfiguration -import com.webank.wedatasphere.dss.linkis.node.execution.log.LinkisJobExecutionLog -import com.webank.wedatasphere.linkis.common.utils.Logging - -/** - * Created by peacewong on 2019/11/5. - */ -class FlowExecutionLog(log:Logging) extends LinkisJobExecutionLog { - override def info(message: scala.Any, t: Throwable): Unit = { - if(message != null && FlowExecutionEntranceConfiguration.NODE_PRINT_FLAG.getValue) log.info(message.toString, t) - } - - override def warn(message: scala.Any, t: Throwable): Unit = { - if(message != null && FlowExecutionEntranceConfiguration.NODE_PRINT_FLAG.getValue) log.warn(message.toString, t) - } - - override def error(message: scala.Any, t: Throwable): Unit = { - if(message != null && FlowExecutionEntranceConfiguration.NODE_PRINT_FLAG.getValue) log.error(message.toString, t) - } -} diff --git a/dss-flow-execution-entrance/src/main/scala/com/webank/wedatasphere/dss/flow/execution/entrance/node/AppJointJobBuilder.scala b/dss-flow-execution-entrance/src/main/scala/com/webank/wedatasphere/dss/flow/execution/entrance/node/AppJointJobBuilder.scala deleted file mode 100644 index f8173adfb6..0000000000 --- a/dss-flow-execution-entrance/src/main/scala/com/webank/wedatasphere/dss/flow/execution/entrance/node/AppJointJobBuilder.scala +++ /dev/null @@ -1,180 +0,0 @@ -/* - * Copyright 2019 WeBank - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package com.webank.wedatasphere.dss.flow.execution.entrance.node - -import java.{lang, util} - -import com.webank.wedatasphere.dss.appjoint.scheduler.entity.{ReadNode, SchedulerNode, ShareNode} -import com.webank.wedatasphere.dss.common.entity.Resource -import com.webank.wedatasphere.dss.flow.execution.entrance.conf.FlowExecutionEntranceConfiguration -import com.webank.wedatasphere.dss.flow.execution.entrance.job._ -import com.webank.wedatasphere.dss.flow.execution.entrance.utils.FlowExecutionUtils -import com.webank.wedatasphere.dss.linkis.node.execution.conf.LinkisJobExecutionConfiguration -import com.webank.wedatasphere.dss.linkis.node.execution.entity.BMLResource -import com.webank.wedatasphere.dss.linkis.node.execution.execution.impl.LinkisNodeExecutionImpl -import com.webank.wedatasphere.dss.linkis.node.execution.job._ -import com.webank.wedatasphere.dss.linkis.node.execution.parser.JobParamsParser -import org.apache.commons.lang.StringUtils - -/** - * Created by johnnwang on 2019/11/5. - */ -object AppJointJobBuilder { - - val signalKeyCreator = new FlowExecutionJobSignalKeyCreator - - init() - - def init(): Unit ={ - val jobParamsParser = new JobParamsParser - - jobParamsParser.setSignalKeyCreator(signalKeyCreator) - - LinkisNodeExecutionImpl.getLinkisNodeExecution.asInstanceOf[LinkisNodeExecutionImpl].registerJobParser(jobParamsParser) - } - - def builder():FlowBuilder = new FlowBuilder - - class FlowBuilder extends Builder { - - - private var jobProps: java.util.Map[String, String] = _ - - private var node: SchedulerNode = _ - - def setJobProps(jobProps: java.util.Map[String, String]): FlowBuilder = { - this.jobProps = jobProps - this - } - - def setNode(node: SchedulerNode): FlowBuilder = { - this.node = node - this - } - - override protected def getJobType: String = jobProps.getOrDefault(LinkisJobExecutionConfiguration.LINKIS_TYPE, LinkisJobExecutionConfiguration.LINKIS_DEFAULT_TYPE.getValue(jobProps)) - - override protected def isReadNode: lang.Boolean = this.node.isInstanceOf[ReadNode] - - override protected def isShareNod: lang.Boolean = this.node.isInstanceOf[ShareNode] - - override protected def isSignalSharedNode: lang.Boolean = { - FlowExecutionUtils.isSignalNode(getJobType) - } - - override protected def createReadJob(isLinkisType: Boolean): ReadJob = { - if(isLinkisType){ - val readJob = new FlowExecutionCommonLinkisReadJob - readJob.setReadNode(this.node.asInstanceOf[ReadNode]) - readJob.setJobProps(this.jobProps) - readJob - } else { - val readJob = new FlowExecutionAppJointLinkisReadJob - readJob.setReadNode(this.node.asInstanceOf[ReadNode]) - readJob.setJobProps(this.jobProps) - readJob - } - } - - - override protected def createSharedJob(isLinkisType: Boolean): SharedJob = { - if(isLinkisType){ - val sharedJob = new FlowExecutionCommonLinkisSharedJob - sharedJob.setJobProps(this.jobProps) - sharedJob.setSharedNum(this.node.asInstanceOf[ShareNode].getShareTimes) - sharedJob - } else { - val sharedJob = new FlowExecutionAppJointLinkisSharedJob - sharedJob.setJobProps(this.jobProps) - sharedJob.setSharedNum(this.node.asInstanceOf[ShareNode].getShareTimes) - sharedJob - } - } - - - - override protected def createSignalSharedJob(isLinkisType: Boolean): SignalSharedJob = { - if(isLinkisType){ - null - } else { - val signalJob = new FlowExecutionAppJointSignalSharedJob - signalJob.setSignalKeyCreator(signalKeyCreator) - signalJob.setJobProps(this.jobProps) - signalJob - } - } - - override protected def creatLinkisJob(isLinkisType: Boolean): LinkisJob = { - if(isLinkisType){ - val linkisJob = new FlowExecutionCommonLinkisJob - linkisJob.setJobProps(this.jobProps) - linkisJob - } else { - val linkisJob = new FlowExecutionAppJointLinkisJob - linkisJob.setJobProps(this.jobProps) - linkisJob - } - } - - override protected def fillJobInfo(job: Job): Unit = { - if(StringUtils.isNotEmpty(jobProps.get(FlowExecutionEntranceConfiguration.COMMAND))) - job.setCode(jobProps.get(FlowExecutionEntranceConfiguration.COMMAND)) - job.setParams(new util.HashMap[String,AnyRef]()) - job.setRuntimeParams(new util.HashMap[String,AnyRef]()) - } - - override protected def fillLinkisJobInfo(linkisJob: LinkisJob): Unit = { - this.node.getDssNode.getParams.get(FlowExecutionEntranceConfiguration.NODE_CONFIGURATION_KEY) match { - case configuration:util.Map[String, AnyRef] => - linkisJob.setConfiguration(configuration) - case _ => - } - this.node.getDssNode.getParams.remove(FlowExecutionEntranceConfiguration.FLOW_VAR_MAP) match { - case flowVar:util.Map[String, AnyRef] => - linkisJob.setVariables(flowVar) - case _ => - } - linkisJob.setSource(getSource) - } - - override protected def fillCommonLinkisJobInfo(linkisAppjointJob: CommonLinkisJob): Unit = { - linkisAppjointJob.setJobResourceList(FlowExecutionUtils.resourcesAdaptation(this.node.getDssNode.getResources)) - this.node.getDssNode.getParams.remove(FlowExecutionEntranceConfiguration.PROJECT_RESOURCES) match { - case projectResources:util.List[Resource] => - linkisAppjointJob.setProjectResourceList(FlowExecutionUtils.resourcesAdaptation(projectResources)) - case _ => - } - this.node.getDssNode.getParams.remove(FlowExecutionEntranceConfiguration.FLOW_RESOURCES) match { - case flowResources:util.HashMap[String, util.List[BMLResource]] => - linkisAppjointJob.setFlowNameAndResources(flowResources) - case _ => - } - } - - - private def getSource: util.Map[String, String] = { - val source = new util.HashMap[String, String]() - source.put("projectName", jobProps.get(FlowExecutionEntranceConfiguration.PROJECT_NAME)) - source.put("flowName", jobProps.get(FlowExecutionEntranceConfiguration.FLOW_NAME)) - source.put("nodeName", jobProps.get(FlowExecutionEntranceConfiguration.JOB_ID)) - source - } - - - } -} diff --git a/dss-flow-execution-entrance/src/main/scala/com/webank/wedatasphere/dss/flow/execution/entrance/node/DefaultNodeRunner.scala b/dss-flow-execution-entrance/src/main/scala/com/webank/wedatasphere/dss/flow/execution/entrance/node/DefaultNodeRunner.scala deleted file mode 100644 index fbd13c8493..0000000000 --- a/dss-flow-execution-entrance/src/main/scala/com/webank/wedatasphere/dss/flow/execution/entrance/node/DefaultNodeRunner.scala +++ /dev/null @@ -1,164 +0,0 @@ -/* - * Copyright 2019 WeBank - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package com.webank.wedatasphere.dss.flow.execution.entrance.node - -import java.util - -import com.webank.wedatasphere.dss.appjoint.scheduler.entity.SchedulerNode -import com.webank.wedatasphere.dss.flow.execution.entrance.conf.FlowExecutionEntranceConfiguration -import com.webank.wedatasphere.dss.linkis.node.execution.job.{JobTypeEnum, LinkisJob} -import com.webank.wedatasphere.dss.flow.execution.entrance.listener.NodeRunnerListener -import com.webank.wedatasphere.dss.flow.execution.entrance.log.FlowExecutionLog -import com.webank.wedatasphere.dss.flow.execution.entrance.node.NodeExecutionState.NodeExecutionState -import com.webank.wedatasphere.dss.linkis.node.execution.execution.impl.LinkisNodeExecutionImpl -import com.webank.wedatasphere.dss.linkis.node.execution.listener.LinkisExecutionListener -import com.webank.wedatasphere.linkis.common.exception.ErrorException -import com.webank.wedatasphere.linkis.common.utils.{Logging, Utils} - -/** - * Created by johnnwang on 2019/11/5. - */ -class DefaultNodeRunner extends NodeRunner with Logging { - - private var node: SchedulerNode = _ - - private var linkisJob: LinkisJob = _ - - private var canceled: Boolean = false - - private var status: NodeExecutionState = NodeExecutionState.Inited - - private var nodeRunnerListener: NodeRunnerListener = _ - - private var executedInfo: String = _ - - private var startTime: Long = _ - - private var nowTime:Long = _ - - private var lastGetStatusTime: Long = 0 - - override def getNode: SchedulerNode = this.node - - def setNode(schedulerNode: SchedulerNode): Unit = { - this.node = schedulerNode - } - - override def getLinkisJob: LinkisJob = this.linkisJob - - - override def isCanceled: Boolean = this.canceled - - override def getStatus: NodeExecutionState = { - this.status - } - - override def isLinkisJobCompleted: Boolean = Utils.tryCatch{ - - val interval = System.currentTimeMillis() - lastGetStatusTime - - if ( interval < FlowExecutionEntranceConfiguration.NODE_STATUS_INTERVAL.getValue){ - return false - } - - lastGetStatusTime = System.currentTimeMillis() - - if(NodeExecutionState.isCompleted(getStatus)) return true - val toState = NodeExecutionState.withName(LinkisNodeExecutionImpl.getLinkisNodeExecution.getState(this.linkisJob)) - if (NodeExecutionState.isCompleted(toState)) { - val listener = LinkisNodeExecutionImpl.getLinkisNodeExecution.asInstanceOf[LinkisExecutionListener] - listener.onStatusChanged(getStatus.toString, toState.toString, this.linkisJob) - this.transitionState(toState) - info(s"Finished to execute node of ${this.node.getName}") - true - } else { - false - } - }{ - case e:ErrorException => logger.warn(s"failed to get ${this.node.getName} state", e) - false - case t :Throwable => logger.error(s"failed to get ${this.node.getName} state", t) - true - } - - override def setNodeRunnerListener(nodeRunnerListener: NodeRunnerListener): Unit = this.nodeRunnerListener = nodeRunnerListener - - override def getNodeRunnerListener: NodeRunnerListener = this.nodeRunnerListener - - override def run(): Unit = { - info(s"start to run node of ${node.getName}") - try { - val jobProps = node.getDssNode.getParams.remove(FlowExecutionEntranceConfiguration.PROPS_MAP) match { - case propsMap: util.Map[String, String] => propsMap - case _ => new util.HashMap[String, String]() - } - this.linkisJob = AppJointJobBuilder.builder().setNode(node).setJobProps(jobProps).build.asInstanceOf[LinkisJob] - this.linkisJob.setLogObj(new FlowExecutionLog(this)) - //set start time - this.setStartTime(System.currentTimeMillis()) - if (JobTypeEnum.EmptyJob == this.linkisJob.getJobType) { - warn("This node is empty type") - this.transitionState(NodeExecutionState.Succeed) - return - } - - LinkisNodeExecutionImpl.getLinkisNodeExecution.runJob(this.linkisJob) - info(s"start to run node of ${node.getName}") - /*LinkisNodeExecutionImpl.getLinkisNodeExecution.waitForComplete(this.linkisJob) - val listener = LinkisNodeExecutionImpl.getLinkisNodeExecution.asInstanceOf[LinkisExecutionListener] - val toState = LinkisNodeExecutionImpl.getLinkisNodeExecution.getState(this.linkisJob) - listener.onStatusChanged(getStatus.toString, toState, this.linkisJob) - this.transitionState(NodeExecutionState.withName(toState)) - info(s"Finished to execute node of ${node.getName}")*/ - } catch { - case t: Throwable => - warn(s"Failed to execute node of ${node.getName}", t) - this.transitionState(NodeExecutionState.Failed) - - } - - } - - - override def cancel(): Unit = if (!this.canceled) this synchronized { - if (this.canceled) return - this.canceled = true - if (this.linkisJob != null && this.linkisJob.getJobExecuteResult != null) - LinkisNodeExecutionImpl.getLinkisNodeExecution.cancel(this.linkisJob) - this.transitionState(NodeExecutionState.Cancelled) - warn(s"This node(${node.getName}) has been canceled") - } - - override def pause(): Unit = {} - - override def resume(): Boolean = true - - override def setStatus(nodeExecutionState: NodeExecutionState): Unit = this.status = nodeExecutionState - - override def getNodeExecutedInfo(): String = this.executedInfo - - override def setNodeExecutedInfo(info: String): Unit = this.executedInfo = info - - override def getStartTime(): Long = this.startTime - - override def setStartTime(startTime: Long): Unit = this.startTime = startTime - - override def getNowTime(): Long = this.nowTime - - override def setNowTime(nowTime: Long): Unit = this.nowTime = nowTime -} diff --git a/dss-flow-execution-entrance/src/main/scala/com/webank/wedatasphere/dss/flow/execution/entrance/node/NodeExecutionState.scala b/dss-flow-execution-entrance/src/main/scala/com/webank/wedatasphere/dss/flow/execution/entrance/node/NodeExecutionState.scala deleted file mode 100644 index 2df6ff3205..0000000000 --- a/dss-flow-execution-entrance/src/main/scala/com/webank/wedatasphere/dss/flow/execution/entrance/node/NodeExecutionState.scala +++ /dev/null @@ -1,44 +0,0 @@ -/* - * Copyright 2019 WeBank - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package com.webank.wedatasphere.dss.flow.execution.entrance.node - - -/** - * Created by peacewong on 2019/11/5. - */ -object NodeExecutionState extends Enumeration { - - type NodeExecutionState = Value - - val Inited, WaitForRetry, Scheduled, Running, Succeed, Failed, Cancelled, Skipped ,Timeout = Value - - def isRunning(jobState: NodeExecutionState) = Running == jobState - - def isScheduled(jobState: NodeExecutionState) = Scheduled == jobState - - def isInited(jobState: NodeExecutionState) = Inited == jobState - - def isCompleted(jobState: NodeExecutionState) = jobState match { - case Inited | Scheduled | Running | WaitForRetry => false - case _ => true - } - - def isSucceed(jobState: NodeExecutionState) = Succeed == jobState - - def isFailed(jobState: NodeExecutionState):Boolean = Failed == jobState -} diff --git a/dss-flow-execution-entrance/src/main/scala/com/webank/wedatasphere/dss/flow/execution/entrance/node/NodeRunner.scala b/dss-flow-execution-entrance/src/main/scala/com/webank/wedatasphere/dss/flow/execution/entrance/node/NodeRunner.scala deleted file mode 100644 index f0a40fd57a..0000000000 --- a/dss-flow-execution-entrance/src/main/scala/com/webank/wedatasphere/dss/flow/execution/entrance/node/NodeRunner.scala +++ /dev/null @@ -1,105 +0,0 @@ -/* - * Copyright 2019 WeBank - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package com.webank.wedatasphere.dss.flow.execution.entrance.node - -import com.webank.wedatasphere.dss.linkis.node.execution.job.LinkisJob -import com.webank.wedatasphere.dss.flow.execution.entrance.FlowContext -import com.webank.wedatasphere.dss.flow.execution.entrance.listener.NodeRunnerListener -import com.webank.wedatasphere.dss.flow.execution.entrance.node.NodeExecutionState.NodeExecutionState -import com.webank.wedatasphere.linkis.common.utils.{Logging, Utils} -import java.util.concurrent.Future - -import com.webank.wedatasphere.dss.appjoint.scheduler.entity.SchedulerNode - -/** - * Created by peacewong on 2019/11/5. - */ -abstract class NodeRunner extends Runnable with Logging{ - - private[flow] var future: Future[_] = _ - - private var flowContext:FlowContext = _ - - - - def getFlowContext:FlowContext = this.flowContext - - def setFlowContext(flowContext: FlowContext): Unit = { - this.flowContext = flowContext - } - - def getNode:SchedulerNode - - def setNode(node: SchedulerNode):Unit - - def getLinkisJob: LinkisJob - - def cancel(): Unit - - def pause(): Unit - - def resume(): Boolean - - def isCanceled: Boolean - - def getStatus: NodeExecutionState - - def isLinkisJobCompleted: Boolean - - def setStatus(nodeExecutionState: NodeExecutionState): Unit - - def setNodeRunnerListener(nodeRunnerListener: NodeRunnerListener): Unit - - def getNodeRunnerListener: NodeRunnerListener - - def getNodeExecutedInfo(): String - - def setNodeExecutedInfo(info: String ):Unit - - def getStartTime(): Long - - def setStartTime(startTime: Long): Unit - - def getNowTime():Long - - def setNowTime(nowTime: Long):Unit - - protected def transitionState(toState: NodeExecutionState): Unit = Utils.tryAndWarn{ - if (getStatus == toState) return - info(s"from state $getStatus to $toState") - this.getNodeRunnerListener.onStatusChanged(getStatus, toState, this.getNode) - this.setStatus(toState) - } - - - def tunToScheduled(): Boolean = if (! NodeExecutionState.isInited(this.getStatus)) false else this synchronized { - if (! NodeExecutionState.isInited(this.getStatus)) false else { - transitionState(NodeExecutionState.Scheduled) - true - } - } - - def fromScheduledTunToState(state: NodeExecutionState): Boolean = if (! NodeExecutionState.isScheduled(this.getStatus) ) false else this synchronized { - if (! NodeExecutionState.isScheduled(this.getStatus)) false else { - transitionState(state) - true - } - } - - -} diff --git a/dss-flow-execution-entrance/src/main/scala/com/webank/wedatasphere/dss/flow/execution/entrance/resolver/FlowDependencyResolver.scala b/dss-flow-execution-entrance/src/main/scala/com/webank/wedatasphere/dss/flow/execution/entrance/resolver/FlowDependencyResolver.scala deleted file mode 100644 index 424d1424ee..0000000000 --- a/dss-flow-execution-entrance/src/main/scala/com/webank/wedatasphere/dss/flow/execution/entrance/resolver/FlowDependencyResolver.scala +++ /dev/null @@ -1,28 +0,0 @@ -/* - * Copyright 2019 WeBank - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package com.webank.wedatasphere.dss.flow.execution.entrance.resolver - -import com.webank.wedatasphere.dss.flow.execution.entrance.job.FlowEntranceJob - -/** - * created by chaogefeng on 2019/11/5 10:20 - * Description: - */ -trait FlowDependencyResolver { - def resolvedFlow(flowJob:FlowEntranceJob) -} diff --git a/dss-flow-execution-entrance/src/main/scala/com/webank/wedatasphere/dss/flow/execution/entrance/resolver/FlowDependencyResolverImpl.scala b/dss-flow-execution-entrance/src/main/scala/com/webank/wedatasphere/dss/flow/execution/entrance/resolver/FlowDependencyResolverImpl.scala deleted file mode 100644 index ecd93ffd1c..0000000000 --- a/dss-flow-execution-entrance/src/main/scala/com/webank/wedatasphere/dss/flow/execution/entrance/resolver/FlowDependencyResolverImpl.scala +++ /dev/null @@ -1,66 +0,0 @@ -/* - * Copyright 2019 WeBank - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package com.webank.wedatasphere.dss.flow.execution.entrance.resolver - -import java.util - -import com.webank.wedatasphere.dss.flow.execution.entrance.{FlowContext} -import com.webank.wedatasphere.dss.flow.execution.entrance.job.FlowEntranceJob - -import scala.collection.JavaConversions._ -import com.webank.wedatasphere.linkis.common.utils.Logging -import org.springframework.stereotype.Component - - -/** - * created by chaogefeng on 2019/11/5 10:24 - * Description: - */ -@Component -class FlowDependencyResolverImpl extends FlowDependencyResolver with Logging { - override def resolvedFlow(flowJob: FlowEntranceJob) = { - - info(s"${flowJob.getId} Start to get executable node") - - val flowContext: FlowContext = flowJob.getFlowContext - val nodes = flowContext.getPendingNodes.toMap.values.map(_.getNode) - - def isAllParentDependencyCompleted(parents:util.List[String]): Boolean = { - for (parent <- parents){ - if( ! flowContext.isNodeCompleted(parent)) return false - } - true - } - nodes.foreach{ node => - val nodeName = node.getName - def isCanExecutable: Boolean = { - (flowContext.getPendingNodes.containsKey(nodeName) - && !FlowContext.isNodeRunning(nodeName, flowContext) - && !flowContext.isNodeCompleted(nodeName) - && isAllParentDependencyCompleted(node.getDependencys)) - } - if (isCanExecutable) flowContext synchronized { - if (isCanExecutable) flowContext.getPendingNodes.get(nodeName).tunToScheduled() - } - } - info(s"${flowJob.getId} Finished to get executable node(${flowContext.getScheduledNodes.size()})") - - } - - -} \ No newline at end of file diff --git a/dss-flow-execution-entrance/src/main/scala/com/webank/wedatasphere/dss/flow/execution/entrance/utils/FlowExecutionUtils.scala b/dss-flow-execution-entrance/src/main/scala/com/webank/wedatasphere/dss/flow/execution/entrance/utils/FlowExecutionUtils.scala deleted file mode 100644 index 46ad833417..0000000000 --- a/dss-flow-execution-entrance/src/main/scala/com/webank/wedatasphere/dss/flow/execution/entrance/utils/FlowExecutionUtils.scala +++ /dev/null @@ -1,92 +0,0 @@ -/* - * Copyright 2019 WeBank - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package com.webank.wedatasphere.dss.flow.execution.entrance.utils - -import java.util - -import com.webank.wedatasphere.dss.appjoint.scheduler.entity.SchedulerNode -import com.webank.wedatasphere.dss.common.entity.Resource -import com.webank.wedatasphere.dss.flow.execution.entrance.conf.FlowExecutionEntranceConfiguration -import com.webank.wedatasphere.dss.linkis.node.execution.WorkflowContext -import com.webank.wedatasphere.dss.linkis.node.execution.conf.LinkisJobExecutionConfiguration -import com.webank.wedatasphere.dss.linkis.node.execution.entity.BMLResource - -import scala.collection.JavaConversions._ - -/** - * Created by peacewong on 2019/11/6. - */ -object FlowExecutionUtils { - - def isSkippedNode(node:SchedulerNode):Boolean = { - FlowExecutionEntranceConfiguration.SKIP_NODES.getValue.split(",").exists(_.equalsIgnoreCase(node.getNodeType)) - } - - - def isSignalNode(jobType: String) : Boolean = { - FlowExecutionEntranceConfiguration.SIGNAL_NODES.getValue.split(",").exists(_.equalsIgnoreCase(jobType)) - } - - def isAppJointJob(engineType: String): Boolean = LinkisJobExecutionConfiguration.APPJOINT.equalsIgnoreCase(engineType) - - - def getSharedInfo(jobProps: util.Map[String, String], nodeId: String): AnyRef = WorkflowContext.getAppJointContext.getValue(getSharedKey(jobProps, nodeId)) - - def getSharedNodesAndJobId(jobProps: util.Map[String, String], nodes: Array[String]): util.Map[String, AnyRef] = { - val map: util.Map[String, AnyRef] = new util.HashMap[String, AnyRef] - for (nodeId <- nodes) { - map.put(nodeId, getSharedInfo(jobProps, nodeId)) - } - map - } - - def getSharedKey(jobProps: util.Map[String, String], nodeId: String): String = { - val projectId: String = jobProps.get(FlowExecutionEntranceConfiguration.PROJECT_NAME) - val flowId: String = jobProps.get(FlowExecutionEntranceConfiguration.FLOW_NAME) - val flowExecId: String = jobProps.get(FlowExecutionEntranceConfiguration.FLOW_EXEC_ID) - projectId + "." + flowId + "." + flowExecId + "." + nodeId - } - - - def setSharedInfo(jobProps: util.Map[String, String], jobId: String): Unit = { - val nodeId: String = jobProps.get(FlowExecutionEntranceConfiguration.JOB_ID) - val shareNum: Int = jobProps.get(FlowExecutionEntranceConfiguration.SHARED_NODE_TOKEN).toInt - WorkflowContext.getAppJointContext.setValue(getSharedKey(jobProps, nodeId), jobId, shareNum) - } - - def resourcesAdaptation(resources: util.List[Resource]): util.ArrayList[BMLResource] = { - val bmlResources = new util.ArrayList[BMLResource]() - for (resource <- resources){ - bmlResources.add(resourceConvertToBMLResource(resource)) - } - bmlResources - } - - def resourceConvertToBMLResource(resource: Resource):BMLResource = { - val bmlResource = new BMLResource() - bmlResource.setFileName(resource.getFileName) - bmlResource.setResourceId(resource.getResourceId) - bmlResource.setVersion(resource.getVersion) - bmlResource - } - - def isReadNode(nodeType: String): Boolean = { - FlowExecutionEntranceConfiguration.EMAIL_TYPE.equalsIgnoreCase(nodeType) - } - -} diff --git a/dss-linkis-node-execution/pom.xml b/dss-linkis-node-execution/pom.xml deleted file mode 100644 index f773b12f24..0000000000 --- a/dss-linkis-node-execution/pom.xml +++ /dev/null @@ -1,69 +0,0 @@ - - - - - 4.0.0 - - - dss - com.webank.wedatasphere.dss - 0.9.1 - - - dss-linkis-node-execution - - - - com.webank.wedatasphere.linkis - linkis-ujes-client - ${linkis.version} - - - - com.webank.wedatasphere.linkis - linkis-workspace-httpclient - ${linkis.version} - - - - - - - - - - org.apache.maven.plugins - maven-compiler-plugin - 3.2 - - 1.8 - 1.8 - UTF-8 - - - - - org.apache.maven.plugins - maven-deploy-plugin - - - - - \ No newline at end of file diff --git a/dss-linkis-node-execution/src/main/java/com/webank/wedatasphere/dss/linkis/node/execution/WorkflowContext.java b/dss-linkis-node-execution/src/main/java/com/webank/wedatasphere/dss/linkis/node/execution/WorkflowContext.java deleted file mode 100644 index a10b6962a5..0000000000 --- a/dss-linkis-node-execution/src/main/java/com/webank/wedatasphere/dss/linkis/node/execution/WorkflowContext.java +++ /dev/null @@ -1,44 +0,0 @@ -/* - * Copyright 2019 WeBank - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package com.webank.wedatasphere.dss.linkis.node.execution; - -import java.util.Iterator; -import java.util.List; -import java.util.Map; - -/** - * Created by enjoyyin on 2019/10/29. - */ -public interface WorkflowContext { - - Object getValue(String key); - - List getValues(String prefix); - - Iterator keyIterator(); - - Map getSubMapByPrefix(String keyPrefix); - - - void setValue(String key, Object value, int readTimes); - - static WorkflowContext getAppJointContext() { - return WorkflowContextFactory.getWorkflowContext(); - } - -} \ No newline at end of file diff --git a/dss-linkis-node-execution/src/main/java/com/webank/wedatasphere/dss/linkis/node/execution/WorkflowContextFactory.java b/dss-linkis-node-execution/src/main/java/com/webank/wedatasphere/dss/linkis/node/execution/WorkflowContextFactory.java deleted file mode 100644 index fdc2ed1e7f..0000000000 --- a/dss-linkis-node-execution/src/main/java/com/webank/wedatasphere/dss/linkis/node/execution/WorkflowContextFactory.java +++ /dev/null @@ -1,59 +0,0 @@ -/* - * Copyright 2019 WeBank - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package com.webank.wedatasphere.dss.linkis.node.execution; - -import com.webank.wedatasphere.dss.linkis.node.execution.conf.LinkisJobExecutionConfiguration; -import com.webank.wedatasphere.dss.linkis.node.execution.exception.LinkisJobExecutionWarnException; -import org.apache.commons.lang.ClassUtils; -import org.apache.commons.lang.StringUtils; - -/** - * Created by enjoyyin on 2019/10/29. - */ -public class WorkflowContextFactory { - - private static Class clazz = WorkflowContextImpl.class; - private static WorkflowContext workflowContext = null; - - public static void register(Class clazz) { - WorkflowContextFactory.clazz = clazz; - } - - static WorkflowContext getWorkflowContext() { - if(workflowContext == null) { - synchronized (WorkflowContextFactory.class) { - if(workflowContext == null) { - String className = LinkisJobExecutionConfiguration.APPJOINT_CONTEXT_CLASS.acquireNew(); - if(clazz == WorkflowContextImpl.class && StringUtils.isNotBlank(className)) { - try { - clazz = ClassUtils.getClass(className); - } catch (ClassNotFoundException e) { - throw new LinkisJobExecutionWarnException(90150, "find class + " + className + " failed!", e); - } - } - try { - workflowContext = clazz.newInstance(); - } catch (Exception e) { - throw new LinkisJobExecutionWarnException(90150, "initial class + " + className + " failed!", e); - } - } - } - } - return workflowContext; - } -} diff --git a/dss-linkis-node-execution/src/main/java/com/webank/wedatasphere/dss/linkis/node/execution/WorkflowContextImpl.java b/dss-linkis-node-execution/src/main/java/com/webank/wedatasphere/dss/linkis/node/execution/WorkflowContextImpl.java deleted file mode 100644 index 8f2ff84fac..0000000000 --- a/dss-linkis-node-execution/src/main/java/com/webank/wedatasphere/dss/linkis/node/execution/WorkflowContextImpl.java +++ /dev/null @@ -1,96 +0,0 @@ -/* - * Copyright 2019 WeBank - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package com.webank.wedatasphere.dss.linkis.node.execution; - -import com.google.common.cache.Cache; -import com.google.common.cache.CacheBuilder; -import com.webank.wedatasphere.dss.linkis.node.execution.conf.LinkisJobExecutionConfiguration; -import com.webank.wedatasphere.dss.linkis.node.execution.entity.ContextInfo; -import org.apache.commons.lang.StringUtils; - -import java.util.*; -import java.util.concurrent.TimeUnit; - -/** - * Created by johnnwang on 2019/9/26. - */ -public class WorkflowContextImpl implements WorkflowContext { - - private Cache contextCache = CacheBuilder.newBuilder() - .maximumSize(LinkisJobExecutionConfiguration.LINKIS_CACHE_MAX_SIZE.getValue()) - .expireAfterWrite(LinkisJobExecutionConfiguration.LINKIS_CACHE_EXPIRE_TIME.getValue(), TimeUnit.HOURS) - .build(); - - - @Override - public Object getValue(String key) { - ContextInfo contextInfo = contextCache.getIfPresent(key); - if (contextInfo != null && contextInfo.getReadNum() > 0) { - int readNum = contextInfo.getReadNum() - 1; - Object value = contextInfo.getValue(); - if (readNum <= 0) { - contextCache.invalidate(key); - } - contextInfo.setReadNum(contextInfo.getReadNum() - 1); - return value; - } - if (contextInfo != null && contextInfo.getReadNum() < 0) { - return contextInfo.getValue(); - } - return null; - } - - @Override - public List getValues(String prefix) { - List values = new ArrayList<>(); - Iterator keys = keyIterator(); - while (keys.hasNext()) { - String key = keys.next(); - if (key.startsWith(prefix)) { - values.add(getValue(key)); - } - } - return values; - } - - @Override - public Iterator keyIterator() { - return contextCache.asMap().keySet().iterator(); - } - - @Override - public Map getSubMapByPrefix(String keyPrefix) { - Map map = new HashMap<>(); - Iterator keys = keyIterator(); - while (keys.hasNext()) { - String key = keys.next(); - if (key.startsWith(keyPrefix)) { - map.put(StringUtils.substringAfter(key, keyPrefix), getValue(key)); - } - } - return map; - } - - @Override - public void setValue(String key, Object value, int readNum) { - - contextCache.put(key, new ContextInfo(value, readNum)); - - } - -} diff --git a/dss-linkis-node-execution/src/main/java/com/webank/wedatasphere/dss/linkis/node/execution/conf/LinkisJobExecutionConfiguration.java b/dss-linkis-node-execution/src/main/java/com/webank/wedatasphere/dss/linkis/node/execution/conf/LinkisJobExecutionConfiguration.java deleted file mode 100644 index 9b75d73a01..0000000000 --- a/dss-linkis-node-execution/src/main/java/com/webank/wedatasphere/dss/linkis/node/execution/conf/LinkisJobExecutionConfiguration.java +++ /dev/null @@ -1,101 +0,0 @@ -/* - * Copyright 2019 WeBank - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package com.webank.wedatasphere.dss.linkis.node.execution.conf; - - - -import com.webank.wedatasphere.linkis.common.conf.CommonVars; - -/** - * Created by peacewong on 2019/9/19. - */ -public class LinkisJobExecutionConfiguration { - - - public static final String LINKIS_TYPE = "linkistype"; - - public static final String APPJOINT = "appjoint"; - - - - - - - - - public static final String PROJECT_PREFIX = "project"; - public static final String FLOW_PREFIX = "flow"; - public static final String JOB_PREFIX = "job"; - - public static final String FLOW_VARIABLE_PREFIX = "flow.variable."; - - public static final String NODE_CONF_PREFIX = "node.conf."; - - public static final String CONF_STARTUP = "startup"; - public static final String CONF_RUNTIME = "runtime"; - public static final String CONF_SPECIAL = "special"; - - public static final String RESOURCES_NAME = ".resources"; - - - public static final String WORKFLOW_SHARED_NODES_JOBIDS = "workflow.shared.nodes.jobids"; - - public static final String LINKIS_SUBMIT_USER = "wds.linkis.schedulis.submit.user"; - - public static final String LINKIS_CONTROL_EMPTY_NODE = "linkis.control.empty"; - - public static final String LINKIS_DATA_EXCHANGE_NODE = "linkis.data.exchange"; - - public final static CommonVars JOB_DEFAULT_TYPE = CommonVars.apply("wds.linkis.flow.default.job.type","linkis"); - - public final static CommonVars LINKIS_DEFAULT_TYPE = CommonVars.apply("wds.linkis.flow.default.linkis.type","spark.sql"); - - - public final static CommonVars LINKIS_CONNECTION_TIMEOUT = CommonVars.apply("wds.linkis.flow.connection.timeout",30000); - - public final static CommonVars LINKIS_JOB_REQUEST_STATUS_TIME = CommonVars.apply("wds.linkis.flow.connection.timeout",3000); - - public final static CommonVars LINKIS_ADMIN_USER = CommonVars.apply("wds.linkis.client.flow.adminuser","ws"); - - - - - - public final static CommonVars LINKIS_AUTHOR_USER_TOKEN = CommonVars.apply("wds.linkis.client.flow.author.user.token","WS-AUTH"); - - public final static CommonVars LINKIS_JOB_CREATOR = CommonVars.apply("wds.linkis.flow.job.creator","nodeexecution"); - - public final static CommonVars LINKIS_URL = CommonVars.apply("wds.linkis.gateway.url","wtss"); - - public final static CommonVars LINKIS_API_VERSION = CommonVars.apply("wds.linkis.server.version","v1"); - - public final static CommonVars LINKIS_CACHE_MAX_SIZE = CommonVars.apply("wds.linkis.context.cache.max.size",100000); - - public final static CommonVars LINKIS_CACHE_EXPIRE_TIME = CommonVars.apply("wds.linkis.context.cache.expire.time",24); - - public final static CommonVars RESULT_PRINT_SIZE = CommonVars.apply("wds.linkis.result.print.size",10); - - public final static CommonVars APPJOINT_CONTEXT_CLASS = CommonVars.apply("wds.dss.appjoint.context",""); - - - public final static CommonVars LOG_SIZE = CommonVars.apply("wds.linkis.log.size",-1); - public final static CommonVars LOG_ARRAY_LEN = CommonVars.apply("wds.linkis.log.array.len",4); - - public final static String SUCCEED= "Succeed"; - -} diff --git a/dss-linkis-node-execution/src/main/java/com/webank/wedatasphere/dss/linkis/node/execution/entity/BMLResource.java b/dss-linkis-node-execution/src/main/java/com/webank/wedatasphere/dss/linkis/node/execution/entity/BMLResource.java deleted file mode 100644 index 24e6cd5915..0000000000 --- a/dss-linkis-node-execution/src/main/java/com/webank/wedatasphere/dss/linkis/node/execution/entity/BMLResource.java +++ /dev/null @@ -1,52 +0,0 @@ -/* - * Copyright 2019 WeBank - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package com.webank.wedatasphere.dss.linkis.node.execution.entity; - -/** - * Created by peacewong on 2019/9/21. - */ -public class BMLResource { - - private String resourceId; - private String version; - private String fileName; - - public String getResourceId() { - return resourceId; - } - - public void setResourceId(String resourceId) { - this.resourceId = resourceId; - } - - public String getVersion() { - return version; - } - - public void setVersion(String version) { - this.version = version; - } - - public String getFileName() { - return fileName; - } - - public void setFileName(String fileName) { - this.fileName = fileName; - } -} diff --git a/dss-linkis-node-execution/src/main/java/com/webank/wedatasphere/dss/linkis/node/execution/entity/ContextInfo.java b/dss-linkis-node-execution/src/main/java/com/webank/wedatasphere/dss/linkis/node/execution/entity/ContextInfo.java deleted file mode 100644 index 240bf7cc42..0000000000 --- a/dss-linkis-node-execution/src/main/java/com/webank/wedatasphere/dss/linkis/node/execution/entity/ContextInfo.java +++ /dev/null @@ -1,52 +0,0 @@ -/* - * Copyright 2019 WeBank - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package com.webank.wedatasphere.dss.linkis.node.execution.entity; - -/** - * Created by peacewong on 2019/9/26. - */ -public class ContextInfo { - - - - private Object value; - - private int readNum; - - - public ContextInfo(Object value, int readNum) { - this.value = value; - this.readNum = readNum; - } - - public Object getValue() { - return value; - } - - public void setValue(Object value) { - this.value = value; - } - - public int getReadNum() { - return readNum; - } - - public void setReadNum(int readNum) { - this.readNum = readNum; - } -} diff --git a/dss-linkis-node-execution/src/main/java/com/webank/wedatasphere/dss/linkis/node/execution/exception/LinkisJobExecutionErrorException.java b/dss-linkis-node-execution/src/main/java/com/webank/wedatasphere/dss/linkis/node/execution/exception/LinkisJobExecutionErrorException.java deleted file mode 100644 index 7db0810ad8..0000000000 --- a/dss-linkis-node-execution/src/main/java/com/webank/wedatasphere/dss/linkis/node/execution/exception/LinkisJobExecutionErrorException.java +++ /dev/null @@ -1,30 +0,0 @@ -/* - * Copyright 2019 WeBank - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package com.webank.wedatasphere.dss.linkis.node.execution.exception; - - -import com.webank.wedatasphere.linkis.common.exception.ErrorException; - -/** - * Created by peacewong on 2019/9/19. - */ -public class LinkisJobExecutionErrorException extends ErrorException { - public LinkisJobExecutionErrorException(int errCode, String desc) { - super(errCode, desc); - } -} diff --git a/dss-linkis-node-execution/src/main/java/com/webank/wedatasphere/dss/linkis/node/execution/exception/LinkisJobExecutionWarnException.java b/dss-linkis-node-execution/src/main/java/com/webank/wedatasphere/dss/linkis/node/execution/exception/LinkisJobExecutionWarnException.java deleted file mode 100644 index 5e77e97a56..0000000000 --- a/dss-linkis-node-execution/src/main/java/com/webank/wedatasphere/dss/linkis/node/execution/exception/LinkisJobExecutionWarnException.java +++ /dev/null @@ -1,35 +0,0 @@ -/* - * Copyright 2019 WeBank - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package com.webank.wedatasphere.dss.linkis.node.execution.exception; - -import com.webank.wedatasphere.linkis.common.exception.WarnException; - -/** - * Created by enjoyyin on 2019/10/29. - */ -public class LinkisJobExecutionWarnException extends WarnException { - - public LinkisJobExecutionWarnException(int errCode, String desc) { - super(errCode, desc); - } - - public LinkisJobExecutionWarnException(int errCode, String desc, Throwable cause) { - super(errCode, desc); - initCause(cause); - } -} diff --git a/dss-linkis-node-execution/src/main/java/com/webank/wedatasphere/dss/linkis/node/execution/execution/LinkisNodeExecution.java b/dss-linkis-node-execution/src/main/java/com/webank/wedatasphere/dss/linkis/node/execution/execution/LinkisNodeExecution.java deleted file mode 100644 index 73acc8088b..0000000000 --- a/dss-linkis-node-execution/src/main/java/com/webank/wedatasphere/dss/linkis/node/execution/execution/LinkisNodeExecution.java +++ /dev/null @@ -1,46 +0,0 @@ -/* - * Copyright 2019 WeBank - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package com.webank.wedatasphere.dss.linkis.node.execution.execution; - -import com.webank.wedatasphere.dss.linkis.node.execution.job.Job; - -/** - * Created by peacewong on 2019/11/2. - */ -public interface LinkisNodeExecution { - - void runJob(Job job) throws Exception; - - String getState(Job job); - - String getLog(Job job); - - void waitForComplete(Job job) throws Exception; - - void cancel(Job job) throws Exception; - - double getProgress(Job job); - - Boolean isCompleted(Job job); - - int getResultSize(Job job); - - String getResult(Job job, int index, int maxSize); - - -} diff --git a/dss-linkis-node-execution/src/main/java/com/webank/wedatasphere/dss/linkis/node/execution/execution/impl/LinkisNodeExecutionImpl.java b/dss-linkis-node-execution/src/main/java/com/webank/wedatasphere/dss/linkis/node/execution/execution/impl/LinkisNodeExecutionImpl.java deleted file mode 100644 index 264e8288c4..0000000000 --- a/dss-linkis-node-execution/src/main/java/com/webank/wedatasphere/dss/linkis/node/execution/execution/impl/LinkisNodeExecutionImpl.java +++ /dev/null @@ -1,203 +0,0 @@ -/* - * Copyright 2019 WeBank - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package com.webank.wedatasphere.dss.linkis.node.execution.execution.impl; - -import com.webank.wedatasphere.dss.linkis.node.execution.WorkflowContext; -import com.webank.wedatasphere.dss.linkis.node.execution.exception.LinkisJobExecutionErrorException; -import com.webank.wedatasphere.dss.linkis.node.execution.execution.LinkisNodeExecution; -import com.webank.wedatasphere.dss.linkis.node.execution.job.SharedJob; -import com.webank.wedatasphere.dss.linkis.node.execution.job.SignalSharedJob; -import com.webank.wedatasphere.dss.linkis.node.execution.listener.LinkisExecutionListener; -import com.webank.wedatasphere.dss.linkis.node.execution.parser.JobRuntimeParamsParser; -import com.webank.wedatasphere.dss.linkis.node.execution.service.impl.BuildJobActionImpl; -import com.webank.wedatasphere.dss.linkis.node.execution.conf.LinkisJobExecutionConfiguration; -import com.webank.wedatasphere.dss.linkis.node.execution.job.Job; -import com.webank.wedatasphere.dss.linkis.node.execution.parser.CodeParser; -import com.webank.wedatasphere.dss.linkis.node.execution.parser.JobParser; -import com.webank.wedatasphere.dss.linkis.node.execution.utils.LinkisUjesClientUtils; -import com.webank.wedatasphere.linkis.common.utils.Utils; -import com.webank.wedatasphere.linkis.ujes.client.UJESClient; -import com.webank.wedatasphere.linkis.ujes.client.request.JobExecuteAction; -import com.webank.wedatasphere.linkis.ujes.client.request.ResultSetAction; -import com.webank.wedatasphere.linkis.ujes.client.response.JobInfoResult; -import com.webank.wedatasphere.linkis.ujes.client.response.JobLogResult; -import org.apache.commons.lang.StringUtils; - -import java.util.ArrayList; -import java.util.Map; - -/** - * Created by peacewong on 2019/11/2. - */ -public class LinkisNodeExecutionImpl implements LinkisNodeExecution , LinkisExecutionListener { - - private static LinkisNodeExecution linkisExecution = new LinkisNodeExecutionImpl(); - - private LinkisNodeExecutionImpl() { - registerJobParser(new CodeParser()); - /*registerJobParser(new JobParamsParser());*/ - registerJobParser(new JobRuntimeParamsParser()); - } - - - public static LinkisNodeExecution getLinkisNodeExecution() { - return linkisExecution; - } - - private UJESClient client; - - private ArrayList jobParsers = new ArrayList<>(); - - public UJESClient getClient(Map props) { - if (client == null) { - synchronized (LinkisNodeExecution.class) { - if (client == null) { - client = LinkisUjesClientUtils.getUJESClient( - LinkisJobExecutionConfiguration.LINKIS_URL.getValue(props), - LinkisJobExecutionConfiguration.LINKIS_ADMIN_USER.getValue(props), - LinkisJobExecutionConfiguration.LINKIS_AUTHOR_USER_TOKEN.getValue(props), - props); - } - } - } - return client; - } - - public void registerJobParser(JobParser jobParser) { - this.jobParsers.add(jobParser); - } - - @Override - public void runJob(Job job) throws Exception { - // job parser - for(JobParser parser : jobParsers){ - parser.parseJob(job); - } - JobExecuteAction jobAction = BuildJobActionImpl.getbuildJobAction().getJobAction(job); - job.setJobExecuteResult(getClient(job.getJobProps()).execute(jobAction)); - job.getLogObj().info("<---------------Start to execute job--------------->"); - } - - @Override - public String getState(Job job) { - return getClient(job.getJobProps()).getJobInfo(job.getJobExecuteResult()).getJobStatus(); - } - - @Override - public String getLog(Job job) { - - JobLogResult jobLogResult = getClient(job.getJobProps()) - .log(job.getJobExecuteResult(), - job.getLogFromLine(), - LinkisJobExecutionConfiguration.LOG_SIZE.getValue()); - - job.setLogFromLine(jobLogResult.fromLine()); - - ArrayList logArray = jobLogResult.getLog(); - - if (logArray != null && logArray.size() - >= LinkisJobExecutionConfiguration.LOG_ARRAY_LEN.getValue() - && StringUtils.isNotEmpty(logArray.get(3))) { - return logArray.get(3); - } - return null; - } - - @Override - public void waitForComplete(Job job) throws Exception { - JobInfoResult jobInfo = getClient(job.getJobProps()).getJobInfo(job.getJobExecuteResult()); - while (!jobInfo.isCompleted()) { - job.getLogObj().info("Update Progress info:" + this.getProgress(job)); - job.getLogObj().info("<----linkis log ---->"); - String log = this.getLog(job); - if (log != null) { - job.getLogObj().info(log); - } - job.getLogObj().info("<----linkis log ---->"); - Utils.sleepQuietly(LinkisJobExecutionConfiguration.LINKIS_JOB_REQUEST_STATUS_TIME.getValue(job.getJobProps())); - jobInfo = getClient(job.getJobProps()).getJobInfo(job.getJobExecuteResult()); - } - if (!jobInfo.isSucceed()) { - throw new LinkisJobExecutionErrorException(90101, "Failed to execute Job: " + jobInfo.getMessage()); - } - } - - @Override - public void cancel(Job job) throws Exception { - getClient(job.getJobProps()).kill(job.getJobExecuteResult()); - } - - @Override - public double getProgress(Job job) { - return getClient(job.getJobProps()).progress(job.getJobExecuteResult()).getProgress(); - } - - @Override - public Boolean isCompleted(Job job) { - return getClient(job.getJobProps()).getJobInfo(job.getJobExecuteResult()).isCompleted(); - } - - @Override - public int getResultSize(Job job) { - JobInfoResult jobInfo = getClient(job.getJobProps()).getJobInfo(job.getJobExecuteResult()); - if (jobInfo.isSucceed()) { - String[] resultSetList = jobInfo.getResultSetList(getClient(job.getJobProps())); - if (resultSetList != null && resultSetList.length > 0) { - return resultSetList.length; - } - } - return 0; - } - - @Override - public String getResult(Job job, int index, int maxSize) { - String resultContent = null; - JobInfoResult jobInfo = getClient(job.getJobProps()).getJobInfo(job.getJobExecuteResult()); - String[] resultSetList = jobInfo.getResultSetList(getClient(job.getJobProps())); - if (resultSetList != null && resultSetList.length > 0) { - Object fileContent = client.resultSet(ResultSetAction.builder() - .setPath(resultSetList[index]) - .setUser(job.getJobExecuteResult().getUser()) - .setPageSize(maxSize).build()).getFileContent(); - if (fileContent instanceof ArrayList) { - ArrayList> resultSetRow = (ArrayList>) fileContent; - resultContent = StringUtils.join(resultSetRow, "\n"); - } else { - resultContent = fileContent.toString(); - } - } - return resultContent; - } - - @Override - public void onStatusChanged(String fromState, String toState, Job job) { - if (LinkisJobExecutionConfiguration.SUCCEED.equalsIgnoreCase(toState)){ - if (job instanceof SignalSharedJob){ - SignalSharedJob signalSharedJob = (SignalSharedJob) job; - String result = getResult(job, 0, -1); - WorkflowContext.getAppJointContext().setValue(signalSharedJob.getSharedKey(), result , -1); - } else if(job instanceof SharedJob){ - String taskId = job.getJobExecuteResult().getTaskID(); - job.getLogObj().info("Set shared info:" + taskId); - SharedJob sharedJob = ((SharedJob)job); - WorkflowContext.getAppJointContext().setValue(sharedJob.getSharedKey(), taskId , sharedJob.getSharedNum()); - } - - } - } -} diff --git a/dss-linkis-node-execution/src/main/java/com/webank/wedatasphere/dss/linkis/node/execution/job/AbstractAppJointLinkisJob.java b/dss-linkis-node-execution/src/main/java/com/webank/wedatasphere/dss/linkis/node/execution/job/AbstractAppJointLinkisJob.java deleted file mode 100644 index 7d9e47f069..0000000000 --- a/dss-linkis-node-execution/src/main/java/com/webank/wedatasphere/dss/linkis/node/execution/job/AbstractAppJointLinkisJob.java +++ /dev/null @@ -1,198 +0,0 @@ -/* - * Copyright 2019 WeBank - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package com.webank.wedatasphere.dss.linkis.node.execution.job; - -import com.webank.wedatasphere.dss.linkis.node.execution.log.LinkisJobExecutionLog; -import com.webank.wedatasphere.linkis.ujes.client.response.JobExecuteResult; - -import java.util.Map; - -/** - * Created by peacewong on 2019/11/3. - */ -public abstract class AbstractAppJointLinkisJob extends AppJointLinkisJob { - - private Map jobProps; - - - - private Map source; - - private JobTypeEnum jobType; - - private Map variables; - - private Map configuration; - - private String code; - - private String engineType; - - private String runType; - - - - private Map params; - - private Map runtimeParams; - - private JobExecuteResult jobExecuteResult; - - private LinkisJobExecutionLog logObj; - - private int logFromLine; - - - - - - - @Override - public Map getSource() { - return this.source; - } - - @Override - public JobTypeEnum getJobType() { - return this.jobType; - } - - - @Override - public Map getVariables() { - return this.variables; - } - - @Override - public Map getConfiguration() { - return this.configuration; - } - - @Override - public String getCode() { - return this.code; - } - - @Override - public void setCode(String code) { - this.code = code; - } - - @Override - public String getEngineType() { - return this.engineType; - } - - @Override - public String getRunType() { - return this.runType; - } - - - - @Override - public Map getParams() { - return this.params; - } - - @Override - public Map getRuntimeParams() { - return this.runtimeParams; - } - - @Override - public JobExecuteResult getJobExecuteResult() { - return this.jobExecuteResult; - } - - @Override - public void setJobExecuteResult(JobExecuteResult jobExecuteResult) { - this.jobExecuteResult = jobExecuteResult; - } - - @Override - public Map getJobProps() { - return this.jobProps; - } - - @Override - public void setJobProps(Map jobProps) { - this.jobProps = jobProps; - } - - @Override - public LinkisJobExecutionLog getLogObj() { - return this.logObj; - } - - @Override - public int getLogFromLine() { - return this.logFromLine; - } - - @Override - public void setLogFromLine(int index) { - this.logFromLine = index; - } - - - @Override - public void setSource(Map source) { - this.source = source; - } - - @Override - public void setJobType(JobTypeEnum jobType) { - this.jobType = jobType; - } - - @Override - public void setVariables(Map variables) { - this.variables = variables; - } - - @Override - public void setConfiguration(Map configuration) { - this.configuration = configuration; - } - - @Override - public void setEngineType(String engineType) { - this.engineType = engineType; - } - - @Override - public void setRunType(String runType) { - this.runType = runType; - } - - @Override - public void setParams(Map params) { - this.params = params; - } - - @Override - public void setRuntimeParams(Map runtimeParams) { - this.runtimeParams = runtimeParams; - } - - @Override - public void setLogObj(LinkisJobExecutionLog logObj) { - this.logObj = logObj; - } -} diff --git a/dss-linkis-node-execution/src/main/java/com/webank/wedatasphere/dss/linkis/node/execution/job/AbstractCommonLinkisJob.java b/dss-linkis-node-execution/src/main/java/com/webank/wedatasphere/dss/linkis/node/execution/job/AbstractCommonLinkisJob.java deleted file mode 100644 index c7f74f2348..0000000000 --- a/dss-linkis-node-execution/src/main/java/com/webank/wedatasphere/dss/linkis/node/execution/job/AbstractCommonLinkisJob.java +++ /dev/null @@ -1,240 +0,0 @@ -/* - * Copyright 2019 WeBank - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package com.webank.wedatasphere.dss.linkis.node.execution.job; - -import com.webank.wedatasphere.dss.linkis.node.execution.entity.BMLResource; -import com.webank.wedatasphere.dss.linkis.node.execution.log.LinkisJobExecutionLog; -import com.webank.wedatasphere.linkis.ujes.client.response.JobExecuteResult; - -import java.util.ArrayList; -import java.util.List; -import java.util.Map; - -/** - * Created by peacewong on 2019/11/3. - */ -public abstract class AbstractCommonLinkisJob extends CommonLinkisJob { - - private Map jobProps; - - - - private Map source; - - private JobTypeEnum jobType; - - private Map variables; - - private Map configuration; - - private String code; - - private String engineType; - - private String runType; - - - - private Map params; - - private Map runtimeParams; - - private JobExecuteResult jobExecuteResult; - - private LinkisJobExecutionLog logObj; - - private int logFromLine; - - - private List jobResourceList; - - private List projectResourceList; - - private Map> fLowNameAndResources; - - - - - @Override - public Map getSource() { - return this.source; - } - - @Override - public JobTypeEnum getJobType() { - return this.jobType; - } - - - @Override - public Map getVariables() { - return this.variables; - } - - @Override - public Map getConfiguration() { - return this.configuration; - } - - @Override - public String getCode() { - return this.code; - } - - @Override - public void setCode(String code) { - this.code = code; - } - - @Override - public String getEngineType() { - return this.engineType; - } - - @Override - public String getRunType() { - return this.runType; - } - - - - @Override - public Map getParams() { - return this.params; - } - - @Override - public Map getRuntimeParams() { - return this.runtimeParams; - } - - @Override - public JobExecuteResult getJobExecuteResult() { - return this.jobExecuteResult; - } - - @Override - public void setJobExecuteResult(JobExecuteResult jobExecuteResult) { - this.jobExecuteResult = jobExecuteResult; - } - - @Override - public Map getJobProps() { - return this.jobProps; - } - - @Override - public void setJobProps(Map jobProps) { - this.jobProps = jobProps; - } - - @Override - public LinkisJobExecutionLog getLogObj() { - return this.logObj; - } - - @Override - public int getLogFromLine() { - return this.logFromLine; - } - - @Override - public void setLogFromLine(int index) { - this.logFromLine = index; - } - - - @Override - public void setSource(Map source) { - this.source = source; - } - - @Override - public void setJobType(JobTypeEnum jobType) { - this.jobType = jobType; - } - - @Override - public void setVariables(Map variables) { - this.variables = variables; - } - - @Override - public void setConfiguration(Map configuration) { - this.configuration = configuration; - } - - @Override - public void setEngineType(String engineType) { - this.engineType = engineType; - } - - @Override - public void setRunType(String runType) { - this.runType = runType; - } - - @Override - public void setParams(Map params) { - this.params = params; - } - - @Override - public void setRuntimeParams(Map runtimeParams) { - this.runtimeParams = runtimeParams; - } - - @Override - public void setLogObj(LinkisJobExecutionLog logObj) { - this.logObj = logObj; - } - - - @Override - public List getJobResourceList() { - return this.jobResourceList; - } - - - - @Override - public List getProjectResourceList() { - return this.projectResourceList; - } - - @Override - public Map> getFlowNameAndResources() { - return this.fLowNameAndResources; - } - - @Override - public void setJobResourceList(List jobResourceList) { - this.jobResourceList = jobResourceList; - } - - @Override - public void setProjectResourceList(List projectResourceList) { - this.projectResourceList = projectResourceList; - } - - @Override - public void setFlowNameAndResources(Map> fLowNameAndResources) { - this.fLowNameAndResources = fLowNameAndResources; - } - -} diff --git a/dss-linkis-node-execution/src/main/java/com/webank/wedatasphere/dss/linkis/node/execution/job/AppJointLinkisJob.java b/dss-linkis-node-execution/src/main/java/com/webank/wedatasphere/dss/linkis/node/execution/job/AppJointLinkisJob.java deleted file mode 100644 index 2b12e40cf7..0000000000 --- a/dss-linkis-node-execution/src/main/java/com/webank/wedatasphere/dss/linkis/node/execution/job/AppJointLinkisJob.java +++ /dev/null @@ -1,24 +0,0 @@ -/* - * Copyright 2019 WeBank - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package com.webank.wedatasphere.dss.linkis.node.execution.job; - -/** - * Created by peacewong on 2019/11/2. - */ -public abstract class AppJointLinkisJob extends LinkisJob { -} diff --git a/dss-linkis-node-execution/src/main/java/com/webank/wedatasphere/dss/linkis/node/execution/job/Builder.java b/dss-linkis-node-execution/src/main/java/com/webank/wedatasphere/dss/linkis/node/execution/job/Builder.java deleted file mode 100644 index 6409b9f376..0000000000 --- a/dss-linkis-node-execution/src/main/java/com/webank/wedatasphere/dss/linkis/node/execution/job/Builder.java +++ /dev/null @@ -1,150 +0,0 @@ -/* - * Copyright 2019 WeBank - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package com.webank.wedatasphere.dss.linkis.node.execution.job; - -import com.webank.wedatasphere.dss.linkis.node.execution.WorkflowContext; -import com.webank.wedatasphere.dss.linkis.node.execution.conf.LinkisJobExecutionConfiguration; -import com.webank.wedatasphere.dss.linkis.node.execution.exception.LinkisJobExecutionErrorException; -import com.webank.wedatasphere.dss.linkis.node.execution.utils.LinkisJobExecutionUtils; -import org.apache.commons.lang.StringUtils; - -import java.util.HashMap; -import java.util.Map; - -/** - * Created by peacewong on 2019/11/13. - */ -public abstract class Builder { - - protected abstract String getJobType(); - - - protected abstract Boolean isReadNode(); - - protected abstract Boolean isShareNod(); - - protected abstract Boolean isSignalSharedNode(); - - protected abstract ReadJob createReadJob(boolean isLinkisType); - - protected abstract SharedJob createSharedJob(boolean isLinkisType); - - protected abstract SignalSharedJob createSignalSharedJob(boolean isLinkisType); - - - protected abstract LinkisJob creatLinkisJob(boolean isLinkisType); - - protected abstract void fillJobInfo(Job job); - - protected abstract void fillLinkisJobInfo(LinkisJob linkisJob); - - protected abstract void fillCommonLinkisJobInfo(CommonLinkisJob linkisAppjointJob); - - - public Job build() throws Exception { - - LinkisJob job = null; - String jobType = getJobType(); - String[] jobTypeSplit = jobType.split("\\."); - if (jobTypeSplit.length < 3) { - throw new LinkisJobExecutionErrorException(90100, "This is not Linkis job type,this jobtype is " + jobType); - } - String engineType = jobTypeSplit[1]; - //delete linkis.engineType - String runType = StringUtils.substringAfterLast(jobType, jobTypeSplit[0] + "." + jobTypeSplit[1] + "."); - - if (LinkisJobExecutionConfiguration.LINKIS_CONTROL_EMPTY_NODE.equalsIgnoreCase(jobType)) { - job = new AbstractCommonLinkisJob() { - @Override - public String getSubmitUser() { - return null; - } - - @Override - public String getUser() { - return null; - } - - @Override - public String getJobName() { - return null; - } - }; - - job.setJobType(JobTypeEnum.EmptyJob); - return job; - } - if (LinkisJobExecutionUtils.isCommonAppjointJob(engineType)) { - if (isReadNode()) { - ReadJob readJob = createReadJob(false); - readJob.setSharedNodesInfo(getSharedNodesAndJobId(readJob)); - job = (LinkisJob) readJob; - job.setJobType(JobTypeEnum.ReadJob); - } else if (isShareNod()) { - SharedJob sharedJob = createSharedJob(false); - job = (LinkisJob) sharedJob; - job.setJobType(JobTypeEnum.ShareJob); - } else if (isSignalSharedNode()) { - job = (LinkisJob) createSignalSharedJob(false); - job.setJobType(JobTypeEnum.ShareJob); - } else { - job = creatLinkisJob(false); - job.setJobType(JobTypeEnum.CommonJob); - } - } else { - if (isReadNode()) { - ReadJob readJob = createReadJob(true); - readJob.setSharedNodesInfo(getSharedNodesAndJobId(readJob)); - job = (LinkisJob) readJob; - job.setJobType(JobTypeEnum.ReadJob); - } else if (isShareNod()) { - SharedJob sharedJob = createSharedJob(true); - job = (LinkisJob) sharedJob; - job.setJobType(JobTypeEnum.ShareJob); - }else if (isSignalSharedNode()) { - job = (LinkisJob) createSignalSharedJob(true); - job.setJobType(JobTypeEnum.ShareJob); - } else { - job = creatLinkisJob(true); - job.setJobType(JobTypeEnum.CommonJob); - } - fillCommonLinkisJobInfo((CommonLinkisJob) job); - } - - job.setEngineType(engineType); - job.setRunType(runType); - fillJobInfo(job); - fillLinkisJobInfo(job); - - return job; - } - - - private Map getSharedNodesAndJobId(ReadJob job) { - Map map = new HashMap<>(); - String[] sharedIds = job.getShareNodeIds(); - if (sharedIds == null) { - return map; - } - for (String nodeId : sharedIds) { - map.put(nodeId, WorkflowContext.getAppJointContext().getValue(job.getSharedKey(nodeId))); - } - return map; - } - -} diff --git a/dss-linkis-node-execution/src/main/java/com/webank/wedatasphere/dss/linkis/node/execution/job/CommonLinkisJob.java b/dss-linkis-node-execution/src/main/java/com/webank/wedatasphere/dss/linkis/node/execution/job/CommonLinkisJob.java deleted file mode 100644 index 229ee28621..0000000000 --- a/dss-linkis-node-execution/src/main/java/com/webank/wedatasphere/dss/linkis/node/execution/job/CommonLinkisJob.java +++ /dev/null @@ -1,42 +0,0 @@ -/* - * Copyright 2019 WeBank - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package com.webank.wedatasphere.dss.linkis.node.execution.job; - -import com.webank.wedatasphere.dss.linkis.node.execution.entity.BMLResource; - -import java.util.ArrayList; -import java.util.List; -import java.util.Map; - -/** - * Created by peacewong on 2019/11/2. - */ -public abstract class CommonLinkisJob extends LinkisJob { - - public abstract List getJobResourceList(); - - public abstract void setJobResourceList(List jobResourceList); - - public abstract List getProjectResourceList(); - - public abstract void setProjectResourceList(List projectResourceList); - - public abstract Map> getFlowNameAndResources(); - - public abstract void setFlowNameAndResources(Map> fLowNameAndResources); -} diff --git a/dss-linkis-node-execution/src/main/java/com/webank/wedatasphere/dss/linkis/node/execution/job/Job.java b/dss-linkis-node-execution/src/main/java/com/webank/wedatasphere/dss/linkis/node/execution/job/Job.java deleted file mode 100644 index 1811ad105e..0000000000 --- a/dss-linkis-node-execution/src/main/java/com/webank/wedatasphere/dss/linkis/node/execution/job/Job.java +++ /dev/null @@ -1,68 +0,0 @@ -/* - * Copyright 2019 WeBank - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package com.webank.wedatasphere.dss.linkis.node.execution.job; - -import com.webank.wedatasphere.dss.linkis.node.execution.log.LinkisJobExecutionLog; -import com.webank.wedatasphere.linkis.ujes.client.response.JobExecuteResult; - -import java.util.Map; - -/** - * Created by peacewong on 2019/10/15. - */ -public interface Job { - - String getCode(); - - void setCode(String code); - - String getEngineType(); - - void setEngineType(String engineType); - - String getRunType(); - - void setRunType(String runType); - - String getUser(); - - String getJobName(); - - Map getParams(); - - void setParams(Map params); - - Map getRuntimeParams(); - void setRuntimeParams(Map runtimeParams); - - JobExecuteResult getJobExecuteResult(); - - void setJobExecuteResult(JobExecuteResult jobExecuteResult); - - Map getJobProps(); - - void setJobProps(Map jobProps); - - LinkisJobExecutionLog getLogObj(); - void setLogObj(LinkisJobExecutionLog logObj); - - int getLogFromLine(); - - void setLogFromLine(int index); - -} diff --git a/dss-linkis-node-execution/src/main/java/com/webank/wedatasphere/dss/linkis/node/execution/job/JobSignalKeyCreator.java b/dss-linkis-node-execution/src/main/java/com/webank/wedatasphere/dss/linkis/node/execution/job/JobSignalKeyCreator.java deleted file mode 100644 index ce64bf9017..0000000000 --- a/dss-linkis-node-execution/src/main/java/com/webank/wedatasphere/dss/linkis/node/execution/job/JobSignalKeyCreator.java +++ /dev/null @@ -1,25 +0,0 @@ -/* - * Copyright 2019 WeBank - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package com.webank.wedatasphere.dss.linkis.node.execution.job; - -public interface JobSignalKeyCreator { - - String getSignalKeyByJob(Job job); - - String getSignalKeyBySignalSharedJob(SignalSharedJob job); -} diff --git a/dss-linkis-node-execution/src/main/java/com/webank/wedatasphere/dss/linkis/node/execution/job/JobTypeEnum.java b/dss-linkis-node-execution/src/main/java/com/webank/wedatasphere/dss/linkis/node/execution/job/JobTypeEnum.java deleted file mode 100644 index 348ce6e107..0000000000 --- a/dss-linkis-node-execution/src/main/java/com/webank/wedatasphere/dss/linkis/node/execution/job/JobTypeEnum.java +++ /dev/null @@ -1,25 +0,0 @@ -/* - * Copyright 2019 WeBank - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package com.webank.wedatasphere.dss.linkis.node.execution.job; - -/** - * Created by peacewong on 2019/11/2. - */ -public enum JobTypeEnum { - CommonJob, EmptyJob,ReadJob, ShareJob -} diff --git a/dss-linkis-node-execution/src/main/java/com/webank/wedatasphere/dss/linkis/node/execution/job/LinkisJob.java b/dss-linkis-node-execution/src/main/java/com/webank/wedatasphere/dss/linkis/node/execution/job/LinkisJob.java deleted file mode 100644 index b2b6f11417..0000000000 --- a/dss-linkis-node-execution/src/main/java/com/webank/wedatasphere/dss/linkis/node/execution/job/LinkisJob.java +++ /dev/null @@ -1,46 +0,0 @@ -/* - * Copyright 2019 WeBank - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package com.webank.wedatasphere.dss.linkis.node.execution.job; - - -import java.util.Map; - -/** - * Created by peacewong on 2019/10/31. - */ -public abstract class LinkisJob implements Job{ - - - public abstract Map getSource(); - - public abstract void setSource(Map source); - - public abstract JobTypeEnum getJobType(); - public abstract void setJobType(JobTypeEnum jobType); - - public abstract String getSubmitUser(); - - public abstract Map getVariables(); - public abstract void setVariables(Map variables); - - - public abstract Map getConfiguration(); - public abstract void setConfiguration(Map configuration); - - -} diff --git a/dss-linkis-node-execution/src/main/java/com/webank/wedatasphere/dss/linkis/node/execution/job/ReadJob.java b/dss-linkis-node-execution/src/main/java/com/webank/wedatasphere/dss/linkis/node/execution/job/ReadJob.java deleted file mode 100644 index 6ceb08335a..0000000000 --- a/dss-linkis-node-execution/src/main/java/com/webank/wedatasphere/dss/linkis/node/execution/job/ReadJob.java +++ /dev/null @@ -1,35 +0,0 @@ -/* - * Copyright 2019 WeBank - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package com.webank.wedatasphere.dss.linkis.node.execution.job; - -import java.util.List; -import java.util.Map; - -/** - * Created by peacewong on 2019/11/2. - */ -public interface ReadJob { - - Map getSharedNodesInfo(); - - String getSharedKey(String value); - - void setSharedNodesInfo(Map sharedNodesInfo); - - String[] getShareNodeIds(); -} diff --git a/dss-linkis-node-execution/src/main/java/com/webank/wedatasphere/dss/linkis/node/execution/job/RuntimeJob.java b/dss-linkis-node-execution/src/main/java/com/webank/wedatasphere/dss/linkis/node/execution/job/RuntimeJob.java deleted file mode 100644 index 2f803c2adc..0000000000 --- a/dss-linkis-node-execution/src/main/java/com/webank/wedatasphere/dss/linkis/node/execution/job/RuntimeJob.java +++ /dev/null @@ -1,24 +0,0 @@ -/* - * Copyright 2019 WeBank - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package com.webank.wedatasphere.dss.linkis.node.execution.job; - -/** - * Created by peacewong on 2019/11/2. - */ -public interface RuntimeJob { -} diff --git a/dss-linkis-node-execution/src/main/java/com/webank/wedatasphere/dss/linkis/node/execution/job/SharedJob.java b/dss-linkis-node-execution/src/main/java/com/webank/wedatasphere/dss/linkis/node/execution/job/SharedJob.java deleted file mode 100644 index 7fb84c91a6..0000000000 --- a/dss-linkis-node-execution/src/main/java/com/webank/wedatasphere/dss/linkis/node/execution/job/SharedJob.java +++ /dev/null @@ -1,29 +0,0 @@ -/* - * Copyright 2019 WeBank - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package com.webank.wedatasphere.dss.linkis.node.execution.job; - -/** - * Created by peacewong on 2019/11/2. - */ -public interface SharedJob { - - int getSharedNum(); - - - String getSharedKey(); -} \ No newline at end of file diff --git a/dss-linkis-node-execution/src/main/java/com/webank/wedatasphere/dss/linkis/node/execution/job/SignalSharedJob.java b/dss-linkis-node-execution/src/main/java/com/webank/wedatasphere/dss/linkis/node/execution/job/SignalSharedJob.java deleted file mode 100644 index 13a9b8b444..0000000000 --- a/dss-linkis-node-execution/src/main/java/com/webank/wedatasphere/dss/linkis/node/execution/job/SignalSharedJob.java +++ /dev/null @@ -1,42 +0,0 @@ -/* - * Copyright 2019 WeBank - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package com.webank.wedatasphere.dss.linkis.node.execution.job; - -/** - * Created by enjoyyin on 2019/11/13. - */ -public interface SignalSharedJob extends SharedJob { - - String PREFIX = "signal."; - - JobSignalKeyCreator getSignalKeyCreator(); - - void setSignalKeyCreator(JobSignalKeyCreator signalKeyCreator); - - @Override - default int getSharedNum() { - return -1; - } - - @Override - default String getSharedKey() { - return PREFIX + getSignalKeyCreator().getSignalKeyBySignalSharedJob(this) + "." + getMsgSaveKey(); - } - - String getMsgSaveKey(); -} diff --git a/dss-linkis-node-execution/src/main/java/com/webank/wedatasphere/dss/linkis/node/execution/listener/LinkisExecutionListener.java b/dss-linkis-node-execution/src/main/java/com/webank/wedatasphere/dss/linkis/node/execution/listener/LinkisExecutionListener.java deleted file mode 100644 index 67fa6a7868..0000000000 --- a/dss-linkis-node-execution/src/main/java/com/webank/wedatasphere/dss/linkis/node/execution/listener/LinkisExecutionListener.java +++ /dev/null @@ -1,28 +0,0 @@ -/* - * Copyright 2019 WeBank - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package com.webank.wedatasphere.dss.linkis.node.execution.listener; - -import com.webank.wedatasphere.dss.linkis.node.execution.job.Job; - -/** - * Created by peacewong on 2019/11/13. - */ -public interface LinkisExecutionListener { - - void onStatusChanged(String fromState, String toState, Job job); -} diff --git a/dss-linkis-node-execution/src/main/java/com/webank/wedatasphere/dss/linkis/node/execution/log/LinkisJobExecutionLog.java b/dss-linkis-node-execution/src/main/java/com/webank/wedatasphere/dss/linkis/node/execution/log/LinkisJobExecutionLog.java deleted file mode 100644 index cf60ad3dac..0000000000 --- a/dss-linkis-node-execution/src/main/java/com/webank/wedatasphere/dss/linkis/node/execution/log/LinkisJobExecutionLog.java +++ /dev/null @@ -1,41 +0,0 @@ -/* - * Copyright 2019 WeBank - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package com.webank.wedatasphere.dss.linkis.node.execution.log; - -/** - * Created by peacewong on 2019/11/2. - */ -public abstract class LinkisJobExecutionLog { - - public void info(Object message){ - info(message, null); - } - public abstract void info(Object message, Throwable t); - - public void warn(Object message){ - warn(message, null); - } - public abstract void warn(Object message, Throwable t); - - public void error(Object message){ - error(message, null); - } - public abstract void error(Object message, Throwable t); - - -} diff --git a/dss-linkis-node-execution/src/main/java/com/webank/wedatasphere/dss/linkis/node/execution/parser/CodeParser.java b/dss-linkis-node-execution/src/main/java/com/webank/wedatasphere/dss/linkis/node/execution/parser/CodeParser.java deleted file mode 100644 index f4c254e8a3..0000000000 --- a/dss-linkis-node-execution/src/main/java/com/webank/wedatasphere/dss/linkis/node/execution/parser/CodeParser.java +++ /dev/null @@ -1,235 +0,0 @@ -/* - * Copyright 2019 WeBank - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package com.webank.wedatasphere.dss.linkis.node.execution.parser; - -import com.google.gson.reflect.TypeToken; -import com.webank.wedatasphere.dss.linkis.node.execution.conf.LinkisJobExecutionConfiguration; -import com.webank.wedatasphere.dss.linkis.node.execution.exception.LinkisJobExecutionErrorException; -import com.webank.wedatasphere.dss.linkis.node.execution.job.CommonLinkisJob; -import com.webank.wedatasphere.dss.linkis.node.execution.job.Job; -import com.webank.wedatasphere.dss.linkis.node.execution.utils.LinkisJobExecutionUtils; -import com.webank.wedatasphere.dss.linkis.node.execution.entity.BMLResource; -import com.webank.wedatasphere.linkis.filesystem.WorkspaceClientFactory; -import com.webank.wedatasphere.linkis.filesystem.request.WorkspaceClient; -import com.webank.wedatasphere.linkis.filesystem.response.ScriptFromBMLResponse; -import org.apache.commons.lang.StringUtils; - -import java.util.*; -import java.util.regex.Matcher; -import java.util.regex.Pattern; - -/** - * Created by peacewong on 2019/11/2. - */ -public class CodeParser implements JobParser { - - private final static Pattern pb = Pattern.compile("((project)|(flow)|(node))://[^\\s\"]+[$\\s]{0,1}", Pattern.CASE_INSENSITIVE); - - @Override - public void parseJob(Job job) throws Exception{ - if (! ( job instanceof CommonLinkisJob) ) { - return ; - } - CommonLinkisJob linkisAppjointJob = (CommonLinkisJob) job; - Map script = LinkisJobExecutionUtils.gson.fromJson(linkisAppjointJob.getCode(), new TypeToken>() {}.getType()); - List jobResourceList = linkisAppjointJob.getJobResourceList(); - BMLResource scriptResource = null; - if (script == null) { - throw new LinkisJobExecutionErrorException(90102,"Script is empty"); - } - String fileName = (String) script.get("script"); - for(BMLResource bmlResource : jobResourceList){ - if(bmlResource.getFileName().equals(fileName)){ - scriptResource = bmlResource; - break; - } - } - if(null == scriptResource) { - throw new LinkisJobExecutionErrorException(90102,"Failed to get script resource"); - } - - Map executionParams = getExecutionParams(scriptResource, linkisAppjointJob); - - if (executionParams.get("executionCode") != null) { - - String executionCode = (String) executionParams.get("executionCode"); - linkisAppjointJob.getLogObj().info("************************************SUBMIT CODE************************************"); - linkisAppjointJob.getLogObj().info(executionCode); - linkisAppjointJob.getLogObj().info("************************************SUBMIT CODE************************************"); - - //get ResourceNames - ArrayList resourceNames = getResourceNames(executionCode); - - //get ResourceList - - ArrayList bmlResourceArrayList = getResourcesByNames(resourceNames, linkisAppjointJob); - - //parsedCode - linkisAppjointJob.setCode(replaceCodeResourceNames(executionCode, resourceNames, bmlResourceArrayList)); - - - //put resources - if (linkisAppjointJob.getRuntimeParams() != null) { - linkisAppjointJob.getRuntimeParams().put("resources", bmlResourceArrayList); - } - } - if (executionParams.get("params") != null && executionParams.get("params") instanceof Map) { - if (linkisAppjointJob.getParams() != null) { - linkisAppjointJob.getParams().putAll( (Map)executionParams.get("params")); - } - } - - } - - private Map getExecutionParams(BMLResource bmlResource, CommonLinkisJob linkisAppjointJob) { - Map map = new HashMap<>(); - WorkspaceClient client = WorkspaceClientFactory.getClient(linkisAppjointJob.getSubmitUser(), LinkisJobExecutionConfiguration.LINKIS_AUTHOR_USER_TOKEN.getValue(linkisAppjointJob.getJobProps()), LinkisJobExecutionConfiguration.LINKIS_URL.getValue(linkisAppjointJob.getJobProps())); - ScriptFromBMLResponse response = client.requestOpenScriptFromBML(bmlResource.getResourceId(), bmlResource.getVersion(), bmlResource.getFileName()); - map.put("executionCode", response.scriptContent()); - map.put("params", response.metadata()); - return map; - } - - private ArrayList getResourceNames(String code){ - ArrayList bmlResourceNames = new ArrayList(); - Matcher mb = pb.matcher(code); - while (mb.find()) { - bmlResourceNames.add(mb.group().trim()); - } - return bmlResourceNames; - } - - - /** - * 1.Find the project file used in the script - * 2.Find the node file used in the script - * 3.Recursively find the flow file used in the script - * 4.Replace file name with prefixed name - * @param resourceNames - * @param linkisAppjointJob - * @return - */ - private ArrayList getResourcesByNames(ArrayList resourceNames, CommonLinkisJob linkisAppjointJob) { - - ArrayList bmlResourceArrayList = new ArrayList<>(); - - String jobName = linkisAppjointJob.getJobName(); - String flowName = linkisAppjointJob.getSource().get("flowName"); - String projectName = linkisAppjointJob.getSource().get("projectName"); - - - List projectResourceList = linkisAppjointJob.getProjectResourceList(); - - - List jobResourceList = linkisAppjointJob.getJobResourceList(); - for (String resourceName : resourceNames) { - String[] resourceNameSplit = resourceName.split("://"); - String prefix = resourceNameSplit[0].toLowerCase(); - String fileName = resourceNameSplit[1]; - BMLResource resource = null; - String afterFileName = fileName; - switch (prefix) { - case "project": - resource = findResource(projectResourceList, fileName); - afterFileName = LinkisJobExecutionConfiguration.PROJECT_PREFIX + "_" + projectName + "_" + fileName; - break; - case "flow": - resource = findFlowResource(linkisAppjointJob, fileName, flowName); - break; - case "node": - resource = findResource(jobResourceList, fileName); - afterFileName = LinkisJobExecutionConfiguration.JOB_PREFIX + "_" + jobName + "_" + fileName; - break; - default: - } - if (null == resource) { - linkisAppjointJob.getLogObj().error("Failed to find the " + prefix + " resource file of " + fileName); - throw new RuntimeException("Failed to find the " + prefix + " resource file of " + fileName); - } - if (!afterFileName.equals(fileName)) { - resource.setFileName(afterFileName); - } - bmlResourceArrayList.add(resource); - } - return bmlResourceArrayList; - } - - - /** - * Recursively find the flow file used in the script - * Recursive exit condition is top-level flow - * - */ - private BMLResource findFlowResource(CommonLinkisJob linkisAppjointJob, String fileName, String flowName) { - - String fullFlowName = ""; - Map> fLowNameAndResources = linkisAppjointJob.getFlowNameAndResources(); - if (fLowNameAndResources == null){ - return null; - } - Optional>> first = fLowNameAndResources.entrySet().stream().filter(fLowNameAndResource -> fLowNameAndResource.getKey().endsWith(flowName + LinkisJobExecutionConfiguration.RESOURCES_NAME)).findFirst(); - - if(first.isPresent()){ - fullFlowName = first.get().getKey(); - BMLResource resource = findResource(first.get().getValue(), fileName); - if (resource != null) { - resource.setFileName(flowName + "_" + fileName); - return resource; - } - } - - String firstFlow = "flow." + flowName + LinkisJobExecutionConfiguration.RESOURCES_NAME; - if (firstFlow.equals(fullFlowName)) { - return null; - } - //getParentFlowName:flow.flows1.test.resources return:flows1 - String parentFlowName = StringUtils.substringAfterLast(StringUtils.substringBefore(fullFlowName, "." + flowName - + LinkisJobExecutionConfiguration.RESOURCES_NAME), "."); - if (StringUtils.isEmpty(parentFlowName)) { - return null; - } - - return findFlowResource(linkisAppjointJob, fileName, parentFlowName); - } - - - private String replaceCodeResourceNames(String code, ArrayList resourceNameList, ArrayList resourceList){ - if(resourceList.size() != resourceNameList.size()){ - throw new RuntimeException("Failed to parsed resource file"); - } - - String[] names = resourceNameList.toArray(new String[]{}); - - String[] afterNames = new String[resourceList.size()]; - for (int i=0 ; i < afterNames.length ; i++){ - afterNames[i] = resourceList.get(i).getFileName(); - } - return StringUtils.replaceEach(code, names, afterNames); - } - - private BMLResource findResource(List resourceArrayList, String fileName){ - if(resourceArrayList != null && !resourceArrayList.isEmpty()) { - for(BMLResource resource : resourceArrayList){ - if(resource.getFileName().equals(fileName)){ - return resource; - } - } - } - return null; - } -} diff --git a/dss-linkis-node-execution/src/main/java/com/webank/wedatasphere/dss/linkis/node/execution/parser/JobParamsParser.java b/dss-linkis-node-execution/src/main/java/com/webank/wedatasphere/dss/linkis/node/execution/parser/JobParamsParser.java deleted file mode 100644 index 27174fbcc8..0000000000 --- a/dss-linkis-node-execution/src/main/java/com/webank/wedatasphere/dss/linkis/node/execution/parser/JobParamsParser.java +++ /dev/null @@ -1,89 +0,0 @@ -/* - * Copyright 2019 WeBank - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package com.webank.wedatasphere.dss.linkis.node.execution.parser; - -import com.google.gson.reflect.TypeToken; -import com.webank.wedatasphere.dss.linkis.node.execution.WorkflowContext; -import com.webank.wedatasphere.dss.linkis.node.execution.job.JobSignalKeyCreator; -import com.webank.wedatasphere.dss.linkis.node.execution.job.LinkisJob; -import com.webank.wedatasphere.dss.linkis.node.execution.job.Job; -import com.webank.wedatasphere.dss.linkis.node.execution.job.SignalSharedJob; -import com.webank.wedatasphere.dss.linkis.node.execution.utils.LinkisJobExecutionUtils; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import java.util.Collection; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -/** - * Created by johnnwang on 2019/11/3. - */ -public class JobParamsParser implements JobParser { - private static final Logger LOGGER = LoggerFactory.getLogger(JobParamsParser.class); - private JobSignalKeyCreator signalKeyCreator; - - public JobSignalKeyCreator getSignalKeyCreator() { - return signalKeyCreator; - } - - public void setSignalKeyCreator(JobSignalKeyCreator signalKeyCreator) { - this.signalKeyCreator = signalKeyCreator; - } - - @Override - public void parseJob(Job job) throws Exception { - - if (job instanceof LinkisJob) { - - LinkisJob linkisJob = (LinkisJob) job; - - linkisJob.getLogObj().info("Start to put variable and configuration"); - //put variable - Map flowVariables = linkisJob.getVariables(); - putParamsMap(job.getParams(), "variable", flowVariables); - //put signal info - Map sharedValue = WorkflowContext.getAppJointContext() - .getSubMapByPrefix(SignalSharedJob.PREFIX + this.getSignalKeyCreator().getSignalKeyByJob(job)); - if (sharedValue != null) { - Collection values = sharedValue.values(); - for(Object value : values){ - List> list = LinkisJobExecutionUtils.gson.fromJson(value.toString(), List.class); - Map totalMap = new HashMap<>(); - for (Map kv : list) { - totalMap.putAll(kv); - } - putParamsMap(job.getParams(), "variable", totalMap); - } - } - // put configuration - Map configuration = linkisJob.getConfiguration(); - putParamsMap(job.getParams(), "configuration", configuration); - linkisJob.getLogObj().info("Finished to put variable and configuration"); - } - } - - private void putParamsMap(Map params, String key, Map value) { - if (null == value) return; - if (params.get(key) != null) { - ((Map) params.get(key)).putAll(value); - } else { - params.put(key, value); - } - } -} diff --git a/dss-linkis-node-execution/src/main/java/com/webank/wedatasphere/dss/linkis/node/execution/parser/JobParser.java b/dss-linkis-node-execution/src/main/java/com/webank/wedatasphere/dss/linkis/node/execution/parser/JobParser.java deleted file mode 100644 index ce01a5bb96..0000000000 --- a/dss-linkis-node-execution/src/main/java/com/webank/wedatasphere/dss/linkis/node/execution/parser/JobParser.java +++ /dev/null @@ -1,30 +0,0 @@ -/* - * Copyright 2019 WeBank - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package com.webank.wedatasphere.dss.linkis.node.execution.parser; - -import com.webank.wedatasphere.dss.linkis.node.execution.job.Job; - -/** - * Created by peacewong on 2019/11/2. - */ -public interface JobParser { - - void parseJob(Job job) throws Exception; - - -} diff --git a/dss-linkis-node-execution/src/main/java/com/webank/wedatasphere/dss/linkis/node/execution/parser/JobRuntimeParamsParser.java b/dss-linkis-node-execution/src/main/java/com/webank/wedatasphere/dss/linkis/node/execution/parser/JobRuntimeParamsParser.java deleted file mode 100644 index 6c071f1ee5..0000000000 --- a/dss-linkis-node-execution/src/main/java/com/webank/wedatasphere/dss/linkis/node/execution/parser/JobRuntimeParamsParser.java +++ /dev/null @@ -1,41 +0,0 @@ -/* - * Copyright 2019 WeBank - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package com.webank.wedatasphere.dss.linkis.node.execution.parser; - -import com.webank.wedatasphere.dss.linkis.node.execution.conf.LinkisJobExecutionConfiguration; -import com.webank.wedatasphere.dss.linkis.node.execution.job.LinkisJob; -import com.webank.wedatasphere.dss.linkis.node.execution.job.Job; -import com.webank.wedatasphere.dss.linkis.node.execution.job.ReadJob; - -/** - * Created by peacewong on 2019/11/3. - */ -public class JobRuntimeParamsParser implements JobParser{ - - @Override - public void parseJob(Job job) throws Exception{ - if(job instanceof LinkisJob) { - job.getRuntimeParams().put(LinkisJobExecutionConfiguration.LINKIS_SUBMIT_USER,((LinkisJob) job).getSubmitUser()); - } - if (job instanceof ReadJob) { - //put the shared nods be read node need - job.getRuntimeParams() - .put(LinkisJobExecutionConfiguration.WORKFLOW_SHARED_NODES_JOBIDS, ((ReadJob) job).getSharedNodesInfo()); - } - } -} diff --git a/dss-linkis-node-execution/src/main/java/com/webank/wedatasphere/dss/linkis/node/execution/service/BuildJobAction.java b/dss-linkis-node-execution/src/main/java/com/webank/wedatasphere/dss/linkis/node/execution/service/BuildJobAction.java deleted file mode 100644 index 1168b52ca6..0000000000 --- a/dss-linkis-node-execution/src/main/java/com/webank/wedatasphere/dss/linkis/node/execution/service/BuildJobAction.java +++ /dev/null @@ -1,32 +0,0 @@ -/* - * Copyright 2019 WeBank - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package com.webank.wedatasphere.dss.linkis.node.execution.service; - - -import com.webank.wedatasphere.dss.linkis.node.execution.exception.LinkisJobExecutionErrorException; -import com.webank.wedatasphere.dss.linkis.node.execution.job.Job; -import com.webank.wedatasphere.linkis.ujes.client.request.JobExecuteAction; - -/** - * Created by peacewong on 2019/9/21. - */ -public interface BuildJobAction { - - public JobExecuteAction getJobAction(Job job) throws LinkisJobExecutionErrorException; - -} diff --git a/dss-linkis-node-execution/src/main/java/com/webank/wedatasphere/dss/linkis/node/execution/service/impl/BuildJobActionImpl.java b/dss-linkis-node-execution/src/main/java/com/webank/wedatasphere/dss/linkis/node/execution/service/impl/BuildJobActionImpl.java deleted file mode 100644 index 8927be8fc9..0000000000 --- a/dss-linkis-node-execution/src/main/java/com/webank/wedatasphere/dss/linkis/node/execution/service/impl/BuildJobActionImpl.java +++ /dev/null @@ -1,65 +0,0 @@ -/* - * Copyright 2019 WeBank - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package com.webank.wedatasphere.dss.linkis.node.execution.service.impl; - -import com.webank.wedatasphere.dss.linkis.node.execution.exception.LinkisJobExecutionErrorException; -import com.webank.wedatasphere.dss.linkis.node.execution.job.LinkisJob; -import com.webank.wedatasphere.dss.linkis.node.execution.service.BuildJobAction; -import com.webank.wedatasphere.dss.linkis.node.execution.conf.LinkisJobExecutionConfiguration; -import com.webank.wedatasphere.dss.linkis.node.execution.job.Job; -import com.webank.wedatasphere.linkis.ujes.client.request.JobExecuteAction; - -import java.util.HashMap; -import java.util.Map; - - -/** - * Created by peacewong on 2019/11/3. - */ -public class BuildJobActionImpl implements BuildJobAction { - - private static BuildJobAction buildJobAction = new BuildJobActionImpl(); - - private BuildJobActionImpl(){ - - } - - public static BuildJobAction getbuildJobAction(){ - return buildJobAction; - } - - @Override - public JobExecuteAction getJobAction(Job job) throws LinkisJobExecutionErrorException { - - JobExecuteAction.Builder builder = JobExecuteAction.builder().setCreator(LinkisJobExecutionConfiguration.LINKIS_JOB_CREATOR.getValue(job.getJobProps())) - .addExecuteCode(job.getCode()) - .setEngineTypeStr(job.getEngineType()) - .setRunTypeStr(job.getRunType()) - .setUser(job.getUser()) - .setParams(job.getParams()) - .setRuntimeParams(job.getRuntimeParams()); - if(job instanceof LinkisJob){ - Map source = new HashMap<>(); - source.putAll(((LinkisJob) job).getSource()); - builder = builder.setSource(source); - } - return builder.build(); - } - - -} diff --git a/dss-linkis-node-execution/src/main/java/com/webank/wedatasphere/dss/linkis/node/execution/utils/LinkisJobExecutionUtils.java b/dss-linkis-node-execution/src/main/java/com/webank/wedatasphere/dss/linkis/node/execution/utils/LinkisJobExecutionUtils.java deleted file mode 100644 index 82d56b5ec6..0000000000 --- a/dss-linkis-node-execution/src/main/java/com/webank/wedatasphere/dss/linkis/node/execution/utils/LinkisJobExecutionUtils.java +++ /dev/null @@ -1,59 +0,0 @@ -/* - * Copyright 2019 WeBank - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package com.webank.wedatasphere.dss.linkis.node.execution.utils; - -import com.google.gson.Gson; -import com.google.gson.GsonBuilder; -import com.google.gson.reflect.TypeToken; -import com.webank.wedatasphere.dss.linkis.node.execution.conf.LinkisJobExecutionConfiguration; -import com.webank.wedatasphere.dss.linkis.node.execution.entity.BMLResource; -import org.apache.commons.lang.StringUtils; - -import java.util.*; - -/** - * Created by peacewong on 2019/9/21. - */ -public class LinkisJobExecutionUtils { - - - public final static Gson gson = new GsonBuilder().setDateFormat("yyyy-MM-dd'T'HH:mm:ssZ").create(); - - - - public static BMLResource getBMLResourceByJson(String json){ - return gson.fromJson(json, BMLResource.class); - } - - public static ArrayList getResourceListByJson(String json){ - if(StringUtils.isEmpty(json)){ - return null; - } - return gson.fromJson(json,new TypeToken>() {}.getType()); - } - - - public static Boolean isCommonAppjointJob(String engineType){ - - return LinkisJobExecutionConfiguration.APPJOINT.equalsIgnoreCase(engineType); - - } - - - -} diff --git a/dss-linkis-node-execution/src/main/java/com/webank/wedatasphere/dss/linkis/node/execution/utils/LinkisUjesClientUtils.java b/dss-linkis-node-execution/src/main/java/com/webank/wedatasphere/dss/linkis/node/execution/utils/LinkisUjesClientUtils.java deleted file mode 100644 index 61c176d073..0000000000 --- a/dss-linkis-node-execution/src/main/java/com/webank/wedatasphere/dss/linkis/node/execution/utils/LinkisUjesClientUtils.java +++ /dev/null @@ -1,55 +0,0 @@ -/* - * Copyright 2019 WeBank - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package com.webank.wedatasphere.dss.linkis.node.execution.utils; - -import com.webank.wedatasphere.dss.linkis.node.execution.conf.LinkisJobExecutionConfiguration; -import com.webank.wedatasphere.linkis.httpclient.dws.authentication.TokenAuthenticationStrategy; -import com.webank.wedatasphere.linkis.httpclient.dws.config.DWSClientConfig; -import com.webank.wedatasphere.linkis.httpclient.dws.config.DWSClientConfigBuilder; -import com.webank.wedatasphere.linkis.ujes.client.UJESClient; -import com.webank.wedatasphere.linkis.ujes.client.UJESClientImpl; - -import java.util.Map; -import java.util.concurrent.TimeUnit; - -/** - * Created by peacewong on 2019/9/20. - */ -public class LinkisUjesClientUtils { - - public static DWSClientConfig getClientConfig(String url, String user, String token, Map jobProps){ - DWSClientConfig clientConfig = ((DWSClientConfigBuilder) (DWSClientConfigBuilder.newBuilder() - .addUJESServerUrl(url) - .connectionTimeout(LinkisJobExecutionConfiguration.LINKIS_CONNECTION_TIMEOUT.getValue(jobProps)) - .discoveryEnabled(false).discoveryFrequency(1, TimeUnit.MINUTES) - .loadbalancerEnabled(true) - .maxConnectionSize(5) - .retryEnabled(false).readTimeout(LinkisJobExecutionConfiguration.LINKIS_CONNECTION_TIMEOUT.getValue(jobProps)) - .setAuthenticationStrategy(new TokenAuthenticationStrategy()) - .setAuthTokenKey(user).setAuthTokenValue(token))) - .setDWSVersion(LinkisJobExecutionConfiguration.LINKIS_API_VERSION.getValue(jobProps)).build(); - return clientConfig; - } - - public static UJESClient getUJESClient(String url, String user, String token, Map jobProps){ - return new UJESClientImpl(getClientConfig(url,user,token, jobProps)); - } - - - -} diff --git a/dss-scheduler-appjoint-core/pom.xml b/dss-scheduler-appjoint-core/pom.xml deleted file mode 100644 index 72f112372a..0000000000 --- a/dss-scheduler-appjoint-core/pom.xml +++ /dev/null @@ -1,76 +0,0 @@ - - - - - - dss - com.webank.wedatasphere.dss - 0.9.1 - - 4.0.0 - - dss-scheduler-appjoint-core - - - org.springframework - spring-beans - 5.0.7.RELEASE - - - com.webank.wedatasphere.dss - dss-appjoint-core - - - linkis-protocol - com.webank.wedatasphere.linkis - - - linkis-storage - com.webank.wedatasphere.linkis - - - ${dss.version} - - - com.google.code.gson - gson - 2.8.5 - - - - - - - org.apache.maven.plugins - maven-deploy-plugin - - - - net.alchim31.maven - scala-maven-plugin - - - org.apache.maven.plugins - maven-jar-plugin - - - - - \ No newline at end of file diff --git a/dss-scheduler-appjoint-core/src/main/java/com/webank/wedatasphere/dss/appjoint/scheduler/SchedulerAppJoint.java b/dss-scheduler-appjoint-core/src/main/java/com/webank/wedatasphere/dss/appjoint/scheduler/SchedulerAppJoint.java deleted file mode 100644 index 9923d90c73..0000000000 --- a/dss-scheduler-appjoint-core/src/main/java/com/webank/wedatasphere/dss/appjoint/scheduler/SchedulerAppJoint.java +++ /dev/null @@ -1,44 +0,0 @@ -/* - * Copyright 2019 WeBank - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package com.webank.wedatasphere.dss.appjoint.scheduler; - -import com.webank.wedatasphere.dss.appjoint.AppJoint; -import com.webank.wedatasphere.dss.appjoint.scheduler.hooks.ProjectPublishHook; -import com.webank.wedatasphere.dss.appjoint.scheduler.parser.ProjectParser; -import com.webank.wedatasphere.dss.appjoint.scheduler.service.SchedulerProjectService; -import com.webank.wedatasphere.dss.appjoint.scheduler.service.SchedulerSecurityService; -import com.webank.wedatasphere.dss.appjoint.scheduler.tuning.ProjectTuning; - -import java.io.Closeable; - -/** - * Created by enjoyyin on 2019/9/16. - */ -public interface SchedulerAppJoint extends AppJoint, Closeable { - - SchedulerProjectService getProjectService(); - - SchedulerSecurityService getSecurityService(); - - ProjectParser getProjectParser(); - ProjectTuning getProjectTuning(); - - ProjectPublishHook[] getProjectPublishHooks(); - - -} diff --git a/dss-scheduler-appjoint-core/src/main/java/com/webank/wedatasphere/dss/appjoint/scheduler/constant/SchedulerAppJointConstant.java b/dss-scheduler-appjoint-core/src/main/java/com/webank/wedatasphere/dss/appjoint/scheduler/constant/SchedulerAppJointConstant.java deleted file mode 100644 index 25d4e9d84e..0000000000 --- a/dss-scheduler-appjoint-core/src/main/java/com/webank/wedatasphere/dss/appjoint/scheduler/constant/SchedulerAppJointConstant.java +++ /dev/null @@ -1,30 +0,0 @@ -/* - * Copyright 2019 WeBank - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package com.webank.wedatasphere.dss.appjoint.scheduler.constant; - -/** - * Created by enjoyyin on 2019/11/5. - */ -public class SchedulerAppJointConstant { - public final static String PROXY_USER = "user.to.proxy"; - public final static String CONFIGURATION = "configuration"; - public final static String RUNTIME = "runtime"; - public final static String CATEGORY = "category"; - public final static String SENDEMAIL_NODE_TYPE = "linkis.appjoint.sendemail"; - public final static String NODE = "node"; -} diff --git a/dss-scheduler-appjoint-core/src/main/java/com/webank/wedatasphere/dss/appjoint/scheduler/entity/AbstractSchedulerNode.java b/dss-scheduler-appjoint-core/src/main/java/com/webank/wedatasphere/dss/appjoint/scheduler/entity/AbstractSchedulerNode.java deleted file mode 100644 index a5139d7d9c..0000000000 --- a/dss-scheduler-appjoint-core/src/main/java/com/webank/wedatasphere/dss/appjoint/scheduler/entity/AbstractSchedulerNode.java +++ /dev/null @@ -1,90 +0,0 @@ -/* - * Copyright 2019 WeBank - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package com.webank.wedatasphere.dss.appjoint.scheduler.entity; - -import com.webank.wedatasphere.dss.common.entity.node.DSSNode; - -import java.util.List; - -/** - * Created by enjoyyin on 2019/9/25. - */ -public abstract class AbstractSchedulerNode implements SchedulerNode { - - private DSSNode dssNode; - - @Override - public DSSNode getDssNode() { - return this.dssNode; - } - - @Override - public void setDssNode(DSSNode dssNode) { - this.dssNode = dssNode; - } - - @Override - public String getId() { - return dssNode.getId(); - } - - @Override - public void setId(String id) { - dssNode.setId(id); - } - - @Override - public String getNodeType() { - return dssNode.getNodeType(); - } - - @Override - public void setNodeType(String nodeType) { - dssNode.setNodeType(nodeType); - } - - @Override - public String getName() { - return dssNode.getName(); - } - - @Override - public void setName(String name) { - dssNode.setName(name); - } - - @Override - public void addDependency(String nodeName) { - dssNode.addDependency(nodeName); - } - - @Override - public void setDependency(List dependency) { - dssNode.setDependency(dependency); - } - - @Override - public void removeDependency(String nodeName) { - dssNode.removeDependency(nodeName); - } - - @Override - public List getDependencys() { - return dssNode.getDependencys(); - } -} diff --git a/dss-scheduler-appjoint-core/src/main/java/com/webank/wedatasphere/dss/appjoint/scheduler/entity/AbstractSchedulerProject.java b/dss-scheduler-appjoint-core/src/main/java/com/webank/wedatasphere/dss/appjoint/scheduler/entity/AbstractSchedulerProject.java deleted file mode 100644 index 3a8ad55eb8..0000000000 --- a/dss-scheduler-appjoint-core/src/main/java/com/webank/wedatasphere/dss/appjoint/scheduler/entity/AbstractSchedulerProject.java +++ /dev/null @@ -1,112 +0,0 @@ -/* - * Copyright 2019 WeBank - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package com.webank.wedatasphere.dss.appjoint.scheduler.entity; - -import com.webank.wedatasphere.dss.common.entity.project.DSSProject; -import com.webank.wedatasphere.dss.common.entity.project.ProjectVersion; - -import java.util.List; -import java.util.stream.Collectors; - -/** - * Created by enjoyyin on 2019/9/25. - */ -public abstract class AbstractSchedulerProject implements SchedulerProject { - - private Long id; - private String name; - private String description; - - private DSSProject dssProject; - private List schedulerFlows; - - private List projectVersions; - - public List getSchedulerFlows() { - return schedulerFlows; - } - - public void setSchedulerFlows(List schedulerFlows) { - this.schedulerFlows = schedulerFlows; - } - - @Override - public Long getId() { - return this.id; - } - - @Override - public void setId(Long id) { - this.id = id; - } - - @Override - public String getName() { - return this.name; - } - - @Override - public void setName(String name) { - this.name = name; - } - - @Override - public String getDescription() { - return this.description; - } - - @Override - public void setDescription(String description) { - this.description = description; - } - - @Override - public String getPorjectGroup() { - return null; - } - - @Override - public void setProjectGroup(String projectGroup) { - - } - - @Override - public List getProjectVersions() { - return this.projectVersions; - } - - @Override - public void setProjectVersions(List projectVersions) { - this.projectVersions = projectVersions.stream().map(f ->(SchedulerProjectVersionForFlows)f).collect(Collectors.toList()); - } - - @Override - public void addProjectVersion(ProjectVersion projectVersion) { - this.projectVersions.add((SchedulerProjectVersionForFlows)projectVersion); - } - - @Override - public DSSProject getDssProject() { - return this.dssProject; - } - - @Override - public void setDssProject(DSSProject dssProject) { - this.dssProject = dssProject; - } -} diff --git a/dss-scheduler-appjoint-core/src/main/java/com/webank/wedatasphere/dss/appjoint/scheduler/entity/FileProjectVersion.java b/dss-scheduler-appjoint-core/src/main/java/com/webank/wedatasphere/dss/appjoint/scheduler/entity/FileProjectVersion.java deleted file mode 100644 index 0a09f6081d..0000000000 --- a/dss-scheduler-appjoint-core/src/main/java/com/webank/wedatasphere/dss/appjoint/scheduler/entity/FileProjectVersion.java +++ /dev/null @@ -1,32 +0,0 @@ -/* - * Copyright 2019 WeBank - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package com.webank.wedatasphere.dss.appjoint.scheduler.entity; - -import com.webank.wedatasphere.dss.common.entity.project.ProjectVersion; - -import java.io.File; - -/** - * Created by enjoyyin on 2019/9/16. - */ -public interface FileProjectVersion extends ProjectVersion { - - File getProjectFile(); - - void setProjectFile(); -} diff --git a/dss-scheduler-appjoint-core/src/main/java/com/webank/wedatasphere/dss/appjoint/scheduler/entity/ReadNode.java b/dss-scheduler-appjoint-core/src/main/java/com/webank/wedatasphere/dss/appjoint/scheduler/entity/ReadNode.java deleted file mode 100644 index 90f6951e68..0000000000 --- a/dss-scheduler-appjoint-core/src/main/java/com/webank/wedatasphere/dss/appjoint/scheduler/entity/ReadNode.java +++ /dev/null @@ -1,28 +0,0 @@ -/* - * Copyright 2019 WeBank - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package com.webank.wedatasphere.dss.appjoint.scheduler.entity; - -/** - * Created by enjoyyin on 2019/9/7. - */ -public interface ReadNode extends SchedulerNode { - String[] getShareNodeIds(); - void setShareNodeIds(String[] nodeIds); - SchedulerNode getSchedulerNode(); - void setSchedulerNode(SchedulerNode schedulerNode); -} diff --git a/dss-scheduler-appjoint-core/src/main/java/com/webank/wedatasphere/dss/appjoint/scheduler/entity/SchedulerEdge.java b/dss-scheduler-appjoint-core/src/main/java/com/webank/wedatasphere/dss/appjoint/scheduler/entity/SchedulerEdge.java deleted file mode 100644 index aee1e72292..0000000000 --- a/dss-scheduler-appjoint-core/src/main/java/com/webank/wedatasphere/dss/appjoint/scheduler/entity/SchedulerEdge.java +++ /dev/null @@ -1,29 +0,0 @@ -/* - * Copyright 2019 WeBank - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package com.webank.wedatasphere.dss.appjoint.scheduler.entity; - -import com.webank.wedatasphere.dss.common.entity.node.DSSEdge; - -/** - * Created by enjoyyin on 2019/9/7. - */ -public interface SchedulerEdge { - DSSEdge getDssEdge(); - - void setDssEdge(DSSEdge dssEdge); -} diff --git a/dss-scheduler-appjoint-core/src/main/java/com/webank/wedatasphere/dss/appjoint/scheduler/entity/SchedulerEdgeDefault.java b/dss-scheduler-appjoint-core/src/main/java/com/webank/wedatasphere/dss/appjoint/scheduler/entity/SchedulerEdgeDefault.java deleted file mode 100644 index db663fe540..0000000000 --- a/dss-scheduler-appjoint-core/src/main/java/com/webank/wedatasphere/dss/appjoint/scheduler/entity/SchedulerEdgeDefault.java +++ /dev/null @@ -1,38 +0,0 @@ -/* - * Copyright 2019 WeBank - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package com.webank.wedatasphere.dss.appjoint.scheduler.entity; - -import com.webank.wedatasphere.dss.common.entity.node.DSSEdge; - -/** - * Created by allenlliu on 2019/9/19. - */ -public class SchedulerEdgeDefault implements SchedulerEdge { - private DSSEdge dssEdge; - - @Override - public DSSEdge getDssEdge() { - return dssEdge; - } - - @Override - public void setDssEdge(DSSEdge dssEdge) { - this.dssEdge = dssEdge; - } - -} diff --git a/dss-scheduler-appjoint-core/src/main/java/com/webank/wedatasphere/dss/appjoint/scheduler/entity/SchedulerFlow.java b/dss-scheduler-appjoint-core/src/main/java/com/webank/wedatasphere/dss/appjoint/scheduler/entity/SchedulerFlow.java deleted file mode 100644 index e3cd2a5422..0000000000 --- a/dss-scheduler-appjoint-core/src/main/java/com/webank/wedatasphere/dss/appjoint/scheduler/entity/SchedulerFlow.java +++ /dev/null @@ -1,161 +0,0 @@ -/* - * Copyright 2019 WeBank - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package com.webank.wedatasphere.dss.appjoint.scheduler.entity; - -import com.webank.wedatasphere.dss.common.entity.Resource; -import com.webank.wedatasphere.dss.common.entity.flow.Flow; -import com.webank.wedatasphere.dss.common.entity.flow.FlowVersion; - -import java.util.Date; -import java.util.List; -import java.util.Map; -import java.util.stream.Collectors; - -/** - * Created by enjoyyin on 2019/9/7. - */ -public class SchedulerFlow implements Flow { - private Long id; - private String name; - private String description; - private String type; - private List children; - private Boolean isRootFlow; - private String userProxy; - private List schedulerNodes; - private List schedulerEdges; - private List flowResources; - private List> flowProperties; - - public String getUserProxy() { - return userProxy; - } - - public void setUserProxy(String userProxy) { - this.userProxy = userProxy; - } - - public List getSchedulerNodes() { - return schedulerNodes; - } - - public void setSchedulerNodes(List schedulerNodes) { - this.schedulerNodes = schedulerNodes; - } - - public List getSchedulerEdges() { - return schedulerEdges; - } - - public void setSchedulerEdges(List schedulerEdges) { - this.schedulerEdges = schedulerEdges; - } - - public List getFlowResources() { - return flowResources; - } - - public void setFlowResources(List flowResources) { - this.flowResources = flowResources; - } - - public List> getFlowProperties() { - return flowProperties; - } - - public void setFlowProperties(List> flowProperties) { - this.flowProperties = flowProperties; - } - - @Override - public Long getId() { - return this.id; - } - - @Override - public void setId(Long id) { - this.id = id; - } - - @Override - public String getName() { - return this.name; - } - - @Override - public void setName(String name) { - this.name = name; - } - - @Override - public String getFlowType() { - return this.type; - } - - @Override - public void setFlowType(String flowType) { - this.type = flowType; - - } - - @Override - public String getDescription() { - return this.description; - } - - @Override - public void setDescription(String description) { - this.description = description; - } - - @Override - public Boolean getRootFlow() { - return this.isRootFlow; - } - - public void setRootFlow(Boolean rootFlow) { - isRootFlow = rootFlow; - } - - @Override - public List getChildren() { - return this.children; - } - - @Override - public void setChildren(List children) { - this.children = children.stream().map(f ->(SchedulerFlow)f).collect(Collectors.toList()); - } - - - @Override - public List getFlowVersions() { - return null; - } - - @Override - public void setFlowVersions(List flowVersions) { - - } - - - @Override - public void addFlowVersion(FlowVersion flowVersion) { - - } -} diff --git a/dss-scheduler-appjoint-core/src/main/java/com/webank/wedatasphere/dss/appjoint/scheduler/entity/SchedulerNode.java b/dss-scheduler-appjoint-core/src/main/java/com/webank/wedatasphere/dss/appjoint/scheduler/entity/SchedulerNode.java deleted file mode 100644 index 612c2e25d6..0000000000 --- a/dss-scheduler-appjoint-core/src/main/java/com/webank/wedatasphere/dss/appjoint/scheduler/entity/SchedulerNode.java +++ /dev/null @@ -1,31 +0,0 @@ -/* - * Copyright 2019 WeBank - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package com.webank.wedatasphere.dss.appjoint.scheduler.entity; - - -import com.webank.wedatasphere.dss.common.entity.node.DSSNode; -import com.webank.wedatasphere.dss.common.entity.node.Node; - -/** - * Created by enjoyyin on 2019/9/7. - */ -public interface SchedulerNode extends Node { - DSSNode getDssNode(); - - void setDssNode(DSSNode dssNode); -} diff --git a/dss-scheduler-appjoint-core/src/main/java/com/webank/wedatasphere/dss/appjoint/scheduler/entity/SchedulerProject.java b/dss-scheduler-appjoint-core/src/main/java/com/webank/wedatasphere/dss/appjoint/scheduler/entity/SchedulerProject.java deleted file mode 100644 index d4158b0f4c..0000000000 --- a/dss-scheduler-appjoint-core/src/main/java/com/webank/wedatasphere/dss/appjoint/scheduler/entity/SchedulerProject.java +++ /dev/null @@ -1,29 +0,0 @@ -/* - * Copyright 2019 WeBank - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package com.webank.wedatasphere.dss.appjoint.scheduler.entity; - -import com.webank.wedatasphere.dss.common.entity.project.DSSProject; -import com.webank.wedatasphere.dss.common.entity.project.Project; - -/** - * Created by enjoyyin on 2019/9/16. - */ -public interface SchedulerProject extends Project { - DSSProject getDssProject(); - void setDssProject(DSSProject dssProject); -} diff --git a/dss-scheduler-appjoint-core/src/main/java/com/webank/wedatasphere/dss/appjoint/scheduler/entity/SchedulerProjectVersionForFlows.java b/dss-scheduler-appjoint-core/src/main/java/com/webank/wedatasphere/dss/appjoint/scheduler/entity/SchedulerProjectVersionForFlows.java deleted file mode 100644 index bc915cb47c..0000000000 --- a/dss-scheduler-appjoint-core/src/main/java/com/webank/wedatasphere/dss/appjoint/scheduler/entity/SchedulerProjectVersionForFlows.java +++ /dev/null @@ -1,73 +0,0 @@ -/* - * Copyright 2019 WeBank - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package com.webank.wedatasphere.dss.appjoint.scheduler.entity; - -import com.webank.wedatasphere.dss.common.entity.flow.Flow; -import com.webank.wedatasphere.dss.common.entity.project.ProjectVersionForFlows; - -import java.util.ArrayList; -import java.util.List; -import java.util.stream.Collectors; - -/** - * Created by enjoyyin on 2019/9/26. - */ -public class SchedulerProjectVersionForFlows implements ProjectVersionForFlows { - - private List allSchedulerFlows = new ArrayList<>(); - - @Override - public String getVersion() { - return null; - } - - @Override - public void setVersion() { - - } - - @Override - public String getComment() { - return null; - } - - @Override - public void setComment(String comment) { - - } - - @Override - public List getFlows() { - return this.allSchedulerFlows; - } - - @Override - public void setFlows(List flows) { - this.allSchedulerFlows = flows.stream().map(f ->(SchedulerFlow)f).collect(Collectors.toList()); - } - - @Override - public void addFlow(Flow flow) { - this.allSchedulerFlows.add((SchedulerFlow) flow); - } - - @Override - public void removeFlow(Flow flow) { - - } -} diff --git a/dss-scheduler-appjoint-core/src/main/java/com/webank/wedatasphere/dss/appjoint/scheduler/entity/ShareNode.java b/dss-scheduler-appjoint-core/src/main/java/com/webank/wedatasphere/dss/appjoint/scheduler/entity/ShareNode.java deleted file mode 100644 index 41a0bb4948..0000000000 --- a/dss-scheduler-appjoint-core/src/main/java/com/webank/wedatasphere/dss/appjoint/scheduler/entity/ShareNode.java +++ /dev/null @@ -1,28 +0,0 @@ -/* - * Copyright 2019 WeBank - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package com.webank.wedatasphere.dss.appjoint.scheduler.entity; - -/** - * Created by enjoyyin on 2019/9/7. - */ -public interface ShareNode extends SchedulerNode { - int getShareTimes(); - void setShareTimes(int num); - SchedulerNode getSchedulerNode(); - void setSchedulerNode(SchedulerNode node); -} diff --git a/dss-scheduler-appjoint-core/src/main/java/com/webank/wedatasphere/dss/appjoint/scheduler/hooks/AbstractFlowPublishHook.java b/dss-scheduler-appjoint-core/src/main/java/com/webank/wedatasphere/dss/appjoint/scheduler/hooks/AbstractFlowPublishHook.java deleted file mode 100644 index dcb2f2a49d..0000000000 --- a/dss-scheduler-appjoint-core/src/main/java/com/webank/wedatasphere/dss/appjoint/scheduler/hooks/AbstractFlowPublishHook.java +++ /dev/null @@ -1,64 +0,0 @@ -/* - * Copyright 2019 WeBank - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package com.webank.wedatasphere.dss.appjoint.scheduler.hooks; - -import com.webank.wedatasphere.dss.appjoint.scheduler.entity.SchedulerNode; -import com.webank.wedatasphere.dss.appjoint.scheduler.entity.SchedulerFlow; -import com.webank.wedatasphere.dss.common.exception.DSSErrorException; -import com.webank.wedatasphere.dss.common.utils.DSSExceptionUtils; - -import java.util.Arrays; -import java.util.List; - -/** - * Created by enjoyyin on 2019/9/25. - */ -public abstract class AbstractFlowPublishHook implements FlowPublishHook { - - private NodePublishHook[] nodePublishHooks; - - @Override - public void setNodeHooks(NodePublishHook[] nodePublishHooks) { - this.nodePublishHooks = nodePublishHooks; - } - - @Override - public NodePublishHook[] getNodeHooks() { - return this.nodePublishHooks; - } - - @Override - public void prePublish(SchedulerFlow flow) throws DSSErrorException { - // 调用nodeHook - List schedulerNodes = flow.getSchedulerNodes(); - Arrays.stream(getNodeHooks()).forEach(h -> invokePreNodePublishHook(schedulerNodes,h)); - //递归调用自身方法 - if(flow.getChildren() !=null){ - flow.getChildren().forEach(DSSExceptionUtils.handling(this::prePublish)); - } - } - - @Override - public void postPublish(SchedulerFlow flow) { - // TODO: 2019/9/25 递归调用nodepublishHook - } - - private void invokePreNodePublishHook(List schedulerNodes, NodePublishHook nodePublishHook){ - schedulerNodes.forEach(DSSExceptionUtils.handling(nodePublishHook::prePublish)); - } -} diff --git a/dss-scheduler-appjoint-core/src/main/java/com/webank/wedatasphere/dss/appjoint/scheduler/hooks/AbstractNodePublishHook.java b/dss-scheduler-appjoint-core/src/main/java/com/webank/wedatasphere/dss/appjoint/scheduler/hooks/AbstractNodePublishHook.java deleted file mode 100644 index fd43549c52..0000000000 --- a/dss-scheduler-appjoint-core/src/main/java/com/webank/wedatasphere/dss/appjoint/scheduler/hooks/AbstractNodePublishHook.java +++ /dev/null @@ -1,36 +0,0 @@ -/* - * Copyright 2019 WeBank - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package com.webank.wedatasphere.dss.appjoint.scheduler.hooks; - -import com.webank.wedatasphere.dss.appjoint.scheduler.entity.SchedulerNode; -import com.webank.wedatasphere.dss.common.exception.DSSErrorException; - -/** - * Created by enjoyyin on 2019/9/25. - */ -public abstract class AbstractNodePublishHook implements NodePublishHook { - @Override - public void prePublish(SchedulerNode schedulerNode) throws DSSErrorException { - //默认的不做处理 - } - - @Override - public void postPublish(SchedulerNode schedulerNode) { - //默认的不做处理 - } -} diff --git a/dss-scheduler-appjoint-core/src/main/java/com/webank/wedatasphere/dss/appjoint/scheduler/hooks/AbstractProjectPublishHook.java b/dss-scheduler-appjoint-core/src/main/java/com/webank/wedatasphere/dss/appjoint/scheduler/hooks/AbstractProjectPublishHook.java deleted file mode 100644 index 5d059cd1f9..0000000000 --- a/dss-scheduler-appjoint-core/src/main/java/com/webank/wedatasphere/dss/appjoint/scheduler/hooks/AbstractProjectPublishHook.java +++ /dev/null @@ -1,64 +0,0 @@ -/* - * Copyright 2019 WeBank - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package com.webank.wedatasphere.dss.appjoint.scheduler.hooks; - -import com.webank.wedatasphere.dss.appjoint.scheduler.entity.AbstractSchedulerProject; -import com.webank.wedatasphere.dss.appjoint.scheduler.entity.SchedulerFlow; -import com.webank.wedatasphere.dss.appjoint.scheduler.entity.SchedulerProject; -import com.webank.wedatasphere.dss.common.exception.DSSErrorException; -import com.webank.wedatasphere.dss.common.utils.DSSExceptionUtils; - -import java.util.Arrays; -import java.util.List; - -/** - * Created by enjoyyin on 2019/9/25. - */ -public abstract class AbstractProjectPublishHook implements ProjectPublishHook { - - @Override - public FlowPublishHook[] getFlowPublishHooks() { - return flowPublishHooks; - } - - @Override - public void setFlowPublishHooks(FlowPublishHook[] flowPublishHooks) { - this.flowPublishHooks = flowPublishHooks; - } - - private FlowPublishHook[] flowPublishHooks; - - @Override - public void prePublish(SchedulerProject project) throws DSSErrorException { - //这里强转也是因为SchedulerProject ， - //如果azkaban有实现，应该要继承AbstractSchedulerProject 而不是直接继承SchedulerProject - //调用flowHook的prePublish - AbstractSchedulerProject schedulerProject = (AbstractSchedulerProject) project; - List schedulerFlows = schedulerProject.getSchedulerFlows(); - Arrays.stream(getFlowPublishHooks()).forEach(h -> invokePreFlowPublishHook(schedulerFlows,h)); - } - - @Override - public void postPublish(SchedulerProject project) throws DSSErrorException { - //// TODO: 2019/9/25 递归调用flowHooks 的post - } - - private void invokePreFlowPublishHook(List schedulerFlows,FlowPublishHook flowPublishHook){ - schedulerFlows.forEach(DSSExceptionUtils.handling(flowPublishHook::prePublish)); - } -} diff --git a/dss-scheduler-appjoint-core/src/main/java/com/webank/wedatasphere/dss/appjoint/scheduler/hooks/FlowPublishHook.java b/dss-scheduler-appjoint-core/src/main/java/com/webank/wedatasphere/dss/appjoint/scheduler/hooks/FlowPublishHook.java deleted file mode 100644 index e75420de7d..0000000000 --- a/dss-scheduler-appjoint-core/src/main/java/com/webank/wedatasphere/dss/appjoint/scheduler/hooks/FlowPublishHook.java +++ /dev/null @@ -1,32 +0,0 @@ -/* - * Copyright 2019 WeBank - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package com.webank.wedatasphere.dss.appjoint.scheduler.hooks; - - -import com.webank.wedatasphere.dss.appjoint.scheduler.entity.SchedulerFlow; -import com.webank.wedatasphere.dss.common.exception.DSSErrorException; - -/** - * Created by enjoyyin on 2019/11/7. - */ -public interface FlowPublishHook { - void prePublish(SchedulerFlow flow) throws DSSErrorException; - void postPublish(SchedulerFlow flow)throws DSSErrorException; - void setNodeHooks(NodePublishHook[] nodePublishHooks); - NodePublishHook[] getNodeHooks(); -} diff --git a/dss-scheduler-appjoint-core/src/main/java/com/webank/wedatasphere/dss/appjoint/scheduler/hooks/NodePublishHook.java b/dss-scheduler-appjoint-core/src/main/java/com/webank/wedatasphere/dss/appjoint/scheduler/hooks/NodePublishHook.java deleted file mode 100644 index 606dbd1715..0000000000 --- a/dss-scheduler-appjoint-core/src/main/java/com/webank/wedatasphere/dss/appjoint/scheduler/hooks/NodePublishHook.java +++ /dev/null @@ -1,30 +0,0 @@ -/* - * Copyright 2019 WeBank - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package com.webank.wedatasphere.dss.appjoint.scheduler.hooks; - - -import com.webank.wedatasphere.dss.appjoint.scheduler.entity.SchedulerNode; -import com.webank.wedatasphere.dss.common.exception.DSSErrorException; - -/** - * Created by enjoyyin on 2019/9/7. - */ -public interface NodePublishHook { - void prePublish(SchedulerNode schedulerNode) throws DSSErrorException; - void postPublish(SchedulerNode schedulerNode)throws DSSErrorException; -} diff --git a/dss-scheduler-appjoint-core/src/main/java/com/webank/wedatasphere/dss/appjoint/scheduler/hooks/ProjectPublishHook.java b/dss-scheduler-appjoint-core/src/main/java/com/webank/wedatasphere/dss/appjoint/scheduler/hooks/ProjectPublishHook.java deleted file mode 100644 index aac909c5aa..0000000000 --- a/dss-scheduler-appjoint-core/src/main/java/com/webank/wedatasphere/dss/appjoint/scheduler/hooks/ProjectPublishHook.java +++ /dev/null @@ -1,31 +0,0 @@ -/* - * Copyright 2019 WeBank - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package com.webank.wedatasphere.dss.appjoint.scheduler.hooks; - -import com.webank.wedatasphere.dss.appjoint.scheduler.entity.SchedulerProject; -import com.webank.wedatasphere.dss.common.exception.DSSErrorException; - -/** - * Created by enjoyyin on 2019/9/18 - */ -public interface ProjectPublishHook { - void prePublish(SchedulerProject project) throws DSSErrorException; - void postPublish(SchedulerProject project) throws DSSErrorException; - FlowPublishHook[] getFlowPublishHooks(); - void setFlowPublishHooks(FlowPublishHook[] flowPublishHooks); -} diff --git a/dss-scheduler-appjoint-core/src/main/java/com/webank/wedatasphere/dss/appjoint/scheduler/order/Order.java b/dss-scheduler-appjoint-core/src/main/java/com/webank/wedatasphere/dss/appjoint/scheduler/order/Order.java deleted file mode 100644 index 8baf3d0a42..0000000000 --- a/dss-scheduler-appjoint-core/src/main/java/com/webank/wedatasphere/dss/appjoint/scheduler/order/Order.java +++ /dev/null @@ -1,28 +0,0 @@ -/* - * Copyright 2019 WeBank - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package com.webank.wedatasphere.dss.appjoint.scheduler.order; - -/** - * order接口用于标识parser和tunning，hook的调用顺序 - * Created by enjoyyin on 2019/9/30. - */ -public interface Order { - - int getOrder(); - -} diff --git a/dss-scheduler-appjoint-core/src/main/java/com/webank/wedatasphere/dss/appjoint/scheduler/parser/AbstractFlowParser.java b/dss-scheduler-appjoint-core/src/main/java/com/webank/wedatasphere/dss/appjoint/scheduler/parser/AbstractFlowParser.java deleted file mode 100644 index 96a9f2aed6..0000000000 --- a/dss-scheduler-appjoint-core/src/main/java/com/webank/wedatasphere/dss/appjoint/scheduler/parser/AbstractFlowParser.java +++ /dev/null @@ -1,117 +0,0 @@ -/* - * Copyright 2019 WeBank - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package com.webank.wedatasphere.dss.appjoint.scheduler.parser; - -import com.google.gson.Gson; -import com.google.gson.JsonArray; -import com.google.gson.JsonObject; -import com.google.gson.JsonParser; -import com.google.gson.reflect.TypeToken; -import com.webank.wedatasphere.dss.appjoint.scheduler.entity.SchedulerEdge; -import com.webank.wedatasphere.dss.appjoint.scheduler.entity.SchedulerEdgeDefault; -import com.webank.wedatasphere.dss.appjoint.scheduler.entity.SchedulerNode; -import com.webank.wedatasphere.dss.common.entity.Resource; -import com.webank.wedatasphere.dss.appjoint.scheduler.entity.SchedulerFlow; -import com.webank.wedatasphere.dss.common.entity.flow.DSSJSONFlow; -import com.webank.wedatasphere.dss.common.entity.flow.Flow; -import com.webank.wedatasphere.dss.common.entity.node.DSSEdge; -import com.webank.wedatasphere.dss.common.entity.node.DSSEdgeDefault; -import com.webank.wedatasphere.dss.common.entity.node.DSSNode; -import com.webank.wedatasphere.dss.common.entity.node.DSSNodeDefault; -import org.springframework.beans.BeanUtils; - -import java.util.*; - - -/** - * Created by enjoyyin on 2019/9/7. - */ -public abstract class AbstractFlowParser implements FlowParser { - - private NodeParser[] nodeParsers; - - @Override - public void setNodeParsers(NodeParser[] nodeParsers) { - this.nodeParsers = nodeParsers; - } - - @Override - public NodeParser[] getNodeParsers() { - return nodeParsers; - } - - //留白 - protected void downloadFlowResources(){} - - //留白 - protected void dealFlowResources(){} - - protected void dealFlowProperties(Flow flow){} - - @Override - public SchedulerFlow parseFlow(DSSJSONFlow flow) { - downloadFlowResources(); - dealFlowResources(); - dealFlowProperties(flow); - return resolveDSSJSONFlow(flow); - } - - // 解析DSSJSONFlow，生成DSSNode - public SchedulerFlow resolveDSSJSONFlow(DSSJSONFlow jsonFlow){ - SchedulerFlow schedulerFlow = createSchedulerFlow(); - BeanUtils.copyProperties(jsonFlow,schedulerFlow,"children"); - JsonParser parser = new JsonParser(); - JsonObject jsonObject = parser.parse(jsonFlow.getJson()).getAsJsonObject(); - JsonArray nodeJsonArray = jsonObject.getAsJsonArray("nodes"); - Gson gson = new Gson(); - List dssNodes = gson.fromJson(nodeJsonArray, new TypeToken>() { - }.getType()); - List schedulerNodeList = new ArrayList<>(); - List schedulerEdgeList = new ArrayList<>(); - for (DSSNode dssNode : dssNodes) { - Optional firstNodeParser = Arrays.stream(getNodeParsers()) - .filter(p -> p.ifNodeCanParse(dssNode)) - .sorted((p1, p2) -> p2.getOrder() - p1.getOrder()) - .findFirst(); - SchedulerNode schedulerNode = firstNodeParser.orElseThrow(()->new IllegalArgumentException("NodeParser个数应该大于0")).parseNode(dssNode); - schedulerNodeList.add(schedulerNode); - } - JsonArray edgeJsonArray = jsonObject.getAsJsonArray("edges"); - List dssEdges = gson.fromJson(edgeJsonArray, new TypeToken>() { - }.getType()); - for (DSSEdge dssEdge : dssEdges) { - SchedulerEdge schedulerEdge = new SchedulerEdgeDefault(); - schedulerEdge.setDssEdge(dssEdge); - schedulerEdgeList.add(schedulerEdge); - - } - JsonArray proJsonArray = jsonObject.getAsJsonArray("props"); - List> props = gson.fromJson(proJsonArray, new TypeToken>>() { - }.getType()); - JsonArray resourcesJsonArray = jsonObject.getAsJsonArray("resources"); - List resources = gson.fromJson(resourcesJsonArray, new TypeToken>() { - }.getType()); - schedulerFlow.setFlowResources(resources); - schedulerFlow.setFlowProperties(props); - schedulerFlow.setSchedulerEdges(schedulerEdgeList); - schedulerFlow.setSchedulerNodes(schedulerNodeList); - return schedulerFlow; - } - - protected SchedulerFlow createSchedulerFlow(){return new SchedulerFlow();} -} diff --git a/dss-scheduler-appjoint-core/src/main/java/com/webank/wedatasphere/dss/appjoint/scheduler/parser/AbstractNodeParser.java b/dss-scheduler-appjoint-core/src/main/java/com/webank/wedatasphere/dss/appjoint/scheduler/parser/AbstractNodeParser.java deleted file mode 100644 index c12f0ab767..0000000000 --- a/dss-scheduler-appjoint-core/src/main/java/com/webank/wedatasphere/dss/appjoint/scheduler/parser/AbstractNodeParser.java +++ /dev/null @@ -1,40 +0,0 @@ -/* - * Copyright 2019 WeBank - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package com.webank.wedatasphere.dss.appjoint.scheduler.parser; - -import com.webank.wedatasphere.dss.appjoint.scheduler.entity.SchedulerNode; -import com.webank.wedatasphere.dss.appjoint.scheduler.entity.SchedulerEdge; - -import java.util.List; - -/** - * Created by enjoyyin on 2019/9/7. - */ -public abstract class AbstractNodeParser implements NodeParser { - //留白 - protected Boolean downloadNodeResource(){return null;}; - - //留白 - protected Boolean downloadNodeContent(){return null;}; - - //留白 - protected void getNodeVariables(){}; - - protected List resolveDependencys(SchedulerNode node, List flowEdges){return null;}; - -} diff --git a/dss-scheduler-appjoint-core/src/main/java/com/webank/wedatasphere/dss/appjoint/scheduler/parser/AbstractProjectParser.java b/dss-scheduler-appjoint-core/src/main/java/com/webank/wedatasphere/dss/appjoint/scheduler/parser/AbstractProjectParser.java deleted file mode 100644 index a7d481d24b..0000000000 --- a/dss-scheduler-appjoint-core/src/main/java/com/webank/wedatasphere/dss/appjoint/scheduler/parser/AbstractProjectParser.java +++ /dev/null @@ -1,105 +0,0 @@ -/* - * Copyright 2019 WeBank - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package com.webank.wedatasphere.dss.appjoint.scheduler.parser; - -import com.webank.wedatasphere.dss.appjoint.scheduler.entity.AbstractSchedulerProject; -import com.webank.wedatasphere.dss.appjoint.scheduler.entity.SchedulerProjectVersionForFlows; -import com.webank.wedatasphere.dss.appjoint.scheduler.entity.SchedulerFlow; -import com.webank.wedatasphere.dss.appjoint.scheduler.entity.SchedulerProject; -import com.webank.wedatasphere.dss.common.entity.flow.DSSFlow; -import com.webank.wedatasphere.dss.common.entity.flow.DSSJSONFlow; -import com.webank.wedatasphere.dss.common.entity.project.DSSProject; -import com.webank.wedatasphere.dss.common.entity.project.DSSJSONProject; -import com.webank.wedatasphere.dss.common.entity.project.ProjectVersionForFlows; -import org.springframework.beans.BeanUtils; - -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; -import java.util.stream.Collectors; - -/** - * Created by enjoyyin on 2019/9/16. - */ -public abstract class AbstractProjectParser implements ProjectParser { - - private FlowParser[] flowParsers; - - @Override - public void setFlowParsers(FlowParser[] flowParsers) { - this.flowParsers = flowParsers; - } - - @Override - public FlowParser[] getFlowParsers() { - return flowParsers; - } - - public DSSJSONProject parseToDssJsonProject(DSSProject dssProject){ - DSSJSONProject dssJsonProject = new DSSJSONProject(); - BeanUtils.copyProperties(dssProject, dssJsonProject,"flows","projectVersions"); - List dwsFlows = dssProject.getFlows(); - List dssJsonFlows = dwsFlows.stream().map(this::toDssJsonFlow).collect(Collectors.toList()); - dssJsonProject.setFlows(dssJsonFlows); - return dssJsonProject; - } - - private DSSJSONFlow toDssJsonFlow(DSSFlow dssFlow){ - DSSJSONFlow dssJsonFlow = new DSSJSONFlow(); - BeanUtils.copyProperties(dssFlow, dssJsonFlow,"children","flowVersions"); - dssJsonFlow.setJson(dssFlow.getLatestVersion().getJson()); - if(dssFlow.getChildren() != null){ - dssJsonFlow.setChildren(dssFlow.getChildren().stream().map(this::toDssJsonFlow).collect(Collectors.toList())); - } - return dssJsonFlow; - } - - - public SchedulerProject parseProject(DSSJSONProject project){ - AbstractSchedulerProject schedulerProject = createSchedulerProject(); - SchedulerProjectVersionForFlows projectVersionForFlows = new SchedulerProjectVersionForFlows(); - schedulerProject.setProjectVersions(new ArrayList()); - schedulerProject.addProjectVersion(projectVersionForFlows); - BeanUtils.copyProperties(project,schedulerProject,"projectVersions"); - List schedulerFlows = project.getFlows().stream().map(f -> invokeFlowParser(projectVersionForFlows,f, getFlowParsers())).collect(Collectors.toList()); - schedulerProject.setSchedulerFlows(schedulerFlows); - return schedulerProject; - } - - private SchedulerFlow invokeFlowParser(ProjectVersionForFlows projectVersionForFlows, DSSJSONFlow dssJsonFlow, FlowParser[] flowParsers){ - List flowParsersF = Arrays.stream(flowParsers).filter(f -> f.ifFlowCanParse(dssJsonFlow)).collect(Collectors.toList()); - // TODO: 2019/9/25 如果flowParsers数量>1 ||<=0抛出异常 - SchedulerFlow schedulerFlow = flowParsersF.get(0).parseFlow(dssJsonFlow); - //收集所有的不分层级的flow？ - projectVersionForFlows.addFlow(schedulerFlow); - if(dssJsonFlow.getChildren() != null){ - List schedulerFlows = dssJsonFlow.getChildren().stream().map(f -> invokeFlowParser(projectVersionForFlows,f, flowParsers)).collect(Collectors.toList()); - schedulerFlow.setChildren(schedulerFlows); - } - return schedulerFlow; - } - - @Override - public SchedulerProject parseProject(DSSProject dssProject) { - SchedulerProject schedulerProject = parseProject(parseToDssJsonProject(dssProject)); - schedulerProject.setDssProject(dssProject); - return schedulerProject; - } - - protected abstract AbstractSchedulerProject createSchedulerProject(); -} diff --git a/dss-scheduler-appjoint-core/src/main/java/com/webank/wedatasphere/dss/appjoint/scheduler/parser/AbstractReadNodeParser.java b/dss-scheduler-appjoint-core/src/main/java/com/webank/wedatasphere/dss/appjoint/scheduler/parser/AbstractReadNodeParser.java deleted file mode 100644 index 68e8d970e7..0000000000 --- a/dss-scheduler-appjoint-core/src/main/java/com/webank/wedatasphere/dss/appjoint/scheduler/parser/AbstractReadNodeParser.java +++ /dev/null @@ -1,77 +0,0 @@ -/* - * Copyright 2019 WeBank - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package com.webank.wedatasphere.dss.appjoint.scheduler.parser; - -import com.webank.wedatasphere.dss.appjoint.scheduler.entity.SchedulerNode; -import com.webank.wedatasphere.dss.common.entity.node.DSSNode; -import com.webank.wedatasphere.dss.appjoint.scheduler.entity.ReadNode; - -import java.util.Arrays; -import java.util.Map; - -/** - * Created by enjoyyin on 2019/10/31. - */ -public abstract class AbstractReadNodeParser implements ContextNodeParser { - - @Override - public String[] getShareNodeIds(SchedulerNode node) { - //需根据节点的参数进行解析生成 - Map jobParams = node.getDssNode().getParams(); - if(jobParams == null) return null; - Map configuration =(Map) jobParams.get("configuration"); - Map runtime = (Map) configuration.get("runtime"); - if(runtime.get("category").toString().equals("node")) { - String content = runtime.get("content").toString().replace("[","").replace("]",""); - if (content != null) { - String[] res = Arrays.stream(content.split(",")).map(String::trim).toArray(String[]::new); - return res; - } else { - return null; - } - }else { - return null; - } - } - - @Override - public void setReadNodeContext(SchedulerNode node) { - // TODO: 2019/10/31 - } - - @Override - public ReadNode parseNode(SchedulerNode node) { - ReadNode readNode = createReadNode(); - readNode.setDssNode(node.getDssNode()); - readNode.setSchedulerNode(node); - readNode.setShareNodeIds(getShareNodeIds(node)); - return readNode; - } - - @Override - public SchedulerNode parseNode(DSSNode dssNode) { - SchedulerNode schedulerNode = createSchedulerNode(); - schedulerNode.setDssNode(dssNode); - return parseNode(schedulerNode); - } - - - protected abstract ReadNode createReadNode(); - - protected abstract SchedulerNode createSchedulerNode(); -} diff --git a/dss-scheduler-appjoint-core/src/main/java/com/webank/wedatasphere/dss/appjoint/scheduler/parser/ContextNodeParser.java b/dss-scheduler-appjoint-core/src/main/java/com/webank/wedatasphere/dss/appjoint/scheduler/parser/ContextNodeParser.java deleted file mode 100644 index e8ba23db7b..0000000000 --- a/dss-scheduler-appjoint-core/src/main/java/com/webank/wedatasphere/dss/appjoint/scheduler/parser/ContextNodeParser.java +++ /dev/null @@ -1,30 +0,0 @@ -/* - * Copyright 2019 WeBank - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package com.webank.wedatasphere.dss.appjoint.scheduler.parser; - -import com.webank.wedatasphere.dss.appjoint.scheduler.entity.SchedulerNode; -import com.webank.wedatasphere.dss.appjoint.scheduler.entity.ReadNode; - -/** - * Created by enjoyyin on 2019/9/7. - */ -public interface ContextNodeParser extends NodeParser { - String[] getShareNodeIds(SchedulerNode node); - void setReadNodeContext(SchedulerNode node); - ReadNode parseNode(SchedulerNode node); -} diff --git a/dss-scheduler-appjoint-core/src/main/java/com/webank/wedatasphere/dss/appjoint/scheduler/parser/FlowParser.java b/dss-scheduler-appjoint-core/src/main/java/com/webank/wedatasphere/dss/appjoint/scheduler/parser/FlowParser.java deleted file mode 100644 index 85369debce..0000000000 --- a/dss-scheduler-appjoint-core/src/main/java/com/webank/wedatasphere/dss/appjoint/scheduler/parser/FlowParser.java +++ /dev/null @@ -1,36 +0,0 @@ -/* - * Copyright 2019 WeBank - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package com.webank.wedatasphere.dss.appjoint.scheduler.parser; - - -import com.webank.wedatasphere.dss.appjoint.scheduler.entity.SchedulerFlow; -import com.webank.wedatasphere.dss.common.entity.flow.DSSJSONFlow; - -/** - * Created by enjoyyin on 2019/9/7. - */ -public interface FlowParser { - - SchedulerFlow parseFlow(DSSJSONFlow flow); - - void setNodeParsers(NodeParser[] nodeParsers); - - NodeParser[] getNodeParsers(); - - Boolean ifFlowCanParse(DSSJSONFlow flow); -} diff --git a/dss-scheduler-appjoint-core/src/main/java/com/webank/wedatasphere/dss/appjoint/scheduler/parser/NodeParser.java b/dss-scheduler-appjoint-core/src/main/java/com/webank/wedatasphere/dss/appjoint/scheduler/parser/NodeParser.java deleted file mode 100644 index ebba917a16..0000000000 --- a/dss-scheduler-appjoint-core/src/main/java/com/webank/wedatasphere/dss/appjoint/scheduler/parser/NodeParser.java +++ /dev/null @@ -1,31 +0,0 @@ -/* - * Copyright 2019 WeBank - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package com.webank.wedatasphere.dss.appjoint.scheduler.parser; - -import com.webank.wedatasphere.dss.appjoint.scheduler.entity.SchedulerNode; -import com.webank.wedatasphere.dss.common.entity.node.DSSNode; -import com.webank.wedatasphere.dss.appjoint.scheduler.order.Order; - -/** - * Created by enjoyyin on 2019/9/7. - */ -public interface NodeParser extends Order { - SchedulerNode parseNode(DSSNode dssNode); - - Boolean ifNodeCanParse(DSSNode dssNode); -} diff --git a/dss-scheduler-appjoint-core/src/main/java/com/webank/wedatasphere/dss/appjoint/scheduler/parser/ProjectParser.java b/dss-scheduler-appjoint-core/src/main/java/com/webank/wedatasphere/dss/appjoint/scheduler/parser/ProjectParser.java deleted file mode 100644 index db00db311a..0000000000 --- a/dss-scheduler-appjoint-core/src/main/java/com/webank/wedatasphere/dss/appjoint/scheduler/parser/ProjectParser.java +++ /dev/null @@ -1,31 +0,0 @@ -/* - * Copyright 2019 WeBank - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package com.webank.wedatasphere.dss.appjoint.scheduler.parser; - -import com.webank.wedatasphere.dss.appjoint.scheduler.entity.SchedulerProject; -import com.webank.wedatasphere.dss.common.entity.project.DSSProject; - -/** - * Created by enjoyyin on 2019/9/16. - */ -public interface ProjectParser { - SchedulerProject parseProject(DSSProject dssProject); - void setFlowParsers(FlowParser[] flowParser); - FlowParser[] getFlowParsers(); - -} diff --git a/dss-scheduler-appjoint-core/src/main/java/com/webank/wedatasphere/dss/appjoint/scheduler/parser/SendEmailNodeParser.java b/dss-scheduler-appjoint-core/src/main/java/com/webank/wedatasphere/dss/appjoint/scheduler/parser/SendEmailNodeParser.java deleted file mode 100644 index 1bd3606f0d..0000000000 --- a/dss-scheduler-appjoint-core/src/main/java/com/webank/wedatasphere/dss/appjoint/scheduler/parser/SendEmailNodeParser.java +++ /dev/null @@ -1,48 +0,0 @@ -/* - * Copyright 2019 WeBank - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package com.webank.wedatasphere.dss.appjoint.scheduler.parser; - -import com.webank.wedatasphere.dss.appjoint.scheduler.constant.SchedulerAppJointConstant; -import com.webank.wedatasphere.dss.common.entity.node.DSSNode; - -import java.util.Map; - -/** - * Created by enjoyyin on 2019/10/31. - */ -public abstract class SendEmailNodeParser extends AbstractReadNodeParser { - - @Override - public Boolean ifNodeCanParse(DSSNode dssNode) { - //判断是sendemail 并且category是node - Map params = dssNode.getParams(); - if(params != null && !params.isEmpty()){ - Object configuration = params.get(SchedulerAppJointConstant.CONFIGURATION); - if(configuration instanceof Map){ - Object runtime = ((Map) configuration).get(SchedulerAppJointConstant.RUNTIME); - if(runtime instanceof Map){ - Object category = ((Map) runtime).get(SchedulerAppJointConstant.CATEGORY); - if(category != null && SchedulerAppJointConstant.NODE.equals(category.toString())){ - return SchedulerAppJointConstant.SENDEMAIL_NODE_TYPE.equals(dssNode.getNodeType()); - } - } - } - } - return false; - } -} diff --git a/dss-scheduler-appjoint-core/src/main/java/com/webank/wedatasphere/dss/appjoint/scheduler/service/SchedulerProjectService.java b/dss-scheduler-appjoint-core/src/main/java/com/webank/wedatasphere/dss/appjoint/scheduler/service/SchedulerProjectService.java deleted file mode 100644 index 2bc4cea3eb..0000000000 --- a/dss-scheduler-appjoint-core/src/main/java/com/webank/wedatasphere/dss/appjoint/scheduler/service/SchedulerProjectService.java +++ /dev/null @@ -1,31 +0,0 @@ -/* - * Copyright 2019 WeBank - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package com.webank.wedatasphere.dss.appjoint.scheduler.service; - -import com.webank.wedatasphere.dss.appjoint.exception.AppJointErrorException; -import com.webank.wedatasphere.dss.appjoint.service.ProjectService; -import com.webank.wedatasphere.dss.appjoint.service.session.Session; -import com.webank.wedatasphere.dss.common.entity.project.Project; - -/** - * Created by enjoyyin on 2019/10/10. - */ -public interface SchedulerProjectService extends ProjectService { - - void publishProject(Project project, Session session) throws AppJointErrorException; -} diff --git a/dss-scheduler-appjoint-core/src/main/java/com/webank/wedatasphere/dss/appjoint/scheduler/service/SchedulerSecurityService.java b/dss-scheduler-appjoint-core/src/main/java/com/webank/wedatasphere/dss/appjoint/scheduler/service/SchedulerSecurityService.java deleted file mode 100644 index a9652e52c4..0000000000 --- a/dss-scheduler-appjoint-core/src/main/java/com/webank/wedatasphere/dss/appjoint/scheduler/service/SchedulerSecurityService.java +++ /dev/null @@ -1,27 +0,0 @@ -/* - * Copyright 2019 WeBank - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package com.webank.wedatasphere.dss.appjoint.scheduler.service; - -import com.webank.wedatasphere.dss.appjoint.service.SecurityService; - -/** - * Created by enjoyyin on 2019/10/12. - */ -public interface SchedulerSecurityService extends SecurityService { - void reloadToken(); -} diff --git a/dss-scheduler-appjoint-core/src/main/java/com/webank/wedatasphere/dss/appjoint/scheduler/tuning/AbstractFlowTuning.java b/dss-scheduler-appjoint-core/src/main/java/com/webank/wedatasphere/dss/appjoint/scheduler/tuning/AbstractFlowTuning.java deleted file mode 100644 index 42423a69d6..0000000000 --- a/dss-scheduler-appjoint-core/src/main/java/com/webank/wedatasphere/dss/appjoint/scheduler/tuning/AbstractFlowTuning.java +++ /dev/null @@ -1,120 +0,0 @@ -/* - * Copyright 2019 WeBank - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package com.webank.wedatasphere.dss.appjoint.scheduler.tuning; - -import com.webank.wedatasphere.dss.appjoint.scheduler.entity.SchedulerEdge; -import com.webank.wedatasphere.dss.appjoint.scheduler.entity.SchedulerNode; -import com.webank.wedatasphere.dss.appjoint.scheduler.entity.SchedulerFlow; -import com.webank.wedatasphere.dss.appjoint.scheduler.constant.SchedulerAppJointConstant; -import org.apache.commons.lang.StringUtils; - -import java.util.ArrayList; -import java.util.List; -import java.util.stream.Collectors; - -/** - * Created by enjoyyin on 2019/9/25. - */ -public abstract class AbstractFlowTuning implements FlowTuning { - - private NodeTuning[] nodeTunings; - - @Override - public void setNodeTunings(NodeTuning[] nodeTunings) { - this.nodeTunings = nodeTunings; - } - - @Override - public NodeTuning[] getNodeTunings() { - return this.nodeTunings; - } - - @Override - public SchedulerFlow tuningSchedulerFlow(SchedulerFlow schedulerFlow) { - //1.设置依赖，对于多个flowTuning，已经设置的就跳过 - schedulerFlow.getSchedulerNodes().forEach(node ->setDependencies(node,schedulerFlow.getSchedulerNodes(),schedulerFlow.getSchedulerEdges())); - //2.设置usreproxy - setProxyUser(schedulerFlow); - // 2.调用各种nodeTuning进行 - List schedulerNodes = schedulerFlow.getSchedulerNodes(); - for (NodeTuning nodeTuning : getNodeTunings()) { - schedulerNodes = targetNodeTuning(nodeTuning, schedulerNodes); - } - schedulerFlow.setSchedulerNodes(schedulerNodes); - //3.对子flow递归调用本方法 - List children = schedulerFlow.getChildren(); - if (children != null) { - List collect = children.stream().map(this::tuningSchedulerFlow).collect(Collectors.toList()); - schedulerFlow.setChildren(collect); - } - return schedulerFlow; - } - - private void setDependencies(SchedulerNode node,List schedulerNodes,List flowEdges){ - //设置过的flow不在进行设置和解析 - if(node.getDependencys()!= null && !node.getDependencys().isEmpty()) return; - List dependencies = resolveDependencys(node,schedulerNodes ,flowEdges); - dependencies.forEach(node::addDependency); - } - - private List resolveDependencys(SchedulerNode node,List schedulerNodes, List flowEdges) { - List dependencys = new ArrayList<>(); - flowEdges.forEach(edge -> { - if (edge.getDssEdge().getTarget().equals(node.getId())) { - dependencys.add(schedulerNodes.stream().filter(n ->edge.getDssEdge().getSource().equals(n.getId())).findFirst().get().getName()); - } - }); - - return dependencys; - } - - private List targetNodeTuning(NodeTuning nodeTuning, List schedulerNodes) { - return schedulerNodes.stream().map(node -> invokeNodeTuning(nodeTuning, node)).collect(Collectors.toList()); - } - - private SchedulerNode invokeNodeTuning(NodeTuning nodeTuning, SchedulerNode schedulerNode) { - if (nodeTuning.ifNodeCanTuning(schedulerNode)) { - return nodeTuning.tuningSchedulerNode(schedulerNode); - } else { - return schedulerNode; - } - } - - private String getProxyUser(SchedulerFlow schedulerFlow) { - if(schedulerFlow.getFlowProperties() == null) { - return null; - } - StringBuilder sb = new StringBuilder(); - schedulerFlow.getFlowProperties().forEach( map -> { - Object value = map.get(SchedulerAppJointConstant.PROXY_USER); - if(value != null && StringUtils.isNotBlank(value.toString())) { - sb.append(value.toString()); - } - }); - return sb.toString(); - } - - private void setProxyUser(SchedulerFlow schedulerFlow) { - String proxyUser = getProxyUser(schedulerFlow); - if(StringUtils.isNotBlank(proxyUser)) { - schedulerFlow.getSchedulerNodes().forEach(node -> node.getDssNode().setUserProxy(proxyUser)); - schedulerFlow.setUserProxy(proxyUser); - } - } - -} diff --git a/dss-scheduler-appjoint-core/src/main/java/com/webank/wedatasphere/dss/appjoint/scheduler/tuning/AbstractNodeTuning.java b/dss-scheduler-appjoint-core/src/main/java/com/webank/wedatasphere/dss/appjoint/scheduler/tuning/AbstractNodeTuning.java deleted file mode 100644 index 9d15de9bd5..0000000000 --- a/dss-scheduler-appjoint-core/src/main/java/com/webank/wedatasphere/dss/appjoint/scheduler/tuning/AbstractNodeTuning.java +++ /dev/null @@ -1,30 +0,0 @@ -/* - * Copyright 2019 WeBank - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package com.webank.wedatasphere.dss.appjoint.scheduler.tuning; - -import com.webank.wedatasphere.dss.appjoint.scheduler.entity.SchedulerNode; - -/** - * Created by enjoyyin on 2019/9/25. - */ -public abstract class AbstractNodeTuning implements NodeTuning { - @Override - public SchedulerNode tuningSchedulerNode(SchedulerNode node) { - return node; - } -} diff --git a/dss-scheduler-appjoint-core/src/main/java/com/webank/wedatasphere/dss/appjoint/scheduler/tuning/AbstractProjectTuning.java b/dss-scheduler-appjoint-core/src/main/java/com/webank/wedatasphere/dss/appjoint/scheduler/tuning/AbstractProjectTuning.java deleted file mode 100644 index 913ad636f8..0000000000 --- a/dss-scheduler-appjoint-core/src/main/java/com/webank/wedatasphere/dss/appjoint/scheduler/tuning/AbstractProjectTuning.java +++ /dev/null @@ -1,70 +0,0 @@ -/* - * Copyright 2019 WeBank - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package com.webank.wedatasphere.dss.appjoint.scheduler.tuning; - -import com.webank.wedatasphere.dss.appjoint.scheduler.entity.AbstractSchedulerProject; -import com.webank.wedatasphere.dss.appjoint.scheduler.entity.SchedulerFlow; -import com.webank.wedatasphere.dss.appjoint.scheduler.entity.SchedulerProject; - -import java.util.Arrays; -import java.util.List; -import java.util.stream.Collectors; - -/** - * Created by enjoyyin on 2019/9/25. - */ -public abstract class AbstractProjectTuning implements ProjectTuning { - - private FlowTuning [] flowTunings; - - @Override - public void setFlowTunings(FlowTuning[] flowTunings) { - this.flowTunings = flowTunings; - } - - @Override - public FlowTuning[] getFlowTunings() { - return this.flowTunings; - } - - @Override - public SchedulerProject tuningSchedulerProject(SchedulerProject schedulerProject) { - // TODO: 2019/9/25 这里强转是因为没有把 schedulerProject由接口修改为普通类 - // TODO: 2019/9/25 这里如果flow - AbstractSchedulerProject abstractSchedulerProject = (AbstractSchedulerProject) schedulerProject; - List schedulerFlows = abstractSchedulerProject.getSchedulerFlows(); - List flowTunings = Arrays.stream(getFlowTunings()).sorted((t1, t2) -> t2.getOrder() - t1.getOrder()).collect(Collectors.toList()); - for (FlowTuning flowTuning : flowTunings) { - schedulerFlows = targetFlowTuning(flowTuning,schedulerFlows); - } - abstractSchedulerProject.setSchedulerFlows(schedulerFlows); - return abstractSchedulerProject; - } - - private List targetFlowTuning(FlowTuning flowTuning,List schedulerFlows){ - return schedulerFlows.stream().map(flow ->invokeFlowTuning(flowTuning,flow)).collect(Collectors.toList()); - } - - private SchedulerFlow invokeFlowTuning(FlowTuning flowTuning,SchedulerFlow schedulerFlow){ - if (flowTuning.ifFlowCanTuning(schedulerFlow)){ - return flowTuning.tuningSchedulerFlow(schedulerFlow); - }else { - return schedulerFlow; - } - } -} diff --git a/dss-scheduler-appjoint-core/src/main/java/com/webank/wedatasphere/dss/appjoint/scheduler/tuning/AbstractShareNodeFlowTuning.java b/dss-scheduler-appjoint-core/src/main/java/com/webank/wedatasphere/dss/appjoint/scheduler/tuning/AbstractShareNodeFlowTuning.java deleted file mode 100644 index 81dc0d5d31..0000000000 --- a/dss-scheduler-appjoint-core/src/main/java/com/webank/wedatasphere/dss/appjoint/scheduler/tuning/AbstractShareNodeFlowTuning.java +++ /dev/null @@ -1,96 +0,0 @@ -/* - * Copyright 2019 WeBank - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package com.webank.wedatasphere.dss.appjoint.scheduler.tuning; - -import com.webank.wedatasphere.dss.appjoint.scheduler.entity.SchedulerNode; -import com.webank.wedatasphere.dss.appjoint.scheduler.entity.ShareNode; -import com.webank.wedatasphere.dss.appjoint.scheduler.entity.SchedulerFlow; -import com.webank.wedatasphere.dss.appjoint.scheduler.entity.ReadNode; - -import java.util.*; -import java.util.function.Function; -import java.util.stream.Collectors; -import java.util.stream.Stream; - -/** - * Created by enjoyyin on 2019/10/31. - */ -public abstract class AbstractShareNodeFlowTuning extends AbstractFlowTuning implements ContextShareableFlowTuning { - - @Override - public SchedulerFlow tuningSchedulerFlow(SchedulerFlow schedulerFlow) { - List schedulerNodes = schedulerFlow.getSchedulerNodes(); - List tuningSchedulerNode = new ArrayList<>(); - List readNodeList = getReadNodes(schedulerFlow); - ReadNode[] schedulerNodes1 = new ReadNode[readNodeList.size()]; - Map shareNodeMap = getShareNodes(schedulerFlow,readNodeList.toArray(schedulerNodes1)); - schedulerNodes.stream().forEach(x ->{ - Map.Entry findShareNode = shareNodeMap.entrySet().stream().filter(y -> - (y.getKey().getName()).equals(x.getName()) - ).findFirst().orElse(null); - if(findShareNode != null ){ - tuningSchedulerNode.add(findShareNode.getKey()); - - }else { - tuningSchedulerNode.add(x); - } - }); - schedulerFlow.setSchedulerNodes(tuningSchedulerNode); - return super.tuningSchedulerFlow(schedulerFlow); - } - - @Override - public List getReadNodes(SchedulerFlow flow) { - List res = flow.getSchedulerNodes().stream().filter(x -> x instanceof ReadNode).collect(Collectors.toList()); - return res; - } - - @Override - public Map getShareNodes(SchedulerFlow flow, ReadNode[] readNodes) { - List list = new ArrayList<>(); - Map res = new HashMap<>(); - //遍历readNodes，将NodeIds转为name的集合,过滤掉删除了节点但是还滞留在content里面的id - Arrays.stream(readNodes).filter(rn ->rn.getShareNodeIds() != null).forEach(rn ->{ - rn.setShareNodeIds(Arrays.stream(rn.getShareNodeIds()).filter(id -> flow.getSchedulerNodes().stream().anyMatch(sn -> id.equals(sn.getId()))). - map(id -> flow.getSchedulerNodes().stream().filter(sn -> id.equals(sn.getId())).findFirst().get().getName()).toArray(String[]::new)); - }); - Stream.of(readNodes).forEach(x -> - { - if(x.getShareNodeIds() != null) { - list.addAll(Arrays.asList(x.getShareNodeIds())); - } - } - ); - Map nameAndNumMap = list.stream().collect(Collectors.groupingBy(Function.identity(),Collectors.counting())); - nameAndNumMap.keySet().forEach(key ->{ - - SchedulerNode schedulerNode = flow.getSchedulerNodes().stream().filter(x->x.getName().equals(key.toString())).findFirst().orElse(null); - if(schedulerNode != null) { - int shareTimes = (nameAndNumMap.get(key)).intValue(); - ShareNode shareNode = createShareNode(); - shareNode.setDssNode(schedulerNode.getDssNode()); - shareNode.setSchedulerNode(schedulerNode); - shareNode.setShareTimes(shareTimes); - res.put(shareNode, shareTimes); - } - }); - return res; - } - - protected abstract ShareNode createShareNode(); -} diff --git a/dss-scheduler-appjoint-core/src/main/java/com/webank/wedatasphere/dss/appjoint/scheduler/tuning/ContextNodeTuning.java b/dss-scheduler-appjoint-core/src/main/java/com/webank/wedatasphere/dss/appjoint/scheduler/tuning/ContextNodeTuning.java deleted file mode 100644 index c4cea46b4c..0000000000 --- a/dss-scheduler-appjoint-core/src/main/java/com/webank/wedatasphere/dss/appjoint/scheduler/tuning/ContextNodeTuning.java +++ /dev/null @@ -1,28 +0,0 @@ -/* - * Copyright 2019 WeBank - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package com.webank.wedatasphere.dss.appjoint.scheduler.tuning; - -import com.webank.wedatasphere.dss.appjoint.scheduler.entity.ShareNode; - -/** - * Created by enjoyyin on 2019/9/7. - */ -public interface ContextNodeTuning { - void setShareNodeContext(ShareNode node); - ShareNode tuningSchedulerNode(ShareNode node); -} diff --git a/dss-scheduler-appjoint-core/src/main/java/com/webank/wedatasphere/dss/appjoint/scheduler/tuning/ContextShareableFlowTuning.java b/dss-scheduler-appjoint-core/src/main/java/com/webank/wedatasphere/dss/appjoint/scheduler/tuning/ContextShareableFlowTuning.java deleted file mode 100644 index 0c4706f884..0000000000 --- a/dss-scheduler-appjoint-core/src/main/java/com/webank/wedatasphere/dss/appjoint/scheduler/tuning/ContextShareableFlowTuning.java +++ /dev/null @@ -1,34 +0,0 @@ -/* - * Copyright 2019 WeBank - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package com.webank.wedatasphere.dss.appjoint.scheduler.tuning; - -import com.webank.wedatasphere.dss.appjoint.scheduler.entity.SchedulerNode; -import com.webank.wedatasphere.dss.appjoint.scheduler.entity.ShareNode; -import com.webank.wedatasphere.dss.appjoint.scheduler.entity.SchedulerFlow; -import com.webank.wedatasphere.dss.appjoint.scheduler.entity.ReadNode; - -import java.util.List; -import java.util.Map; - -/** - * Created by enjoyyin on 2019/9/7. - */ -public interface ContextShareableFlowTuning { - List getReadNodes(SchedulerFlow flow); - Map getShareNodes(SchedulerFlow flow, ReadNode[] readNodes); -} diff --git a/dss-scheduler-appjoint-core/src/main/java/com/webank/wedatasphere/dss/appjoint/scheduler/tuning/FlowTuning.java b/dss-scheduler-appjoint-core/src/main/java/com/webank/wedatasphere/dss/appjoint/scheduler/tuning/FlowTuning.java deleted file mode 100644 index 2bc5233307..0000000000 --- a/dss-scheduler-appjoint-core/src/main/java/com/webank/wedatasphere/dss/appjoint/scheduler/tuning/FlowTuning.java +++ /dev/null @@ -1,31 +0,0 @@ -/* - * Copyright 2019 WeBank - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package com.webank.wedatasphere.dss.appjoint.scheduler.tuning; - -import com.webank.wedatasphere.dss.appjoint.scheduler.entity.SchedulerFlow; -import com.webank.wedatasphere.dss.appjoint.scheduler.order.Order; - -/** - * Created by enjoyyin on 2019/9/7. - */ -public interface FlowTuning extends Order { - SchedulerFlow tuningSchedulerFlow(SchedulerFlow schedulerFlow); - void setNodeTunings(NodeTuning[] NodeTunings); - NodeTuning[] getNodeTunings(); - Boolean ifFlowCanTuning(SchedulerFlow schedulerFlow); -} diff --git a/dss-scheduler-appjoint-core/src/main/java/com/webank/wedatasphere/dss/appjoint/scheduler/tuning/NodeTuning.java b/dss-scheduler-appjoint-core/src/main/java/com/webank/wedatasphere/dss/appjoint/scheduler/tuning/NodeTuning.java deleted file mode 100644 index dedd846d3c..0000000000 --- a/dss-scheduler-appjoint-core/src/main/java/com/webank/wedatasphere/dss/appjoint/scheduler/tuning/NodeTuning.java +++ /dev/null @@ -1,28 +0,0 @@ -/* - * Copyright 2019 WeBank - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package com.webank.wedatasphere.dss.appjoint.scheduler.tuning; - -import com.webank.wedatasphere.dss.appjoint.scheduler.entity.SchedulerNode; - -/** - * Created by enjoyyin on 2019/9/7. - */ -public interface NodeTuning { - SchedulerNode tuningSchedulerNode(SchedulerNode node); - Boolean ifNodeCanTuning(SchedulerNode node); -} diff --git a/dss-scheduler-appjoint-core/src/main/java/com/webank/wedatasphere/dss/appjoint/scheduler/tuning/ProjectTuning.java b/dss-scheduler-appjoint-core/src/main/java/com/webank/wedatasphere/dss/appjoint/scheduler/tuning/ProjectTuning.java deleted file mode 100644 index b2b40516bf..0000000000 --- a/dss-scheduler-appjoint-core/src/main/java/com/webank/wedatasphere/dss/appjoint/scheduler/tuning/ProjectTuning.java +++ /dev/null @@ -1,29 +0,0 @@ -/* - * Copyright 2019 WeBank - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package com.webank.wedatasphere.dss.appjoint.scheduler.tuning; - -import com.webank.wedatasphere.dss.appjoint.scheduler.entity.SchedulerProject; - -/** - * Created by enjoyyin on 2019/9/18 - */ -public interface ProjectTuning { - SchedulerProject tuningSchedulerProject(SchedulerProject schedulerProject); - void setFlowTunings(FlowTuning[] flowTunings); - FlowTuning[] getFlowTunings(); -} diff --git a/dss-server/bin/start-dss-server.sh b/dss-server/bin/start-dss-server.sh deleted file mode 100644 index 518cd8da17..0000000000 --- a/dss-server/bin/start-dss-server.sh +++ /dev/null @@ -1,39 +0,0 @@ -#!/bin/bash -cd `dirname $0` -cd .. -HOME=`pwd` - -export SERVER_PID=$HOME/bin/linkis.pid -export SERVER_LOG_PATH=$HOME/logs -export SERVER_CLASS=com.webank.wedatasphere.dss.DSSSpringApplication - -if test -z "$SERVER_HEAP_SIZE" -then - export SERVER_HEAP_SIZE="512M" -fi - -if test -z "$SERVER_JAVA_OPTS" -then - export SERVER_JAVA_OPTS=" -Xmx$SERVER_HEAP_SIZE -XX:+UseG1GC -Xloggc:$HOME/logs/linkis-gc.log" -fi - -if [[ -f "${SERVER_PID}" ]]; then - pid=$(cat ${SERVER_PID}) - if kill -0 ${pid} >/dev/null 2>&1; then - echo "Server is already running." - exit 1 - fi -fi - -nohup java $SERVER_JAVA_OPTS -cp $HOME/conf:$HOME/lib/* $SERVER_CLASS 2>&1 > $SERVER_LOG_PATH/linkis.out & -pid=$! -if [[ -z "${pid}" ]]; then - echo "server $SERVER_NAME start failed!" - exit 1 -else - echo "server $SERVER_NAME start succeeded!" - echo $pid > $SERVER_PID - sleep 1 -fi - - diff --git a/dss-server/bin/stop-dss-server.sh b/dss-server/bin/stop-dss-server.sh deleted file mode 100644 index f8b94c2769..0000000000 --- a/dss-server/bin/stop-dss-server.sh +++ /dev/null @@ -1,47 +0,0 @@ -#!/bin/bash - -cd `dirname $0` -cd .. -HOME=`pwd` - -export FLOW_ENTRANCE_HOME_PID=$HOME/bin/linkis.pid - -function wait_for_FLOW_ENTRANCE_HOME_to_die() { - local pid - local count - pid=$1 - timeout=$2 - count=0 - timeoutTime=$(date "+%s") - let "timeoutTime+=$timeout" - currentTime=$(date "+%s") - forceKill=1 - - while [[ $currentTime -lt $timeoutTime ]]; do - $(kill ${pid} > /dev/null 2> /dev/null) - if kill -0 ${pid} > /dev/null 2>&1; then - sleep 3 - else - forceKill=0 - break - fi - currentTime=$(date "+%s") - done - - if [[ forceKill -ne 0 ]]; then - $(kill -9 ${pid} > /dev/null 2> /dev/null) - fi -} - -if [[ ! -f "${FLOW_ENTRANCE_HOME_PID}" ]]; then - echo "DSS SERVER is not running" -else - pid=$(cat ${FLOW_ENTRANCE_HOME_PID}) - if [[ -z "${pid}" ]]; then - echo "DSS SERVER is not running" - else - wait_for_FLOW_ENTRANCE_HOME_to_die $pid 40 - $(rm -f ${FLOW_ENTRANCE_HOME_PID}) - echo "DSS SERVER is stopped." - fi -fi diff --git a/dss-server/pom.xml b/dss-server/pom.xml deleted file mode 100644 index 76d1446ddb..0000000000 --- a/dss-server/pom.xml +++ /dev/null @@ -1,237 +0,0 @@ - - - - - - dss - com.webank.wedatasphere.dss - 0.9.1 - - 4.0.0 - - dss-server - - - - - org.mybatis.spring.boot - mybatis-spring-boot-starter - 1.3.2 - - - spring-boot-starter - org.springframework.boot - - - spring-boot-autoconfigure - org.springframework.boot - - - spring-beans - org.springframework - - - spring-core - org.springframework - - - - - - com.webank.wedatasphere.dss - dss-appjoint-loader - ${dss.version} - - - - - - - - - - - - - - - - - - - - org.springframework.cloud - spring-cloud-starter-netflix-eureka-client - 2.0.0.RELEASE - - - logback-classic - ch.qos.logback - - - log4j-to-slf4j - org.apache.logging.log4j - - - gson - com.google.code.gson - - - jsr311-api - javax.ws.rs - - - - - - org.springframework.cloud - spring-cloud-starter-openfeign - 2.0.0.RELEASE - - - - com.webank.wedatasphere.dss - dss-application - ${dss.version} - - - - linkis-bmlclient - - - gson - com.google.code.gson - - - com.webank.wedatasphere.linkis - ${linkis.version} - - - - com.webank.wedatasphere.linkis - linkis-cloudRPC - ${linkis.version} - - - spring-cloud-starter-netflix-eureka-client - org.springframework.cloud - - - spring-cloud-starter-openfeign - org.springframework.cloud - - - - - - junit - junit - 4.12 - test - - - com.webank.wedatasphere.dss - dss-scheduler-appjoint-core - 0.9.1 - - - com.webank.wedatasphere.dss - dss-user-manager - 0.9.1 - compile - - - com.github.rholder - guava-retrying - 2.0.0 - - - dom4j - dom4j - 1.6.1 - - - com.typesafe - config - 1.4.1 - - - xml-apis - xml-apis - 1.4.01 - - - - - - - org.apache.maven.plugins - maven-deploy-plugin - - - - net.alchim31.maven - scala-maven-plugin - - - org.apache.maven.plugins - maven-jar-plugin - - - org.apache.maven.plugins - maven-assembly-plugin - 2.3 - false - - - make-assembly - package - - single - - - - src/main/assembly/distribution.xml - - - - - - false - dss-server - false - false - - src/main/assembly/distribution.xml - - - - - - - src/main/java - - **/*.xml - - - - - - \ No newline at end of file diff --git a/dss-server/src/main/assembly/distribution.xml b/dss-server/src/main/assembly/distribution.xml deleted file mode 100644 index 0da5c182ca..0000000000 --- a/dss-server/src/main/assembly/distribution.xml +++ /dev/null @@ -1,340 +0,0 @@ - - - - dss-server - - zip - - true - dss-server - - - - - - lib - true - true - false - true - true - - antlr:antlr:jar - aopalliance:aopalliance:jar - asm:asm:jar - cglib:cglib:jar - - - - - - - com.fasterxml.jackson.core:jackson-annotations:jar - com.fasterxml.jackson.core:jackson-core:jar - com.fasterxml.jackson.core:jackson-databind:jar - - com.fasterxml.jackson.datatype:jackson-datatype-jdk8:jar - com.fasterxml.jackson.datatype:jackson-datatype-jsr310:jar - com.fasterxml.jackson.jaxrs:jackson-jaxrs-base:jar - com.fasterxml.jackson.jaxrs:jackson-jaxrs-json-provider:jar - com.fasterxml.jackson.module:jackson-module-jaxb-annotations:jar - com.fasterxml.jackson.module:jackson-module-parameter-names:jar - com.fasterxml.jackson.module:jackson-module-paranamer:jar - com.fasterxml.jackson.module:jackson-module-scala_2.11:jar - - - com.google.code.findbugs:annotations:jar - - com.google.code.gson:gson:jar - com.google.guava:guava:jar - com.google.inject:guice:jar - com.google.protobuf:protobuf-java:jar - com.netflix.archaius:archaius-core:jar - com.netflix.eureka:eureka-client:jar - com.netflix.eureka:eureka-core:jar - com.netflix.hystrix:hystrix-core:jar - com.netflix.netflix-commons:netflix-commons-util:jar - com.netflix.netflix-commons:netflix-eventbus:jar - com.netflix.netflix-commons:netflix-infix:jar - com.netflix.netflix-commons:netflix-statistics:jar - com.netflix.ribbon:ribbon:jar - com.netflix.ribbon:ribbon-core:jar - com.netflix.ribbon:ribbon-eureka:jar - com.netflix.ribbon:ribbon-httpclient:jar - com.netflix.ribbon:ribbon-loadbalancer:jar - com.netflix.ribbon:ribbon-transport:jar - com.netflix.servo:servo-core:jar - com.sun.jersey.contribs:jersey-apache-client4:jar - com.sun.jersey:jersey-client:jar - com.sun.jersey:jersey-core:jar - com.sun.jersey:jersey-json:jar - com.sun.jersey:jersey-server:jar - com.sun.jersey:jersey-servlet:jar - com.sun.xml.bind:jaxb-impl:jar - com.thoughtworks.paranamer:paranamer:jar - com.thoughtworks.xstream:xstream:jar - com.webank.wedatasphere.linkis:linkis-common:jar - com.webank.wedatasphere.linkis:linkis-module:jar - commons-beanutils:commons-beanutils:jar - - commons-cli:commons-cli:jar - commons-codec:commons-codec:jar - commons-collections:commons-collections:jar - commons-configuration:commons-configuration:jar - commons-daemon:commons-daemon:jar - commons-dbcp:commons-dbcp:jar - - commons-httpclient:commons-httpclient:jar - commons-io:commons-io:jar - commons-jxpath:commons-jxpath:jar - commons-lang:commons-lang:jar - commons-logging:commons-logging:jar - commons-net:commons-net:jar - commons-pool:commons-pool:jar - io.micrometer:micrometer-core:jar - io.netty:netty:jar - io.netty:netty-all:jar - io.netty:netty-buffer:jar - io.netty:netty-codec:jar - io.netty:netty-codec-http:jar - io.netty:netty-common:jar - io.netty:netty-handler:jar - io.netty:netty-transport:jar - io.netty:netty-transport-native-epoll:jar - io.reactivex:rxjava:jar - io.reactivex:rxnetty:jar - io.reactivex:rxnetty-contexts:jar - io.reactivex:rxnetty-servo:jar - javax.activation:activation:jar - javax.annotation:javax.annotation-api:jar - javax.inject:javax.inject:jar - javax.servlet:javax.servlet-api:jar - javax.servlet.jsp:jsp-api:jar - javax.validation:validation-api:jar - javax.websocket:javax.websocket-api:jar - javax.ws.rs:javax.ws.rs-api:jar - javax.xml.bind:jaxb-api:jar - javax.xml.stream:stax-api:jar - joda-time:joda-time:jar - log4j:log4j:jar - mysql:mysql-connector-java:jar - org.antlr:antlr-runtime:jar - org.antlr:stringtemplate:jar - org.apache.commons:commons-compress:jar - org.apache.commons:commons-math:jar - org.apache.commons:commons-math3:jar - org.apache.curator:curator-client:jar - org.apache.curator:curator-framework:jar - org.apache.curator:curator-recipes:jar - org.apache.directory.api:api-asn1-api:jar - org.apache.directory.api:api-util:jar - org.apache.directory.server:apacheds-i18n:jar - org.apache.directory.server:apacheds-kerberos-codec:jar - org.apache.hadoop:hadoop-annotations:jar - org.apache.hadoop:hadoop-auth:jar - org.apache.hadoop:hadoop-common:jar - org.apache.hadoop:hadoop-hdfs:jar - org.apache.htrace:htrace-core:jar - org.apache.httpcomponents:httpclient:jar - org.apache.httpcomponents:httpcore:jar - org.apache.logging.log4j:log4j-api:jar - org.apache.logging.log4j:log4j-core:jar - org.apache.logging.log4j:log4j-jul:jar - org.apache.logging.log4j:log4j-slf4j-impl:jar - org.apache.zookeeper:zookeeper:jar - org.aspectj:aspectjweaver:jar - org.bouncycastle:bcpkix-jdk15on:jar - org.bouncycastle:bcprov-jdk15on:jar - org.codehaus.jackson:jackson-jaxrs:jar - org.codehaus.jackson:jackson-xc:jar - org.codehaus.jettison:jettison:jar - org.codehaus.woodstox:stax2-api:jar - org.codehaus.woodstox:woodstox-core-asl:jar - org.eclipse.jetty:jetty-annotations:jar - org.eclipse.jetty:jetty-client:jar - org.eclipse.jetty:jetty-continuation:jar - org.eclipse.jetty:jetty-http:jar - org.eclipse.jetty:jetty-io:jar - - org.eclipse.jetty:jetty-plus:jar - org.eclipse.jetty:jetty-security:jar - org.eclipse.jetty:jetty-server:jar - org.eclipse.jetty:jetty-servlet:jar - org.eclipse.jetty:jetty-servlets:jar - org.eclipse.jetty:jetty-util:jar - org.eclipse.jetty:jetty-webapp:jar - org.eclipse.jetty:jetty-xml:jar - org.eclipse.jetty.websocket:javax-websocket-client-impl:jar - org.eclipse.jetty.websocket:javax-websocket-server-impl:jar - org.eclipse.jetty.websocket:websocket-api:jar - org.eclipse.jetty.websocket:websocket-client:jar - org.eclipse.jetty.websocket:websocket-common:jar - org.eclipse.jetty.websocket:websocket-server:jar - org.eclipse.jetty.websocket:websocket-servlet:jar - org.fusesource.leveldbjni:leveldbjni-all:jar - org.glassfish.hk2:class-model:jar - org.glassfish.hk2:config-types:jar - org.glassfish.hk2.external:aopalliance-repackaged:jar - org.glassfish.hk2.external:asm-all-repackaged:jar - org.glassfish.hk2.external:bean-validator:jar - org.glassfish.hk2.external:javax.inject:jar - org.glassfish.hk2:hk2:jar - org.glassfish.hk2:hk2-api:jar - org.glassfish.hk2:hk2-config:jar - org.glassfish.hk2:hk2-core:jar - org.glassfish.hk2:hk2-locator:jar - org.glassfish.hk2:hk2-runlevel:jar - org.glassfish.hk2:hk2-utils:jar - org.glassfish.hk2:osgi-resource-locator:jar - org.glassfish.hk2:spring-bridge:jar - org.glassfish.jersey.bundles:jaxrs-ri:jar - org.glassfish.jersey.bundles.repackaged:jersey-guava:jar - org.glassfish.jersey.containers:jersey-container-servlet:jar - org.glassfish.jersey.containers:jersey-container-servlet-core:jar - org.glassfish.jersey.core:jersey-client:jar - org.glassfish.jersey.core:jersey-common:jar - org.glassfish.jersey.core:jersey-server:jar - org.glassfish.jersey.ext:jersey-entity-filtering:jar - org.glassfish.jersey.ext:jersey-spring3:jar - org.glassfish.jersey.media:jersey-media-jaxb:jar - org.glassfish.jersey.media:jersey-media-json-jackson:jar - org.glassfish.jersey.media:jersey-media-multipart:jar - org.hdrhistogram:HdrHistogram:jar - org.javassist:javassist:jar - org.json4s:json4s-ast_2.11:jar - org.json4s:json4s-core_2.11:jar - org.json4s:json4s-jackson_2.11:jar - - org.jvnet.mimepull:mimepull:jar - org.jvnet:tiger-types:jar - org.latencyutils:LatencyUtils:jar - org.mortbay.jasper:apache-el:jar - org.mortbay.jetty:jetty:jar - org.mortbay.jetty:jetty-util:jar - org.ow2.asm:asm-analysis:jar - org.ow2.asm:asm-commons:jar - org.ow2.asm:asm-tree:jar - org.reflections:reflections:jar - org.scala-lang.modules:scala-parser-combinators_2.11:jar - org.scala-lang.modules:scala-xml_2.11:jar - org.scala-lang:scala-compiler:jar - org.scala-lang:scala-library:jar - org.scala-lang:scala-reflect:jar - org.scala-lang:scalap:jar - org.slf4j:jul-to-slf4j:jar - org.slf4j:slf4j-api:jar - org.springframework.boot:spring-boot:jar - org.springframework.boot:spring-boot-actuator:jar - org.springframework.boot:spring-boot-actuator-autoconfigure:jar - org.springframework.boot:spring-boot-autoconfigure:jar - org.springframework.boot:spring-boot-starter:jar - org.springframework.boot:spring-boot-starter-actuator:jar - org.springframework.boot:spring-boot-starter-aop:jar - org.springframework.boot:spring-boot-starter-jetty:jar - org.springframework.boot:spring-boot-starter-json:jar - org.springframework.boot:spring-boot-starter-log4j2:jar - org.springframework.boot:spring-boot-starter-web:jar - org.springframework.cloud:spring-cloud-commons:jar - org.springframework.cloud:spring-cloud-config-client:jar - org.springframework.cloud:spring-cloud-context:jar - - org.springframework.cloud:spring-cloud-netflix-core:jar - org.springframework.cloud:spring-cloud-netflix-eureka-client:jar - - org.springframework.cloud:spring-cloud-starter:jar - org.springframework.cloud:spring-cloud-starter-config:jar - org.springframework.cloud:spring-cloud-starter-eureka:jar - org.springframework.cloud:spring-cloud-starter-netflix-archaius:jar - org.springframework.cloud:spring-cloud-starter-netflix-eureka-client:jar - org.springframework.cloud:spring-cloud-starter-netflix-ribbon:jar - org.springframework.security:spring-security-crypto:jar - org.springframework.security:spring-security-rsa:jar - org.springframework:spring-aop:jar - org.springframework:spring-beans:jar - org.springframework:spring-context:jar - org.springframework:spring-core:jar - org.springframework:spring-expression:jar - org.springframework:spring-jcl:jar - org.springframework:spring-web:jar - org.springframework:spring-webmvc:jar - org.tukaani:xz:jar - org.yaml:snakeyaml:jar - - xerces:xercesImpl:jar - xmlenc:xmlenc:jar - xmlpull:xmlpull:jar - xpp3:xpp3_min:jar - - - - - - - ${basedir}/src/main/resources - - * - - 0777 - conf - unix - - - ${project.parent.basedir}/dss-user-manager/src/main/resources/config - - * - - 0777 - conf/config - unix - - - ${project.parent.basedir}/dss-user-manager/src/main/resources/default - - * - - 0777 - conf/default - unix - - - ${basedir}/bin - - * - - 0777 - bin - unix - - - . - - */** - - logs - - - - - diff --git a/dss-server/src/main/java/com/webank/wedatasphere/dss/DSSSpringApplication.java b/dss-server/src/main/java/com/webank/wedatasphere/dss/DSSSpringApplication.java deleted file mode 100644 index 8a7344f9b2..0000000000 --- a/dss-server/src/main/java/com/webank/wedatasphere/dss/DSSSpringApplication.java +++ /dev/null @@ -1,222 +0,0 @@ -/* - * Copyright 2019 WeBank - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package com.webank.wedatasphere.dss; - -import com.webank.wedatasphere.linkis.DataWorkCloudApplication; -import com.webank.wedatasphere.linkis.common.ServiceInstance; -import com.webank.wedatasphere.linkis.common.conf.BDPConfiguration; -import com.webank.wedatasphere.linkis.common.conf.Configuration; -import com.webank.wedatasphere.linkis.common.exception.DWCException; -import com.webank.wedatasphere.linkis.common.utils.Utils; -import com.webank.wedatasphere.linkis.server.BDPJettyServerHelper; -import com.webank.wedatasphere.linkis.server.conf.DataWorkCloudCustomExcludeFilter; -import com.webank.wedatasphere.linkis.server.conf.ServerConfiguration; -import org.apache.commons.lang.StringUtils; -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.eclipse.jetty.server.Handler; -import org.eclipse.jetty.server.Server; -import org.eclipse.jetty.servlet.FilterHolder; -import org.eclipse.jetty.webapp.WebAppContext; -import org.springframework.boot.SpringApplication; -import org.springframework.boot.autoconfigure.SpringBootApplication; -import org.springframework.boot.builder.SpringApplicationBuilder; -import org.springframework.boot.context.event.ApplicationPreparedEvent; -import org.springframework.boot.web.embedded.jetty.JettyServerCustomizer; -import org.springframework.boot.web.embedded.jetty.JettyServletWebServerFactory; -import org.springframework.boot.web.server.WebServerFactoryCustomizer; -import org.springframework.boot.web.servlet.support.SpringBootServletInitializer; -import org.springframework.cloud.client.discovery.EnableDiscoveryClient; -import org.springframework.cloud.context.config.annotation.RefreshScope; -import org.springframework.cloud.context.scope.refresh.RefreshScopeRefreshedEvent; -import org.springframework.context.ApplicationContext; -import org.springframework.context.ApplicationListener; -import org.springframework.context.ConfigurableApplicationContext; -import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.ComponentScan; -import org.springframework.context.annotation.FilterType; -import org.springframework.core.env.CompositePropertySource; -import org.springframework.core.env.Environment; -import org.springframework.core.env.PropertySource; -import org.springframework.core.env.StandardEnvironment; -import org.springframework.web.filter.CharacterEncodingFilter; - -import javax.servlet.DispatcherType; -import javax.ws.rs.ext.RuntimeDelegate; -import java.lang.reflect.Field; -import java.util.EnumSet; - -/** - * Created by enjoyyin on 2018/8/6. - */ -@SpringBootApplication -@EnableDiscoveryClient -@RefreshScope -@ComponentScan(basePackages = "com.webank.wedatasphere", - excludeFilters = @ComponentScan.Filter(type = FilterType.CUSTOM, classes = {DataWorkCloudCustomExcludeFilter.class})) -public class DSSSpringApplication extends SpringBootServletInitializer { - private static final Log logger = LogFactory.getLog(DSSSpringApplication.class); - - private static ConfigurableApplicationContext applicationContext; - private static ServiceInstance serviceInstance; - - public static ApplicationContext getApplicationContext() { - return applicationContext; - } - - public static void main(String[] args) throws Exception { - RuntimeDelegate.setInstance(new org.glassfish.jersey.internal.RuntimeDelegateImpl()); - final SpringApplication application = new SpringApplication(DSSSpringApplication.class); - application.addListeners(new ApplicationListener(){ - public void onApplicationEvent(ApplicationPreparedEvent applicationPreparedEvent) { - logger.info("add config from config server..."); - if(applicationContext == null) { - applicationContext = applicationPreparedEvent.getApplicationContext(); - try { - setApplicationContext(applicationContext); - } catch (Exception e) { - logger.error(e); - } - } - addRemoteConfig(); - logger.info("initialize DataWorkCloud spring application..."); - initDWCApplication(); - } - }); - application.addListeners(new ApplicationListener() { - public void onApplicationEvent(RefreshScopeRefreshedEvent applicationEvent) { - logger.info("refresh config from config server..."); - updateRemoteConfig(); - } - }); - String listeners = ServerConfiguration.BDP_SERVER_SPRING_APPLICATION_LISTENERS().getValue(); - if(StringUtils.isNotBlank(listeners)) { - for (String listener : listeners.split(",")) { - application.addListeners((ApplicationListener) Class.forName(listener).newInstance()); - } - } - applicationContext = application.run(args); - setApplicationContext(applicationContext); - } - - private static void setApplicationContext(ConfigurableApplicationContext applicationContext) throws Exception { - Field applicationContextField = DataWorkCloudApplication.class.getDeclaredField("applicationContext"); - applicationContextField.setAccessible(true); - applicationContextField.set(null,applicationContext); - } - - public static void updateRemoteConfig() { - addOrUpdateRemoteConfig(applicationContext.getEnvironment(), true); - } - - public static void addRemoteConfig() { - addOrUpdateRemoteConfig(applicationContext.getEnvironment(), false); - } - - private static void addOrUpdateRemoteConfig(Environment env, boolean isUpdateOrNot) { - StandardEnvironment environment = (StandardEnvironment) env; - PropertySource propertySource = environment.getPropertySources().get("bootstrapProperties"); - if(propertySource == null) { - return; - } - CompositePropertySource source = (CompositePropertySource) propertySource; - for (String key: source.getPropertyNames()) { - Object val = source.getProperty(key); - if(val == null) { - continue; - } - if(isUpdateOrNot) { - logger.info("update remote config => " + key + " = " + source.getProperty(key)); - BDPConfiguration.set(key, val.toString()); - } else { - logger.info("add remote config => " + key + " = " + source.getProperty(key)); - BDPConfiguration.setIfNotExists(key, val.toString()); - } - } - } - - private static void initDWCApplication() { - serviceInstance = new ServiceInstance(); - serviceInstance.setApplicationName(applicationContext.getEnvironment().getProperty("spring.application.name")); - serviceInstance.setInstance(Utils.getComputerName() + ":" + applicationContext.getEnvironment().getProperty("server.port")); - try { - setServiceInstance(serviceInstance); - } catch (Exception e) { - logger.error(e); - } - DWCException.setApplicationName(serviceInstance.getApplicationName()); - DWCException.setHostname(Utils.getComputerName()); - DWCException.setHostPort(Integer.parseInt(applicationContext.getEnvironment().getProperty("server.port","9004"))); - } - - private static void setServiceInstance(ServiceInstance serviceInstance) throws Exception { - Field applicationContextField = DataWorkCloudApplication.class.getDeclaredField("serviceInstance"); - applicationContextField.setAccessible(true); - applicationContextField.set(null,serviceInstance); - } - - public static ServiceInstance getServiceInstance() { - return serviceInstance; - } - - public static String getApplicationName() { - return serviceInstance.getApplicationName(); - } - - public static String getInstance() { - return serviceInstance.getInstance(); - } - - public static void setProperty(String key, String value) { - BDPConfiguration.set(key, value); - } - - public static void setPropertyIfNotExists(String key, String value) { - BDPConfiguration.setIfNotExists(key, value); - } - - @Override - protected SpringApplicationBuilder configure(SpringApplicationBuilder builder) { - return builder.sources(DSSSpringApplication.class); - } - - @Bean - public WebServerFactoryCustomizer jettyFactoryCustomizer() { - return new WebServerFactoryCustomizer() { - public void customize(JettyServletWebServerFactory jettyServletWebServerFactory) { - jettyServletWebServerFactory.addServerCustomizers(new JettyServerCustomizer() { - public void customize(Server server) { - Handler[] childHandlersByClass = server.getChildHandlersByClass(WebAppContext.class); - final WebAppContext webApp = (WebAppContext) childHandlersByClass[0]; - FilterHolder filterHolder = new FilterHolder(CharacterEncodingFilter.class); - filterHolder.setInitParameter("encoding", Configuration.BDP_ENCODING().getValue()); - filterHolder.setInitParameter("forceEncoding", "true"); - webApp.addFilter(filterHolder, "/*", EnumSet.allOf(DispatcherType.class)); - BDPJettyServerHelper.setupRestApiContextHandler(webApp); - if(ServerConfiguration.BDP_SERVER_SOCKET_MODE().getValue()) { - BDPJettyServerHelper.setupControllerServer(webApp); - } - if(!ServerConfiguration.BDP_SERVER_DISTINCT_MODE().getValue()) { - BDPJettyServerHelper.setupWebAppContext(webApp); - } - } - }); - } - }; - } -} diff --git a/dss-server/src/main/java/com/webank/wedatasphere/dss/server/conf/DSSServerConf.java b/dss-server/src/main/java/com/webank/wedatasphere/dss/server/conf/DSSServerConf.java deleted file mode 100644 index aa4a454a78..0000000000 --- a/dss-server/src/main/java/com/webank/wedatasphere/dss/server/conf/DSSServerConf.java +++ /dev/null @@ -1,49 +0,0 @@ -/* - * Copyright 2019 WeBank - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package com.webank.wedatasphere.dss.server.conf; - - -import com.webank.wedatasphere.linkis.common.conf.CommonVars; -import com.webank.wedatasphere.linkis.common.conf.CommonVars$; - - -public class DSSServerConf { - public static final CommonVars PUBLISH_TIMEOUT = CommonVars$.MODULE$.apply("wds.dss.server.publish.timeout",60 * 10); - public static final CommonVars CACHE_TIMEOUT = CommonVars$.MODULE$.apply("wds.dss.server.cache.timeout",1000 * 60 * 60); - - public static final CommonVars BDP_SERVER_MYBATIS_MAPPER_LOCATIONS = CommonVars.apply("wds.linkis.server.mybatis.mapperLocations", ""); - public static final CommonVars BDP_SERVER_MYBATIS_TYPEALIASESPACKAGE = CommonVars.apply("wds.linkis.server.mybatis.typeAliasesPackage", ""); - public static final CommonVars BDP_SERVER_MYBATIS_CONFIGLOCATION = CommonVars.apply("wds.linkis.server.mybatis.configLocation", "classpath:/mybatis-config.xml"); - public static final CommonVars BDP_SERVER_MYBATIS_BASEPACKAGE = CommonVars.apply("wds.linkis.server.mybatis.BasePackage", ""); - public static final CommonVars BDP_SERVER_MYBATIS_DATASOURCE_URL = CommonVars.apply("wds.linkis.server.mybatis.datasource.url", ""); - public static final CommonVars BDP_SERVER_MYBATIS_DATASOURCE_USERNAME = CommonVars.apply("wds.linkis.server.mybatis.datasource.username", ""); - public static final CommonVars BDP_SERVER_MYBATIS_DATASOURCE_PASSWORD = CommonVars.apply("wds.linkis.server.mybatis.datasource.password", ""); - public static final CommonVars BDP_SERVER_MYBATIS_DATASOURCE_DRIVER_CLASS_NAME = CommonVars.apply("wds.linkis.server.mybatis.datasource.driver-class-name", "com.mysql.jdbc.Driver"); - public static final CommonVars BDP_SERVER_MYBATIS_DATASOURCE_INITIALSIZE = CommonVars.apply("wds.linkis.server.mybatis.datasource.initialSize", new Integer(1)); - public static final CommonVars BDP_SERVER_MYBATIS_DATASOURCE_MINIDLE = CommonVars.apply("wds.linkis.server.mybatis.datasource.minIdle", new Integer(1)); - public static final CommonVars BDP_SERVER_MYBATIS_DATASOURCE_MAXACTIVE = CommonVars.apply("wds.linkis.server.mybatis.datasource.maxActive", new Integer(20)); - public static final CommonVars BDP_SERVER_MYBATIS_DATASOURCE_MAXWAIT = CommonVars.apply("wds.linkis.server.mybatis.datasource.maxWait", new Integer(6000)); - public static final CommonVars BDP_SERVER_MYBATIS_DATASOURCE_TBERM = CommonVars.apply("wds.linkis.server.mybatis.datasource.timeBetweenEvictionRunsMillis", new Integer(60000)); - public static final CommonVars BDP_SERVER_MYBATIS_DATASOURCE_MEITM = CommonVars.apply("wds.linkis.server.mybatis.datasource.minEvictableIdleTimeMillis", new Integer(300000)); - public static final CommonVars BDP_SERVER_MYBATIS_DATASOURCE_VALIDATIONQUERY = CommonVars.apply("wds.linkis.server.mybatis.datasource.validationQuery", "SELECT 1"); - - public static final CommonVars BDP_SERVER_MYBATIS_DATASOURCE_TESTWHILEIDLE = CommonVars.apply("wds.linkis.server.mybatis.datasource.testWhileIdle", new Boolean(true)); - public static final CommonVars BDP_SERVER_MYBATIS_DATASOURCE_TESTONBORROW = CommonVars.apply("wds.linkis.server.mybatis.datasource.testOnBorrow", new Boolean(false)); - public static final CommonVars BDP_SERVER_MYBATIS_DATASOURCE_TESTONRETURN = CommonVars.apply("wds.linkis.server.mybatis.datasource.testOnReturn", new Boolean(false)); - public static final CommonVars BDP_SERVER_MYBATIS_DATASOURCE_POOLPREPAREDSTATEMENTS = CommonVars.apply("wds.linkis.server.mybatis.datasource.poolPreparedStatements", new Boolean(true)); -} diff --git a/dss-server/src/main/java/com/webank/wedatasphere/dss/server/constant/DSSServerConstant.java b/dss-server/src/main/java/com/webank/wedatasphere/dss/server/constant/DSSServerConstant.java deleted file mode 100644 index 2c5711e154..0000000000 --- a/dss-server/src/main/java/com/webank/wedatasphere/dss/server/constant/DSSServerConstant.java +++ /dev/null @@ -1,33 +0,0 @@ -/* - * Copyright 2019 WeBank - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package com.webank.wedatasphere.dss.server.constant; - - -public class DSSServerConstant { - public static final String DSS_PROJECT_FIRST_VERSION = "v000001"; - public static final String DSS_PROJECT_FIRST_VERSION_COMMENT = "first version"; - public static final String DSS_PROJECT_SOURCE = "create by user"; - public static final String DSS_WORKSPACE_SOURCE = "create by user"; - public static final String PROJECT_VERSION_ID = "projectVersionID"; - public static final String PUBLISH_FLOW_REPORT_FORMATE = "the workflow name is% s, the version number is% s, and the workflow content is empty. Please modify or delete it by yourself"; - public static final String EMVEDDEDFLOWID = "\"embeddedFlowId\":"; - public static final String VERSION_FORMAT = "%06d"; - public static final String VERSION_PREFIX = "v"; - public static final String SUPER_USER_LOGIN_ERROR = "please login with super user"; - -} diff --git a/dss-server/src/main/java/com/webank/wedatasphere/dss/server/dao/DSSUserMapper.java b/dss-server/src/main/java/com/webank/wedatasphere/dss/server/dao/DSSUserMapper.java deleted file mode 100644 index 31d4974e12..0000000000 --- a/dss-server/src/main/java/com/webank/wedatasphere/dss/server/dao/DSSUserMapper.java +++ /dev/null @@ -1,32 +0,0 @@ -/* - * Copyright 2019 WeBank - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package com.webank.wedatasphere.dss.server.dao; - -import com.webank.wedatasphere.dss.application.entity.DSSUser; - -public interface DSSUserMapper { - Long getUserID(String userName); - - String getuserName(Long userID); - - DSSUser getUserByName(String username); - - void registerDSSUser(DSSUser userDb); - - void updateUserFirstLogin(Long userId); -} diff --git a/dss-server/src/main/java/com/webank/wedatasphere/dss/server/dao/FlowMapper.java b/dss-server/src/main/java/com/webank/wedatasphere/dss/server/dao/FlowMapper.java deleted file mode 100644 index ea830d7cf8..0000000000 --- a/dss-server/src/main/java/com/webank/wedatasphere/dss/server/dao/FlowMapper.java +++ /dev/null @@ -1,71 +0,0 @@ -/* - * Copyright 2019 WeBank - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package com.webank.wedatasphere.dss.server.dao; - - -import com.webank.wedatasphere.dss.common.entity.flow.DSSFlow; -import com.webank.wedatasphere.dss.common.entity.flow.DSSFlowVersion; -import org.apache.ibatis.annotations.Param; -import org.springframework.dao.DuplicateKeyException; - -import java.util.List; - - -public interface FlowMapper { - DSSFlow selectFlowByID(Long id); - - List listFlowByTaxonomyID(@Param("projectID") Long projectID, @Param("taxonomyID") Long taxonomyID, @Param("isRootFlow") Boolean isRootFlow); - - List listFlowVersionsByFlowID(@Param("flowID") Long flowID, @Param("projectVersionID") Long projectVersionID); - - void insertFlow(DSSFlow dssFlow) throws DuplicateKeyException; - - void insertFlowVersion(DSSFlowVersion version); - - void batchInsertFlowVersion(@Param("flowVersions") List flowVersions); - - void insertFlowRelation(@Param("flowID") Long flowID, @Param("parentFlowID") Long parentFlowID); - - DSSFlowVersion selectVersionByFlowID(@Param("flowID") Long flowID, @Param("version") String version, @Param("projectVersionID") Long projectVersionID); - - void updateFlowBaseInfo(DSSFlow dssFlow) throws DuplicateKeyException; - - List selectSubFlowIDByParentFlowID(Long parentFlowID); - - void deleteFlowVersions(@Param("flowID") Long flowID, @Param("projectVersionID") Long projectVersionID); - - void deleteFlowBaseInfo(Long flowID); - - void deleteFlowRelation(Long flowID); - - Long selectParentFlowIDByFlowID(Long flowID); - - List listFlowByProjectID(Long projectID); - - List listVersionByFlowIDAndProjectVersionID(@Param("flowID") Long flowID, @Param("projectVersionID") Long projectVersionID); - - Boolean noVersions(Long flowID); - - List listLastFlowVersionsByProjectVersionID(@Param("projectVersionID") Long projectVersionId); - - List listLatestRootFlowVersionByProjectVersionID(Long projectVersionID); - - void batchUpdateFlowVersion(List flowVersions); - - Long getParentFlowID(Long flowID); -} diff --git a/dss-server/src/main/java/com/webank/wedatasphere/dss/server/dao/FlowTaxonomyMapper.java b/dss-server/src/main/java/com/webank/wedatasphere/dss/server/dao/FlowTaxonomyMapper.java deleted file mode 100644 index 640e8b7d03..0000000000 --- a/dss-server/src/main/java/com/webank/wedatasphere/dss/server/dao/FlowTaxonomyMapper.java +++ /dev/null @@ -1,51 +0,0 @@ -/* - * Copyright 2019 WeBank - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package com.webank.wedatasphere.dss.server.dao; - - -import com.webank.wedatasphere.dss.server.entity.DSSFlowTaxonomy; -import org.apache.ibatis.annotations.Param; -import org.springframework.dao.DuplicateKeyException; - -import java.util.List; - - -public interface FlowTaxonomyMapper { - DSSFlowTaxonomy selectFlowTaxonomyByID(Long id); - - void insertFlowTaxonomy(DSSFlowTaxonomy dssFlowTaxonomy) throws DuplicateKeyException; - - void updateFlowTaxonomy(DSSFlowTaxonomy dssFlowTaxonomy) throws DuplicateKeyException; - - Long hasFlows(Long flowTaxonomyID); - - void deleteFlowTaxonomy(Long flowTaxonomyID); - - - void insertFlowTaxonomyRelation(@Param("taxonomyID") Long taxonomyID, @Param("flowID") Long flowID); - - Long selectTaxonomyIDByFlowID(Long id); - - void updateFlowTaxonomyRelation(@Param("flowIDList") List flowIDList, @Param("taxonomyID") Long taxonomyID); - - void deleteFlowTaxonomyRelation(Long flowID); - - void deleteFlowTaxonomyByProjectID(Long projectID); - - List listFlowTaxonomyByProjectID(Long projectID); -} diff --git a/dss-server/src/main/java/com/webank/wedatasphere/dss/server/dao/NodeInfoMapper.java b/dss-server/src/main/java/com/webank/wedatasphere/dss/server/dao/NodeInfoMapper.java deleted file mode 100644 index dcf8d0594e..0000000000 --- a/dss-server/src/main/java/com/webank/wedatasphere/dss/server/dao/NodeInfoMapper.java +++ /dev/null @@ -1,35 +0,0 @@ -/* - * Copyright 2019 WeBank - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package com.webank.wedatasphere.dss.server.dao; - - - - -import com.webank.wedatasphere.dss.application.entity.Application; -import com.webank.wedatasphere.dss.server.entity.NodeInfo; - -import java.util.List; - - -public interface NodeInfoMapper { - List selectNodeInfo(String developProject); - - String getApplicationUrlByNodeType(String nodeType); - - Application getApplicationbyNodeType(String nodeType); -} diff --git a/dss-server/src/main/java/com/webank/wedatasphere/dss/server/dao/ProjectMapper.java b/dss-server/src/main/java/com/webank/wedatasphere/dss/server/dao/ProjectMapper.java deleted file mode 100644 index c2ae56786d..0000000000 --- a/dss-server/src/main/java/com/webank/wedatasphere/dss/server/dao/ProjectMapper.java +++ /dev/null @@ -1,66 +0,0 @@ -/* - * Copyright 2019 WeBank - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package com.webank.wedatasphere.dss.server.dao; - - -import com.webank.wedatasphere.dss.common.entity.project.DSSProject; -import com.webank.wedatasphere.dss.common.entity.project.DSSProjectPublishHistory; -import com.webank.wedatasphere.dss.common.entity.project.DSSProjectVersion; -import org.apache.ibatis.annotations.Param; - -import java.util.List; -import java.util.Map; - - -public interface ProjectMapper { - DSSProject selectProjectByID(Long id); - - DSSProjectVersion selectLatestVersionByProjectID(Long projectID); - - DSSProject selectProjectByVersionID(Long projectVersionID); - - void addProject(DSSProject dssProject); - - void addProjectVersion(DSSProjectVersion dssProjectVersion); - - void updateDescription(@Param("projectID") Long projectID, @Param("description") String description, @Param("product")String product ,@Param("applicationArea")Integer applicationArea ,@Param("business")String business); - - List listProjectVersionsByProjectID(Long projectID); - - Boolean noPublished(Long projectID); - - void deleteProjectVersions(long projectID); - - void deleteProjectBaseInfo(long projectID); - - DSSProjectVersion selectProjectVersionByID(Long id); - - DSSProjectVersion selectProjectVersionByProjectIDAndVersionID(@Param("projectID") Long projectId, @Param("version") String version); - - Integer updateLock(@Param("lock") Integer lock, @Param("projectVersionID") Long projectVersionID); - - DSSProjectPublishHistory selectProjectPublishHistoryByProjectVersionID(Long projectVersionID); - - void insertPublishHistory(DSSProjectPublishHistory dssProjectPublishHistory); - - void updatePublishHistoryState(@Param("projectVersionID") Long projectVersionID, @Param("status") Integer status); - - void addAccessProjectRelation(@Param("appjointProjectIDAndAppID") Map appjointProjectIDAndAppID, @Param("projectID") Long projectID); - - Long getAppjointProjectID(@Param("projectID") Long projectID, @Param("applicationID") Integer applicationID); -} diff --git a/dss-server/src/main/java/com/webank/wedatasphere/dss/server/dao/ProjectTaxonomyMapper.java b/dss-server/src/main/java/com/webank/wedatasphere/dss/server/dao/ProjectTaxonomyMapper.java deleted file mode 100644 index 21fc44e41a..0000000000 --- a/dss-server/src/main/java/com/webank/wedatasphere/dss/server/dao/ProjectTaxonomyMapper.java +++ /dev/null @@ -1,49 +0,0 @@ -/* - * Copyright 2019 WeBank - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package com.webank.wedatasphere.dss.server.dao; - -import com.webank.wedatasphere.dss.server.entity.DSSProjectTaxonomy; -import com.webank.wedatasphere.dss.server.entity.DSSProjectTaxonomyRelation; -import org.apache.ibatis.annotations.Param; -import org.springframework.dao.DuplicateKeyException; - -import java.util.List; - - -public interface ProjectTaxonomyMapper { - DSSProjectTaxonomy selectProjectTaxonomyByID(Long id); - DSSProjectTaxonomyRelation selectProjectTaxonomyRelationByTaxonomyIdOrProjectId(Long taxonomyIdOrProjectId); - List listProjectTaxonomyByUser(String userName); - - //-------------------- - List listProjectIDByTaxonomyID(@Param("taxonomyID") Long taxonomyID, @Param("userName") String userName); - - void insertProjectTaxonomy(DSSProjectTaxonomy dssProjectTaxonomy) throws DuplicateKeyException; - - void updateProjectTaxonomy(DSSProjectTaxonomy dssProjectTaxonomy) throws DuplicateKeyException; - - Long hasProjects(Long projectTaxonomyID); - - void deleteProjectTaxonomy(Long projectTaxonomyID); - - void addProjectTaxonomyRelation(@Param("projectID") Long id, @Param("taxonomyID") Long taxonomyID, @Param("creatorID") Long creatorID); - - void updateProjectTaxonomyRelation(@Param("projectID") Long projectID, @Param("taxonomyID") Long taxonomyID); - - void deleteProjectTaxonomyRelationByProjectID(Long projectID); -} diff --git a/dss-server/src/main/java/com/webank/wedatasphere/dss/server/dao/WorkspaceMapper.java b/dss-server/src/main/java/com/webank/wedatasphere/dss/server/dao/WorkspaceMapper.java deleted file mode 100644 index c3fbaca222..0000000000 --- a/dss-server/src/main/java/com/webank/wedatasphere/dss/server/dao/WorkspaceMapper.java +++ /dev/null @@ -1,68 +0,0 @@ -/* - * Copyright 2019 WeBank - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package com.webank.wedatasphere.dss.server.dao; - -import com.webank.wedatasphere.dss.server.dto.response.*; -import com.webank.wedatasphere.dss.server.entity.*; -import com.webank.wedatasphere.dss.server.dto.response.HomepageDemoInstanceVo; -import com.webank.wedatasphere.dss.server.dto.response.HomepageDemoMenuVo; -import com.webank.wedatasphere.dss.server.dto.response.HomepageVideoVo; -import com.webank.wedatasphere.dss.server.dto.response.WorkspaceFavoriteVo; -import org.apache.ibatis.annotations.Param; - -import java.util.List; - -/** - * Created by schumiyi on 2020/6/22 - */ -public interface WorkspaceMapper { - - List getWorkspaces(); - - List findByWorkspaceName(String name); - - void addWorkSpace(DSSWorkspace dssWorkspace); - - List getHomepageDemoMenusEn(); - List getHomepageDemoMenusCn(); - - List getHomepageInstancesByMenuIdCn(Long id); - List getHomepageInstancesByMenuIdEn(Long id); - - List getHomepageVideosEn(); - List getHomepageVideosCn(); - - DSSWorkspace getWorkspaceById(Long workspaceId); - - List getManagementMenuCn(); - List getManagementMenuEn(); - - List getApplicationMenuCn(); - List getApplicationMenuEn(); - - List getMenuAppInstancesCn(Long id); - List getMenuAppInstancesEn(Long id); - - List getWorkspaceFavoritesCn(@Param("username") String username, @Param("workspaceId") Long workspaceId); - - List getWorkspaceFavoritesEn(@Param("username") String username, @Param("workspaceId") Long workspaceId); - - void addFavorite(DSSFavorite dssFavorite); - - void deleteFavorite(Long favouritesId); -} diff --git a/dss-server/src/main/java/com/webank/wedatasphere/dss/server/dao/impl/dwsUserMapper.xml b/dss-server/src/main/java/com/webank/wedatasphere/dss/server/dao/impl/dwsUserMapper.xml deleted file mode 100644 index ac65dce4fc..0000000000 --- a/dss-server/src/main/java/com/webank/wedatasphere/dss/server/dao/impl/dwsUserMapper.xml +++ /dev/null @@ -1,31 +0,0 @@ - - - - - - - - - - - - \ No newline at end of file diff --git a/dss-server/src/main/java/com/webank/wedatasphere/dss/server/dao/impl/flowMapper.xml b/dss-server/src/main/java/com/webank/wedatasphere/dss/server/dao/impl/flowMapper.xml deleted file mode 100644 index 37dae091a4..0000000000 --- a/dss-server/src/main/java/com/webank/wedatasphere/dss/server/dao/impl/flowMapper.xml +++ /dev/null @@ -1,256 +0,0 @@ - - - - - - - - - - id,`name`,`state`,`source`,`description`,`create_time`,`creator_id`,`is_root_flow`,`rank`,`project_id`,`has_saved`,`uses` - - - - id,`flow_id`,`source`,`version`,`json_path`,`comment`,`update_time`,`updator_id`,`project_version_id` - - - - - - - - - - - - - - - - INSERT INTO dss_flow () - VALUES - (#{id},#{name},#{state},#{source},#{description},#{createTime},#{creatorID},#{isRootFlow},#{rank},#{projectID},#{hasSaved},#{uses}) - - - - INSERT INTO dss_flow_version () - VALUES - (#{id},#{flowID},#{source},#{version},#{jsonPath},#{comment},#{updateTime},#{updatorID},#{projectVersionID}) - - - - INSERT INTO dss_flow_version - ( `flow_id`,`source`,`version`,`json_path`,`comment`,`update_time`,`updator_id`,`project_version_id`) - VALUES - - (#{fv.flowID},#{fv.source},#{fv.version},#{fv.jsonPath},#{fv.comment},#{fv.updateTime},#{fv.updatorID},#{fv.projectVersionID}) - - - - - - UPDATE dss_flow_version - source= #{fv.source} - version= #{fv.version} - json_path= #{fv.jsonPath} - comment= #{fv.comment} - update_time= #{fv.updateTime} - updator_id= #{fv.updatorID} - project_version_id= #{fv.projectVersionID} - flow_id = #{fv.flowID} - - - - - INSERT INTO dss_flow_relation (flow_id,parent_flow_id) - VALUES - (#{flowID},#{parentFlowID}) - - - - - - UPDATE dss_flow - - name=#{name}, - description=#{description}, - has_saved=#{hasSaved}, - uses=#{uses}, - - WHERE id =#{id} - - - - - - DELETE - FROM - dss_flow_version - WHERE flow_id = #{flowID} AND project_version_id = #{projectVersionID} - - - - DELETE - FROM - dss_flow - WHERE id = #{flowID} - - - - DELETE - FROM - dss_flow_relation - WHERE flow_id = #{flowID} - - - - - - - - - - - - - - - - - \ No newline at end of file diff --git a/dss-server/src/main/java/com/webank/wedatasphere/dss/server/dao/impl/flowTaxonomyMapper.xml b/dss-server/src/main/java/com/webank/wedatasphere/dss/server/dao/impl/flowTaxonomyMapper.xml deleted file mode 100644 index c681f768ca..0000000000 --- a/dss-server/src/main/java/com/webank/wedatasphere/dss/server/dao/impl/flowTaxonomyMapper.xml +++ /dev/null @@ -1,95 +0,0 @@ - - - - - - - - - - id, `name`,`description`,`creator_id`,`create_time`,`update_time`,`project_id` - - - - - - INSERT INTO dss_flow_taxonomy () - VALUES - (#{id},#{name},#{description},#{creatorID},#{createTime},#{updateTime},#{projectID}) - - - - UPDATE dss_flow_taxonomy - - name=#{name}, - description=#{description}, - update_time=#{updateTime}, - - WHERE id =#{id} - - - - - - DELETE FROM dss_flow_taxonomy WHERE id = #{flowTaxonomyID} - - - - INSERT INTO dss_flow_taxonomy_relation(taxonomy_id,flow_id) VALUES (#{taxonomyID},#{flowID}) - - - - - - UPDATE dss_flow_taxonomy_relation - SET taxonomy_id = #{taxonomyID} - WHERE #{element} - - - - DELETE - FROM - dss_flow_taxonomy_relation - WHERE flow_id = #{flowID} - - - - DELETE - FROM - `dss_flow_taxonomy` - WHERE - project_id = #{projectID} - - - - - \ No newline at end of file diff --git a/dss-server/src/main/java/com/webank/wedatasphere/dss/server/dao/impl/nodeInfoMapper.xml b/dss-server/src/main/java/com/webank/wedatasphere/dss/server/dao/impl/nodeInfoMapper.xml deleted file mode 100644 index 2da4608304..0000000000 --- a/dss-server/src/main/java/com/webank/wedatasphere/dss/server/dao/impl/nodeInfoMapper.xml +++ /dev/null @@ -1,52 +0,0 @@ - - - - - - - - - - - - - \ No newline at end of file diff --git a/dss-server/src/main/java/com/webank/wedatasphere/dss/server/dao/impl/projectMapper.xml b/dss-server/src/main/java/com/webank/wedatasphere/dss/server/dao/impl/projectMapper.xml deleted file mode 100644 index a1bc4ab8cc..0000000000 --- a/dss-server/src/main/java/com/webank/wedatasphere/dss/server/dao/impl/projectMapper.xml +++ /dev/null @@ -1,177 +0,0 @@ - - - - - - - - - - id,`name`,`source`,`description`,`org_id`,`visibility`,`is_transfer`,`initial_org_id`,`user_id`,`create_time`,`create_by`,`product`,`application_area`,`business`,`workspace_id` - - - - id,`project_id`,`version`,`comment`,`update_time`,`updator_id`,`lock` - - - - id,`project_version_id`,`create_time`,`creator_id`,`update_time`,`comment`,`state`,`version_path`,`expire_time` - - - - - - - - - - INSERT INTO dss_project () - VALUES - (#{id},#{name},#{source},#{description},#{orgID},#{visibility},#{isTransfer},#{initialOrgID},#{userID},#{createTime},#{createBy},#{product},#{applicationArea},#{business},#{workspaceId}) - - - - INSERT INTO dss_project_version () - VALUES - (#{id},#{projectID},#{version},#{comment},#{updateTime},#{updatorID},#{lock}) - - - - UPDATE dss_project - SET description = #{description},product = #{product},application_area = #{applicationArea},business = #{business} - WHERE id = #{projectID} - - - - - - - - - - DELETE - FROM - dss_project_version - WHERE - project_id = #{projectID} - - - - DELETE - FROM - dss_project - WHERE - id = #{projectID} - - - - - - - - UPDATE `dss_project_version` set `lock` = `lock` +1 WHERE id= #{projectVersionID} AND `lock` = #{lock} - - - - - - INSERT INTO dss_project_publish_history () - VALUES - (#{id},#{projectVersionID},#{createTime},#{createID},#{updateTime},#{comment},#{state},#{versionPath},#{expireTime}) - - - - UPDATE `dss_project_publish_history` - SET state = #{status} - WHERE - project_version_id = #{projectVersionID} - - - - INSERT INTO dss_project_applications_project - ( `project_id`,`application_id`,`application_project_id`) - VALUES - - (#{projectID},#{key},#{item}) - - - - - - \ No newline at end of file diff --git a/dss-server/src/main/java/com/webank/wedatasphere/dss/server/dao/impl/projectTaxonomyMapper.xml b/dss-server/src/main/java/com/webank/wedatasphere/dss/server/dao/impl/projectTaxonomyMapper.xml deleted file mode 100644 index 9c0333953c..0000000000 --- a/dss-server/src/main/java/com/webank/wedatasphere/dss/server/dao/impl/projectTaxonomyMapper.xml +++ /dev/null @@ -1,97 +0,0 @@ - - - - - - - - - id, `name`,`description`,`creator_id`,`create_time`,`update_time` - - - - - - - - - - - INSERT INTO dss_project_taxonomy () - VALUES - (#{id},#{name},#{description},#{creatorID},#{createTime},#{updateTime}) - - - - UPDATE dss_project_taxonomy - - name=#{name}, - description=#{description}, - update_time=#{updateTime}, - - WHERE id =#{id} - - - - - - DELETE FROM dss_project_taxonomy WHERE id = #{projectTaxonomyID} - - - - INSERT INTO dss_project_taxonomy_relation VALUES (#{taxonomyID},#{projectID},#{creatorID}) - - - - UPDATE dss_project_taxonomy_relation set taxonomy_id = #{taxonomyID} WHERE project_id = #{projectID} - - - - DELETE - FROM - dss_project_taxonomy_relation - WHERE - project_id = #{projectID} - - \ No newline at end of file diff --git a/dss-server/src/main/java/com/webank/wedatasphere/dss/server/dao/impl/workspaceMapper.xml b/dss-server/src/main/java/com/webank/wedatasphere/dss/server/dao/impl/workspaceMapper.xml deleted file mode 100644 index e1fc05ed9d..0000000000 --- a/dss-server/src/main/java/com/webank/wedatasphere/dss/server/dao/impl/workspaceMapper.xml +++ /dev/null @@ -1,200 +0,0 @@ - - - - - - - - - id,`name`,`label`,`description`,`department`,`product`,`source`,`create_by`,`create_time`,`last_update_user`,`last_update_time` - - - - m.`id`,m.`title_cn` AS `title`, m.`desc_cn` AS `description`,m.`labels_cn` AS `labels`, - m.`access_button_cn` AS `access_button`,m.`manual_button_cn` AS `manualButton`,m.`is_active`, - m.`manual_button_url`,m.`icon`,m.`order`,app.`homepage_url` AS `access_button_url`,app.project_url, app.`name` - - - m.`id`,m.`title_en` AS `title`, m.`desc_en` AS `description`,m.`labels_en` AS `labels`, - m.`access_button_en` AS `access_button`,m.`manual_button_en` AS `manualButton`,m.`is_active`, - m.`manual_button_url`,m.`icon`,m.`order`,app.`homepage_url` AS `access_button_url`,app.project_url, app.`name` - - - - id,`username`,`workspace_id`,`menu_application_id`,`order`,`create_by`,`create_time`,`last_update_user`,`last_update_time` - - - - - - - - - - INSERT INTO dss_workspace () - VALUES - (#{id},#{name},#{label},#{description},#{department},#{product},#{source},#{createBy},#{createTime},#{lastUpdateUser},#{lastUpdateTime}) - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - INSERT INTO dss_onestop_user_favorites () - VALUES - (#{id},#{username},#{workspaceId},#{menuApplicationId},#{order},#{createBy},#{createTime},#{lastUpdateUser},#{lastUpdateTime}) - - - - DELETE - FROM - dss_onestop_user_favorites - WHERE - id = #{favouritesId} - - \ No newline at end of file diff --git a/dss-server/src/main/java/com/webank/wedatasphere/dss/server/db/DSSMybatisConfig.java b/dss-server/src/main/java/com/webank/wedatasphere/dss/server/db/DSSMybatisConfig.java deleted file mode 100644 index 2ce87d062c..0000000000 --- a/dss-server/src/main/java/com/webank/wedatasphere/dss/server/db/DSSMybatisConfig.java +++ /dev/null @@ -1,148 +0,0 @@ -/* - * Copyright 2019 WeBank - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package com.webank.wedatasphere.dss.server.db; - -import com.webank.wedatasphere.dss.server.conf.DSSServerConf; -import org.apache.commons.collections.CollectionUtils; -import org.apache.commons.dbcp.BasicDataSource; -import org.apache.commons.lang.StringUtils; -import org.apache.ibatis.session.SqlSessionFactory; -import org.mybatis.spring.SqlSessionFactoryBean; -import org.mybatis.spring.mapper.MapperScannerConfigurer; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.springframework.beans.factory.annotation.Qualifier; -import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.Configuration; -import org.springframework.context.annotation.Primary; -import org.springframework.core.io.DefaultResourceLoader; -import org.springframework.core.io.Resource; -import org.springframework.core.io.support.PathMatchingResourcePatternResolver; -import org.springframework.jdbc.datasource.DataSourceTransactionManager; -import org.springframework.transaction.PlatformTransactionManager; -import org.springframework.transaction.annotation.EnableTransactionManagement; - -import javax.sql.DataSource; -import java.io.IOException; -import java.util.ArrayList; -import java.util.List; - - -@Configuration -@EnableTransactionManagement(order = 2) -public class DSSMybatisConfig { - - private Logger logger = LoggerFactory.getLogger(this.getClass()); - - - @Bean(name = "dataSource", destroyMethod = "close") - @Primary - public DataSource dataSource() { - BasicDataSource datasource = new BasicDataSource(); - String dbUrl = DSSServerConf.BDP_SERVER_MYBATIS_DATASOURCE_URL.getValue(); - String username = DSSServerConf.BDP_SERVER_MYBATIS_DATASOURCE_USERNAME.getValue(); - String password = DSSServerConf.BDP_SERVER_MYBATIS_DATASOURCE_PASSWORD.getValue(); - String driverClassName = DSSServerConf.BDP_SERVER_MYBATIS_DATASOURCE_DRIVER_CLASS_NAME.getValue(); - int initialSize = DSSServerConf.BDP_SERVER_MYBATIS_DATASOURCE_INITIALSIZE.getValue(); - int minIdle = DSSServerConf.BDP_SERVER_MYBATIS_DATASOURCE_MINIDLE.getValue(); - int maxActive = DSSServerConf.BDP_SERVER_MYBATIS_DATASOURCE_MAXACTIVE.getValue(); - int maxWait = DSSServerConf.BDP_SERVER_MYBATIS_DATASOURCE_MAXWAIT.getValue(); - int timeBetweenEvictionRunsMillis = DSSServerConf.BDP_SERVER_MYBATIS_DATASOURCE_TBERM.getValue(); - int minEvictableIdleTimeMillis = DSSServerConf.BDP_SERVER_MYBATIS_DATASOURCE_MEITM.getValue(); - String validationQuery = DSSServerConf.BDP_SERVER_MYBATIS_DATASOURCE_VALIDATIONQUERY.getValue(); - boolean testWhileIdle = DSSServerConf.BDP_SERVER_MYBATIS_DATASOURCE_TESTWHILEIDLE.getValue(); - boolean testOnBorrow = DSSServerConf.BDP_SERVER_MYBATIS_DATASOURCE_TESTONBORROW.getValue(); - boolean testOnReturn = DSSServerConf.BDP_SERVER_MYBATIS_DATASOURCE_TESTONRETURN.getValue(); - boolean poolPreparedStatements = DSSServerConf.BDP_SERVER_MYBATIS_DATASOURCE_POOLPREPAREDSTATEMENTS.getValue(); - datasource.setUrl(dbUrl); - datasource.setUsername(username); - datasource.setPassword(password); - datasource.setDriverClassName(driverClassName); - datasource.setInitialSize(initialSize); - datasource.setMinIdle(minIdle); - datasource.setMaxActive(maxActive); - datasource.setMaxWait(maxWait); - datasource.setTimeBetweenEvictionRunsMillis(timeBetweenEvictionRunsMillis); - datasource.setMinEvictableIdleTimeMillis(minEvictableIdleTimeMillis); - datasource.setValidationQuery(validationQuery); - datasource.setTestWhileIdle(testWhileIdle); - datasource.setTestOnBorrow(testOnBorrow); - datasource.setTestOnReturn(testOnReturn); - datasource.setPoolPreparedStatements(poolPreparedStatements); - logger.info("Database connection address information(数据库连接地址信息)=" + dbUrl); - return datasource; - } - - @Bean(name = "sqlSessionFactory") - @Primary - public SqlSessionFactory sqlSessionFactory(@Qualifier("dataSource") DataSource dataSource) throws Exception { - String typeAliasesPackage = DSSServerConf.BDP_SERVER_MYBATIS_TYPEALIASESPACKAGE.getValue(); - //Configure the mapper scan to find all mapper.xml mapping files(配置mapper的扫描，找到所有的mapper.xml映射文件) - String mapperLocations = DSSServerConf.BDP_SERVER_MYBATIS_MAPPER_LOCATIONS.getValue(); - //Load the global configuration file(加载全局的配置文件) - String configLocation = DSSServerConf.BDP_SERVER_MYBATIS_CONFIGLOCATION.getValue(); - try { - SqlSessionFactoryBean sessionFactoryBean = new SqlSessionFactoryBean(); - sessionFactoryBean.setDataSource(dataSource); - - logger.info("Mybatis typeAliasesPackage=" + typeAliasesPackage); - logger.info("Mybatis mapperLocations=" + mapperLocations); - logger.info("Mybatis configLocation=" + configLocation); - // Read configuration(读取配置) - sessionFactoryBean.setTypeAliasesPackage(typeAliasesPackage); - - //Set the location of the mapper.xml file(设置mapper.xml文件所在位置) - if(StringUtils.isNotBlank(mapperLocations)) { - String[] mapperArray = mapperLocations.split(","); - List resources = new ArrayList<>(); - for(String mapperLocation : mapperArray){ - CollectionUtils.addAll(resources,new PathMatchingResourcePatternResolver().getResources(mapperLocation)); - } - sessionFactoryBean.setMapperLocations(resources.toArray(new Resource[0])); - } - /* Resource[] resources = new PathMatchingResourcePatternResolver().getResources(mapperLocations); - sessionFactoryBean.setMapperLocations(resources);*/ -// Set the location of the mybatis-config.xml configuration file(设置mybatis-config.xml配置文件位置) - sessionFactoryBean.setConfigLocation(new DefaultResourceLoader().getResource(configLocation)); - - return sessionFactoryBean.getObject(); - } catch (IOException e) { - logger.error("mybatis resolver mapper*xml is error",e); - return null; - } catch (Exception e) { - logger.error("mybatis sqlSessionFactoryBean create error",e); - return null; - } - } - - @Primary - public PlatformTransactionManager annotationDrivenTransactionManager(@Qualifier("dataSource") DataSource dataSource) { - return new DataSourceTransactionManager(dataSource); - } - - @Bean - public MapperScannerConfigurer mapperScannerConfigurer(@Qualifier("sqlSessionFactory") SqlSessionFactory sqlSessionFactory) { - MapperScannerConfigurer mapperScannerConfigurer = new MapperScannerConfigurer(); - mapperScannerConfigurer.setSqlSessionFactory(sqlSessionFactory); - //Each table corresponds to the XXMapper.java interface type Java file - //每张表对应的XXMapper.java interface类型的Java文件 - mapperScannerConfigurer.setBasePackage(DSSServerConf.BDP_SERVER_MYBATIS_BASEPACKAGE.getValue()); - return mapperScannerConfigurer; - } - -} \ No newline at end of file diff --git a/dss-server/src/main/java/com/webank/wedatasphere/dss/server/dto/response/HomepageDemoInstanceVo.java b/dss-server/src/main/java/com/webank/wedatasphere/dss/server/dto/response/HomepageDemoInstanceVo.java deleted file mode 100644 index b880f0f2ad..0000000000 --- a/dss-server/src/main/java/com/webank/wedatasphere/dss/server/dto/response/HomepageDemoInstanceVo.java +++ /dev/null @@ -1,96 +0,0 @@ -/* - * Copyright 2019 WeBank - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ -package com.webank.wedatasphere.dss.server.dto.response; - -/** - * Created by schumiyi on 2020/6/23 - */ -public class HomepageDemoInstanceVo { - - private Long id; - private Long menuId; - private String name; - private String url; - private String title; - private String description; - private String icon; - private Integer order; - - public Long getId() { - return id; - } - - public void setId(Long id) { - this.id = id; - } - - public Long getMenuId() { - return menuId; - } - - public void setMenuId(Long menuId) { - this.menuId = menuId; - } - - public String getName() { - return name; - } - - public void setName(String name) { - this.name = name; - } - - public String getUrl() { - return url; - } - - public void setUrl(String url) { - this.url = url; - } - - public String getTitle() { - return title; - } - - public void setTitle(String title) { - this.title = title; - } - - public String getDescription() { - return description; - } - - public void setDescription(String description) { - this.description = description; - } - - public String getIcon() { - return icon; - } - - public void setIcon(String icon) { - this.icon = icon; - } - - public Integer getOrder() { - return order; - } - - public void setOrder(Integer order) { - this.order = order; - } -} diff --git a/dss-server/src/main/java/com/webank/wedatasphere/dss/server/dto/response/HomepageDemoMenuVo.java b/dss-server/src/main/java/com/webank/wedatasphere/dss/server/dto/response/HomepageDemoMenuVo.java deleted file mode 100644 index c7d4fdcbd3..0000000000 --- a/dss-server/src/main/java/com/webank/wedatasphere/dss/server/dto/response/HomepageDemoMenuVo.java +++ /dev/null @@ -1,89 +0,0 @@ -/* - * Copyright 2019 WeBank - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ -package com.webank.wedatasphere.dss.server.dto.response; - -import java.util.List; - -/** - * Created by schumiyi on 2020/6/23 - */ -public class HomepageDemoMenuVo { - - private Long id; - private String name; - private String title; - private String description; - private String icon; - private Integer order; - private List demoInstances; - - public Long getId() { - return id; - } - - public void setId(Long id) { - this.id = id; - } - - public String getName() { - return name; - } - - public void setName(String name) { - this.name = name; - } - - public String getTitle() { - return title; - } - - public void setTitle(String title) { - this.title = title; - } - - public String getDescription() { - return description; - } - - public void setDescription(String description) { - this.description = description; - } - - public String getIcon() { - return icon; - } - - public void setIcon(String icon) { - this.icon = icon; - } - - public Integer getOrder() { - return order; - } - - public void setOrder(Integer order) { - this.order = order; - } - - public List getDemoInstances() { - return demoInstances; - } - - public void setDemoInstances(List demoInstances) { - this.demoInstances = demoInstances; - } -} diff --git a/dss-server/src/main/java/com/webank/wedatasphere/dss/server/dto/response/HomepageVideoVo.java b/dss-server/src/main/java/com/webank/wedatasphere/dss/server/dto/response/HomepageVideoVo.java deleted file mode 100644 index 0f9ce857df..0000000000 --- a/dss-server/src/main/java/com/webank/wedatasphere/dss/server/dto/response/HomepageVideoVo.java +++ /dev/null @@ -1,78 +0,0 @@ -/* - * Copyright 2019 WeBank - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ -package com.webank.wedatasphere.dss.server.dto.response; - -/** - * Created by schumiyi on 2020/6/23 - */ -public class HomepageVideoVo { - - private Long id; - private String name; - private String url; - private String title; - private String description; - private Integer order; - - public Long getId() { - return id; - } - - public void setId(Long id) { - this.id = id; - } - - public String getName() { - return name; - } - - public void setName(String name) { - this.name = name; - } - - public String getUrl() { - return url; - } - - public void setUrl(String url) { - this.url = url; - } - - public String getTitle() { - return title; - } - - public void setTitle(String title) { - this.title = title; - } - - public String getDescription() { - return description; - } - - public void setDescription(String description) { - this.description = description; - } - - public Integer getOrder() { - return order; - } - - public void setOrder(Integer order) { - this.order = order; - } -} diff --git a/dss-server/src/main/java/com/webank/wedatasphere/dss/server/dto/response/OnestopMenuAppInstanceVo.java b/dss-server/src/main/java/com/webank/wedatasphere/dss/server/dto/response/OnestopMenuAppInstanceVo.java deleted file mode 100644 index f22ca6b337..0000000000 --- a/dss-server/src/main/java/com/webank/wedatasphere/dss/server/dto/response/OnestopMenuAppInstanceVo.java +++ /dev/null @@ -1,140 +0,0 @@ -/* - * Copyright 2019 WeBank - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ -package com.webank.wedatasphere.dss.server.dto.response; - -/** - * Created by schumiyi on 2020/6/24 - */ -public class OnestopMenuAppInstanceVo { - private Long id; - private String title; - private String description; - private String labels; - private String accessButton; - private String accessButtonUrl; - private String manualButton; - private String manualButtonUrl; - private String projectUrl; - private String name; - private Boolean isActive; - private String icon; - private Integer order; - - public Long getId() { - return id; - } - - public void setId(Long id) { - this.id = id; - } - - public String getTitle() { - return title; - } - - public void setTitle(String title) { - this.title = title; - } - - public String getDescription() { - return description; - } - - public void setDescription(String description) { - this.description = description; - } - - public String getLabels() { - return labels; - } - - public void setLabels(String labels) { - this.labels = labels; - } - - public String getAccessButton() { - return accessButton; - } - - public void setAccessButton(String accessButton) { - this.accessButton = accessButton; - } - - public String getAccessButtonUrl() { - return accessButtonUrl; - } - - public void setAccessButtonUrl(String accessButtonUrl) { - this.accessButtonUrl = accessButtonUrl; - } - - public String getManualButton() { - return manualButton; - } - - public void setManualButton(String manualButton) { - this.manualButton = manualButton; - } - - public String getManualButtonUrl() { - return manualButtonUrl; - } - - public void setManualButtonUrl(String manualButtonUrl) { - this.manualButtonUrl = manualButtonUrl; - } - - public String getIcon() { - return icon; - } - - public void setIcon(String icon) { - this.icon = icon; - } - - public Integer getOrder() { - return order; - } - - public void setOrder(Integer order) { - this.order = order; - } - - public String getProjectUrl() { - return projectUrl; - } - - public void setProjectUrl(String projectUrl) { - this.projectUrl = projectUrl; - } - - public Boolean getActive() { - return isActive; - } - - public void setActive(Boolean active) { - isActive = active; - } - - public String getName() { - return name; - } - - public void setName(String name) { - this.name = name; - } -} diff --git a/dss-server/src/main/java/com/webank/wedatasphere/dss/server/dto/response/OnestopMenuVo.java b/dss-server/src/main/java/com/webank/wedatasphere/dss/server/dto/response/OnestopMenuVo.java deleted file mode 100644 index 214b5d8539..0000000000 --- a/dss-server/src/main/java/com/webank/wedatasphere/dss/server/dto/response/OnestopMenuVo.java +++ /dev/null @@ -1,62 +0,0 @@ -/* - * Copyright 2019 WeBank - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package com.webank.wedatasphere.dss.server.dto.response; - -import java.util.List; - -/** - * Created by schumiyi on 2020/6/24 - */ -public class OnestopMenuVo { - private Long id; - private String title; - private Integer order; - private List appInstances; - - public Long getId() { - return id; - } - - public void setId(Long id) { - this.id = id; - } - - public String getTitle() { - return title; - } - - public void setTitle(String title) { - this.title = title; - } - - public Integer getOrder() { - return order; - } - - public void setOrder(Integer order) { - this.order = order; - } - - public List getAppInstances() { - return appInstances; - } - - public void setAppInstances(List appInstances) { - this.appInstances = appInstances; - } -} diff --git a/dss-server/src/main/java/com/webank/wedatasphere/dss/server/dto/response/WorkspaceDepartmentVo.java b/dss-server/src/main/java/com/webank/wedatasphere/dss/server/dto/response/WorkspaceDepartmentVo.java deleted file mode 100644 index 4fc436a33b..0000000000 --- a/dss-server/src/main/java/com/webank/wedatasphere/dss/server/dto/response/WorkspaceDepartmentVo.java +++ /dev/null @@ -1,44 +0,0 @@ -/* - * Copyright 2019 WeBank - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package com.webank.wedatasphere.dss.server.dto.response; - -/** - * Created by schumiyi on 2020/6/23 - */ -public class WorkspaceDepartmentVo { - - private Long id; - - private String name; - - public Long getId() { - return id; - } - - public void setId(Long id) { - this.id = id; - } - - public String getName() { - return name; - } - - public void setName(String name) { - this.name = name; - } -} diff --git a/dss-server/src/main/java/com/webank/wedatasphere/dss/server/dto/response/WorkspaceFavoriteVo.java b/dss-server/src/main/java/com/webank/wedatasphere/dss/server/dto/response/WorkspaceFavoriteVo.java deleted file mode 100644 index f9953d6c58..0000000000 --- a/dss-server/src/main/java/com/webank/wedatasphere/dss/server/dto/response/WorkspaceFavoriteVo.java +++ /dev/null @@ -1,82 +0,0 @@ -/* - * Copyright 2019 WeBank - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package com.webank.wedatasphere.dss.server.dto.response; - -/** - * Created by Adamyuanyuan on 2020/6/25 - */ -public class WorkspaceFavoriteVo { - private Long id; - - private Long menuApplicationId; - - private String name; - - private String url; - - private String icon; - - private String title; - - public Long getId() { - return id; - } - - public void setId(Long id) { - this.id = id; - } - public String getName() { - return name; - } - - public void setName(String name) { - this.name = name; - } - - public Long getMenuApplicationId() { - return menuApplicationId; - } - - public void setMenuApplicationId(Long menuApplicationId) { - this.menuApplicationId = menuApplicationId; - } - - public String getUrl() { - return url; - } - - public void setUrl(String url) { - this.url = url; - } - - public String getIcon() { - return icon; - } - - public void setIcon(String icon) { - this.icon = icon; - } - - public String getTitle() { - return title; - } - - public void setTitle(String title) { - this.title = title; - } -} diff --git a/dss-server/src/main/java/com/webank/wedatasphere/dss/server/entity/ApplicationArea.java b/dss-server/src/main/java/com/webank/wedatasphere/dss/server/entity/ApplicationArea.java deleted file mode 100644 index aab3600d8d..0000000000 --- a/dss-server/src/main/java/com/webank/wedatasphere/dss/server/entity/ApplicationArea.java +++ /dev/null @@ -1,53 +0,0 @@ -/* - * Copyright 2019 WeBank - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package com.webank.wedatasphere.dss.server.entity; - - -public enum ApplicationArea { - // Operational optimization, risk management, marketing recommendation, trend prediction, customer management - // 运营优化,风险管理,营销推荐,趋势预测,客户管理; - OO("运营优化","Operational Optimization",0), - RM("风险管理","Risk Management",1), - MR("营销推荐","Marketing Recommendation",2), - TP("趋势预测","Trend Prediction",3), - CM("客户管理","Customer Management",4); - - - private int id; - private String name; - private String enName; - - ApplicationArea(String name, String enName, int id) { - this.name = name; - this.enName = enName; - this.id = id; - } - - public String getName() { - return name; - } - - public int getId(){ - return this.id; - } - - public String getEnName(){ - return this.enName; - } - -} diff --git a/dss-server/src/main/java/com/webank/wedatasphere/dss/server/entity/BaseEntity.java b/dss-server/src/main/java/com/webank/wedatasphere/dss/server/entity/BaseEntity.java deleted file mode 100644 index d01380d9e2..0000000000 --- a/dss-server/src/main/java/com/webank/wedatasphere/dss/server/entity/BaseEntity.java +++ /dev/null @@ -1,49 +0,0 @@ -package com.webank.wedatasphere.dss.server.entity; - -import java.util.Date; - -/** - * Created by schumiyi on 2020/6/22 - */ -public class BaseEntity { - - private String createBy; - - private Date createTime = new Date(); - - private Date lastUpdateTime = new Date(); - - private String lastUpdateUser; - - public String getCreateBy() { - return createBy; - } - - public void setCreateBy(String createBy) { - this.createBy = createBy; - } - - public Date getCreateTime() { - return createTime; - } - - public void setCreateTime(Date createTime) { - this.createTime = createTime; - } - - public Date getLastUpdateTime() { - return lastUpdateTime; - } - - public void setLastUpdateTime(Date lastUpdateTime) { - this.lastUpdateTime = lastUpdateTime; - } - - public String getLastUpdateUser() { - return lastUpdateUser; - } - - public void setLastUpdateUser(String lastUpdateUser) { - this.lastUpdateUser = lastUpdateUser; - } -} diff --git a/dss-server/src/main/java/com/webank/wedatasphere/dss/server/entity/DSSFavorite.java b/dss-server/src/main/java/com/webank/wedatasphere/dss/server/entity/DSSFavorite.java deleted file mode 100644 index 9d9956442a..0000000000 --- a/dss-server/src/main/java/com/webank/wedatasphere/dss/server/entity/DSSFavorite.java +++ /dev/null @@ -1,73 +0,0 @@ -/* - * Copyright 2019 WeBank - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ -package com.webank.wedatasphere.dss.server.entity; - -/** - * Created by Adamyuanyuan on 2020/6/25 - */ -public class DSSFavorite extends BaseEntity{ - - private Long id; - - private String username; - - private Long workspaceId; - - private Long menuApplicationId; - - private Integer order; - - public Long getId() { - return id; - } - - public void setId(Long id) { - this.id = id; - } - - public String getUsername() { - return username; - } - - public void setUsername(String username) { - this.username = username; - } - - public Long getWorkspaceId() { - return workspaceId; - } - - public void setWorkspaceId(Long workspaceId) { - this.workspaceId = workspaceId; - } - - public Long getMenuApplicationId() { - return menuApplicationId; - } - - public void setMenuApplicationId(Long menuApplicationId) { - this.menuApplicationId = menuApplicationId; - } - - public Integer getOrder() { - return order; - } - - public void setOrder(Integer order) { - this.order = order; - } -} diff --git a/dss-server/src/main/java/com/webank/wedatasphere/dss/server/entity/DSSFlowTaxonomy.java b/dss-server/src/main/java/com/webank/wedatasphere/dss/server/entity/DSSFlowTaxonomy.java deleted file mode 100644 index 314e21ea5d..0000000000 --- a/dss-server/src/main/java/com/webank/wedatasphere/dss/server/entity/DSSFlowTaxonomy.java +++ /dev/null @@ -1,99 +0,0 @@ -/* - * Copyright 2019 WeBank - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package com.webank.wedatasphere.dss.server.entity; - -import com.webank.wedatasphere.dss.common.entity.flow.DSSFlow; - -import java.util.Date; -import java.util.List; - - -public class DSSFlowTaxonomy { - private Long id; - private String name; - private String description; - private Long creatorID; - private Date createTime; - private Date updateTime; - private Long projectID; - private List dssFlowList; - - public List getDssFlowList() { - return dssFlowList; - } - - public void setDssFlowList(List dssFlowList) { - this.dssFlowList = dssFlowList; - } - - public Long getId() { - return id; - } - - public void setId(Long id) { - this.id = id; - } - - public String getName() { - return name; - } - - public void setName(String name) { - this.name = name; - } - - public String getDescription() { - return description; - } - - public void setDescription(String description) { - this.description = description; - } - - public Long getCreatorID() { - return creatorID; - } - - public void setCreatorID(Long creatorID) { - this.creatorID = creatorID; - } - - public Date getCreateTime() { - return createTime; - } - - public void setCreateTime(Date createTime) { - this.createTime = createTime; - } - - public Date getUpdateTime() { - return updateTime; - } - - public void setUpdateTime(Date updateTime) { - this.updateTime = updateTime; - } - - public Long getProjectID() { - return projectID; - } - - public void setProjectID(Long projectID) { - this.projectID = projectID; - } -} diff --git a/dss-server/src/main/java/com/webank/wedatasphere/dss/server/entity/DSSProjectTaxonomy.java b/dss-server/src/main/java/com/webank/wedatasphere/dss/server/entity/DSSProjectTaxonomy.java deleted file mode 100644 index 302b806752..0000000000 --- a/dss-server/src/main/java/com/webank/wedatasphere/dss/server/entity/DSSProjectTaxonomy.java +++ /dev/null @@ -1,90 +0,0 @@ -/* - * Copyright 2019 WeBank - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package com.webank.wedatasphere.dss.server.entity; - -import com.webank.wedatasphere.dss.common.entity.project.DSSProject; - -import java.util.Date; -import java.util.List; - - -public class DSSProjectTaxonomy { - private Long id; - private String name; - private String description; - private Long creatorID; - private Date createTime; - private Date updateTime; - private List dssProjectList; - - public List getDssProjectList() { - return dssProjectList; - } - - public void setDssProjectList(List dssProjectList) { - this.dssProjectList = dssProjectList; - } - - public Long getId() { - return id; - } - - public void setId(Long id) { - this.id = id; - } - - public String getName() { - return name; - } - - public void setName(String name) { - this.name = name; - } - - public String getDescription() { - return description; - } - - public void setDescription(String description) { - this.description = description; - } - - public Long getCreatorID() { - return creatorID; - } - - public void setCreatorID(Long creatorID) { - this.creatorID = creatorID; - } - - public Date getCreateTime() { - return createTime; - } - - public void setCreateTime(Date createTime) { - this.createTime = createTime; - } - - public Date getUpdateTime() { - return updateTime; - } - - public void setUpdateTime(Date updateTime) { - this.updateTime = updateTime; - } -} diff --git a/dss-server/src/main/java/com/webank/wedatasphere/dss/server/entity/DSSProjectTaxonomyRelation.java b/dss-server/src/main/java/com/webank/wedatasphere/dss/server/entity/DSSProjectTaxonomyRelation.java deleted file mode 100644 index accdf3e4b1..0000000000 --- a/dss-server/src/main/java/com/webank/wedatasphere/dss/server/entity/DSSProjectTaxonomyRelation.java +++ /dev/null @@ -1,49 +0,0 @@ -/* - * Copyright 2019 WeBank - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package com.webank.wedatasphere.dss.server.entity; - -public class DSSProjectTaxonomyRelation { - private Long taxonomyId; - private Long projectId; - private Long creatorId; - - - public Long getTaxonomyId() { - return taxonomyId; - } - - public void setTaxonomyId(Long taxonomyId) { - this.taxonomyId = taxonomyId; - } - - public Long getProjectId() { - return projectId; - } - - public void setProjectId(Long projectId) { - this.projectId = projectId; - } - - public Long getCreatorId() { - return creatorId; - } - - public void setCreatorId(Long creatorId) { - this.creatorId = creatorId; - } -} diff --git a/dss-server/src/main/java/com/webank/wedatasphere/dss/server/entity/DSSWorkspace.java b/dss-server/src/main/java/com/webank/wedatasphere/dss/server/entity/DSSWorkspace.java deleted file mode 100644 index 305bffeb4c..0000000000 --- a/dss-server/src/main/java/com/webank/wedatasphere/dss/server/entity/DSSWorkspace.java +++ /dev/null @@ -1,93 +0,0 @@ -/* - * Copyright 2019 WeBank - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ -package com.webank.wedatasphere.dss.server.entity; - -/** - * Created by schumiyi on 2020/6/22 - */ -public class DSSWorkspace extends BaseEntity { - - private Long id; - - private String name; - - private String label; - - private String description; - - private String department; - - private String product; - - private String source; - - public Long getId() { - return id; - } - - public void setId(Long id) { - this.id = id; - } - - public String getName() { - return name; - } - - public void setName(String name) { - this.name = name; - } - - public String getLabel() { - return label; - } - - public void setLabel(String label) { - this.label = label; - } - - public String getDescription() { - return description; - } - - public void setDescription(String description) { - this.description = description; - } - - public String getDepartment() { - return department; - } - - public void setDepartment(String department) { - this.department = department; - } - - public String getProduct() { - return product; - } - - public void setProduct(String product) { - this.product = product; - } - - public String getSource() { - return source; - } - - public void setSource(String source) { - this.source = source; - } -} diff --git a/dss-server/src/main/java/com/webank/wedatasphere/dss/server/entity/NodeInfo.java b/dss-server/src/main/java/com/webank/wedatasphere/dss/server/entity/NodeInfo.java deleted file mode 100644 index 4ce3c66202..0000000000 --- a/dss-server/src/main/java/com/webank/wedatasphere/dss/server/entity/NodeInfo.java +++ /dev/null @@ -1,103 +0,0 @@ -/* - * Copyright 2019 WeBank - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package com.webank.wedatasphere.dss.server.entity; - - -public class NodeInfo { - private Integer id; - private String icon; - private String nodeType; - private Integer applicationId; - private Boolean submitToScheduler; //是否可以提交执行 - private Boolean enableCopy; - private Boolean shouldCreationBeforeNode; - private Boolean supportJump; - private String jumpUrl; - - public Integer getId() { - return id; - } - - public void setId(Integer id) { - this.id = id; - } - - public String getIcon() { - return icon; - } - - public void setIcon(String icon) { - this.icon = icon; - } - - public String getNodeType() { - return nodeType; - } - - public void setNodeType(String nodeType) { - this.nodeType = nodeType; - } - - public Integer getApplicationId() { - return applicationId; - } - - public void setApplicationId(Integer applicationId) { - this.applicationId = applicationId; - } - - public Boolean getSubmitToScheduler() { - return submitToScheduler; - } - - public void setSubmitToScheduler(Boolean submitToScheduler) { - this.submitToScheduler = submitToScheduler; - } - - public Boolean getEnableCopy() { - return enableCopy; - } - - public void setEnableCopy(Boolean enableCopy) { - this.enableCopy = enableCopy; - } - - public Boolean getShouldCreationBeforeNode() { - return shouldCreationBeforeNode; - } - - public void setShouldCreationBeforeNode(Boolean shouldCreationBeforeNode) { - this.shouldCreationBeforeNode = shouldCreationBeforeNode; - } - - public Boolean getSupportJump() { - return supportJump; - } - - public void setSupportJump(Boolean supportJump) { - this.supportJump = supportJump; - } - - public String getJumpUrl() { - return jumpUrl; - } - - public void setJumpUrl(String jumpUrl) { - this.jumpUrl = jumpUrl; - } -} diff --git a/dss-server/src/main/java/com/webank/wedatasphere/dss/server/function/FunctionInvoker.java b/dss-server/src/main/java/com/webank/wedatasphere/dss/server/function/FunctionInvoker.java deleted file mode 100644 index d4ef8bc4bd..0000000000 --- a/dss-server/src/main/java/com/webank/wedatasphere/dss/server/function/FunctionInvoker.java +++ /dev/null @@ -1,104 +0,0 @@ -/* - * Copyright 2019 WeBank - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package com.webank.wedatasphere.dss.server.function; - -import com.webank.wedatasphere.dss.appjoint.AppJoint; -import com.webank.wedatasphere.dss.appjoint.exception.AppJointErrorException; -import com.webank.wedatasphere.dss.appjoint.execution.core.AppJointNode; -import com.webank.wedatasphere.dss.appjoint.service.NodeService; -import com.webank.wedatasphere.dss.appjoint.service.SecurityService; -import com.webank.wedatasphere.dss.appjoint.service.session.Session; -import com.webank.wedatasphere.dss.application.entity.Application; -import com.webank.wedatasphere.dss.application.service.ApplicationService; -import com.webank.wedatasphere.dss.common.entity.project.DSSProject; -import com.webank.wedatasphere.dss.common.entity.project.Project; -import com.webank.wedatasphere.dss.server.dao.ProjectMapper; -import org.apache.commons.math3.util.Pair; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.stereotype.Component; - -import java.util.ArrayList; -import java.util.List; -import java.util.Map; - - -@Component -public class FunctionInvoker { - - @Autowired - private ApplicationService applicationService; - @Autowired - private ProjectMapper projectMapper; - - private final Logger logger = LoggerFactory.getLogger(this.getClass()); - - public Map nodeServiceFunction(String userName, Map requestBody, AppJointNode node, NodeServiceFunction function) throws AppJointErrorException { - AppJoint appJoint = applicationService.getAppjoint(node.getNodeType()); - SecurityService securityService = appJoint.getSecurityService(); - Session session = null; - Map jobContent = null; - if(securityService != null){ - logger.info("appjoint securityService is exist,{}login",userName); - session = securityService.login(userName); - } - NodeService nodeService = appJoint.getNodeService(); - if(nodeService != null){ - logger.info("appJoint NodeService is exist"); - jobContent = function.accept(nodeService,session, node, requestBody); - } - return jobContent; - } - - public List> projectServiceAddFunction(DSSProject project, ProjectServiceAddFunction projectServiceAddFunction, List appJoints) throws AppJointErrorException { - ArrayList> projects = new ArrayList<>(); - for (AppJoint appJoint : appJoints) { - Project appJointProject = null; - Session session = null; - if (appJoint.getSecurityService() != null) { - logger.info("[addProject]securityService is exist,{}login...", project.getUserName()); - session = appJoint.getSecurityService().login(project.getUserName()); - } - if (appJoint.getProjectService() != null) { - logger.info("[addProject]projectService is exist"); - appJointProject = projectServiceAddFunction.accept(appJoint.getProjectService(), project, session); - if(appJointProject != null) projects.add(new Pair(appJointProject,appJoint.getAppJointName())); - } - } - return projects; - } - - public void projectServiceFunction(DSSProject project, ProjectServiceFunction projectServiceFunction, List appJoints) throws AppJointErrorException { - for (AppJoint appJoint : appJoints) { - Session session = null; - if(appJoint.getSecurityService() != null){ - logger.info("securityService is exist,{}login..",project.getUserName()); - session = appJoint.getSecurityService().login(project.getUserName()); - } - //通过projectID获取到appjointID - Application application = applicationService.getApplication(appJoint.getAppJointName()); - Long appjointProjectID = projectMapper.getAppjointProjectID(project.getId(), application.getId()); - if(appjointProjectID != null) project.setId(appjointProjectID); - if(appJoint.getProjectService() != null){ - logger.info("projectService is exist"); - projectServiceFunction.accept(appJoint.getProjectService(),project,session); - } - } - } -} diff --git a/dss-server/src/main/java/com/webank/wedatasphere/dss/server/function/FunctionPool.java b/dss-server/src/main/java/com/webank/wedatasphere/dss/server/function/FunctionPool.java deleted file mode 100644 index 7efe477b2c..0000000000 --- a/dss-server/src/main/java/com/webank/wedatasphere/dss/server/function/FunctionPool.java +++ /dev/null @@ -1,39 +0,0 @@ -/* - * Copyright 2019 WeBank - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ -package com.webank.wedatasphere.dss.server.function; - -import com.webank.wedatasphere.dss.appjoint.execution.core.AppJointNode; -import com.webank.wedatasphere.dss.appjoint.service.NodeService; -import com.webank.wedatasphere.dss.appjoint.service.session.Session; - -import java.util.Map; - - -public class FunctionPool { - - public static NodeServiceFunction deleteNode = (NodeService nodeService, Session session, AppJointNode node, Map requestBody)->{ - node.setJobContent(requestBody); - nodeService.deleteNode(session,node); - return null; - }; - - public static NodeServiceFunction createNode = NodeService::createNode; - - public static NodeServiceFunction updateNode = NodeService::updateNode; - - // TODO: 2019/11/26 把projectService也纳入 -} diff --git a/dss-server/src/main/java/com/webank/wedatasphere/dss/server/function/NodeServiceFunction.java b/dss-server/src/main/java/com/webank/wedatasphere/dss/server/function/NodeServiceFunction.java deleted file mode 100644 index 71873f0dee..0000000000 --- a/dss-server/src/main/java/com/webank/wedatasphere/dss/server/function/NodeServiceFunction.java +++ /dev/null @@ -1,30 +0,0 @@ -/* - * Copyright 2019 WeBank - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package com.webank.wedatasphere.dss.server.function; - -import com.webank.wedatasphere.dss.appjoint.exception.AppJointErrorException; -import com.webank.wedatasphere.dss.appjoint.execution.core.AppJointNode; -import com.webank.wedatasphere.dss.appjoint.service.NodeService; -import com.webank.wedatasphere.dss.appjoint.service.session.Session; - -import java.util.Map; - - -public interface NodeServiceFunction{ - Map accept(NodeService nodeService, Session session, AppJointNode node, Map requestBody) throws AppJointErrorException; -} diff --git a/dss-server/src/main/java/com/webank/wedatasphere/dss/server/function/ProjectServiceAddFunction.java b/dss-server/src/main/java/com/webank/wedatasphere/dss/server/function/ProjectServiceAddFunction.java deleted file mode 100644 index b5eb0b3227..0000000000 --- a/dss-server/src/main/java/com/webank/wedatasphere/dss/server/function/ProjectServiceAddFunction.java +++ /dev/null @@ -1,28 +0,0 @@ -/* - * Copyright 2019 WeBank - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package com.webank.wedatasphere.dss.server.function; - -import com.webank.wedatasphere.dss.appjoint.exception.AppJointErrorException; -import com.webank.wedatasphere.dss.appjoint.service.ProjectService; -import com.webank.wedatasphere.dss.appjoint.service.session.Session; -import com.webank.wedatasphere.dss.common.entity.project.Project; - - -public interface ProjectServiceAddFunction { - Project accept(ProjectService projectService, Project project, Session session) throws AppJointErrorException; -} diff --git a/dss-server/src/main/java/com/webank/wedatasphere/dss/server/function/ProjectServiceFunction.java b/dss-server/src/main/java/com/webank/wedatasphere/dss/server/function/ProjectServiceFunction.java deleted file mode 100644 index b79e83bd46..0000000000 --- a/dss-server/src/main/java/com/webank/wedatasphere/dss/server/function/ProjectServiceFunction.java +++ /dev/null @@ -1,28 +0,0 @@ -/* - * Copyright 2019 WeBank - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package com.webank.wedatasphere.dss.server.function; - -import com.webank.wedatasphere.dss.appjoint.exception.AppJointErrorException; -import com.webank.wedatasphere.dss.appjoint.service.ProjectService; -import com.webank.wedatasphere.dss.appjoint.service.session.Session; -import com.webank.wedatasphere.dss.common.entity.project.Project; - - -public interface ProjectServiceFunction { - void accept(ProjectService projectService, Project project, Session session) throws AppJointErrorException; -} diff --git a/dss-server/src/main/java/com/webank/wedatasphere/dss/server/lock/Lock.java b/dss-server/src/main/java/com/webank/wedatasphere/dss/server/lock/Lock.java deleted file mode 100644 index 0e381b2ec3..0000000000 --- a/dss-server/src/main/java/com/webank/wedatasphere/dss/server/lock/Lock.java +++ /dev/null @@ -1,30 +0,0 @@ -/* - * Copyright 2019 WeBank - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package com.webank.wedatasphere.dss.server.lock; - -import java.lang.annotation.ElementType; -import java.lang.annotation.Retention; -import java.lang.annotation.RetentionPolicy; -import java.lang.annotation.Target; - - -@Retention(RetentionPolicy.RUNTIME) -@Target({ElementType.METHOD}) -public @interface Lock { - String type() default LockEnum.EQUAL; -} diff --git a/dss-server/src/main/java/com/webank/wedatasphere/dss/server/lock/LockAspect.java b/dss-server/src/main/java/com/webank/wedatasphere/dss/server/lock/LockAspect.java deleted file mode 100644 index 473f224c51..0000000000 --- a/dss-server/src/main/java/com/webank/wedatasphere/dss/server/lock/LockAspect.java +++ /dev/null @@ -1,110 +0,0 @@ -/* - * Copyright 2019 WeBank - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package com.webank.wedatasphere.dss.server.lock; - -import com.webank.wedatasphere.dss.server.constant.DSSServerConstant; -import com.webank.wedatasphere.dss.server.dao.ProjectMapper; -import com.webank.wedatasphere.dss.common.entity.project.DSSProjectVersion; -import com.webank.wedatasphere.dss.common.exception.DSSErrorException; -import org.apache.commons.lang.ArrayUtils; -import org.aspectj.lang.ProceedingJoinPoint; -import org.aspectj.lang.Signature; -import org.aspectj.lang.annotation.Around; -import org.aspectj.lang.annotation.Aspect; -import org.aspectj.lang.annotation.Pointcut; -import org.aspectj.lang.reflect.MethodSignature; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.core.Ordered; -import org.springframework.stereotype.Component; - -import java.lang.reflect.Method; -import java.util.List; - - -@Aspect -@Component -public class LockAspect implements Ordered { - private Logger logger = LoggerFactory.getLogger(this.getClass()); - - @Autowired - private ProjectMapper projectMapper; - - @Pointcut(value = "@annotation(com.webank.wedatasphere.dss.server.lock.Lock)") - private void cut() { - } - - @Around("cut()") - public Object around(ProceedingJoinPoint point) throws Throwable { - Signature signature = point.getSignature(); - if (!(signature instanceof MethodSignature)) { - throw new IllegalArgumentException("该注解只能用于方法"); - } - MethodSignature methodSignature = (MethodSignature) signature; - Object target = point.getTarget(); - Method currentMethod = target.getClass().getMethod(methodSignature.getName(), methodSignature.getParameterTypes()); - Lock lockAnnotation = currentMethod.getAnnotation(Lock.class); - logger.info("调用方法：" + currentMethod.getName()); - Object[] args = point.getArgs(); - String[] parameterNames = methodSignature.getParameterNames(); - int index = ArrayUtils.indexOf(parameterNames, DSSServerConstant.PROJECT_VERSION_ID); - Long projectVersionID = (Long) args[index]; - if (projectVersionID == null) { - logger.info("执行删除工程，直接通过"); - return point.proceed(); - } - logger.info("projectVersionID为：" + projectVersionID); - DSSProjectVersion dssProjectVersion = projectMapper.selectProjectVersionByID(projectVersionID); - Integer lock = dssProjectVersion.getLock(); - try { - Object proceed = point.proceed(); - judge(lockAnnotation, dssProjectVersion, lock, projectVersionID); - return proceed; - } catch (Exception e) { - logger.info("执行过程出现异常", e); - throw e; - } - } - - private void judge(Lock lockAnnotation, DSSProjectVersion dssProjectVersion, Integer lock, Long projectVersionID) throws DSSErrorException { - if (lockAnnotation.type().equals(LockEnum.ADD)) { - logger.info("projectVersion会增加"); - List dssProjectVersions = projectMapper.listProjectVersionsByProjectID(dssProjectVersion.getProjectID()); - if (dssProjectVersions.size() < 2 || !dssProjectVersions.get(1).getId().equals(projectVersionID)) { - throw new DSSErrorException(67457, "已经有别的用户对此project进行了版本更新操作，不能进行此操作！"); - } - } else { - logger.info("projectVersion不会增加"); - DSSProjectVersion latest = projectMapper.selectLatestVersionByProjectID(dssProjectVersion.getProjectID()); - if (!latest.getId().equals(projectVersionID)) { - throw new DSSErrorException(67455, "目前project版本已经不是最新，不能进行此操作！"); - } - } - Integer row = projectMapper.updateLock(lock, projectVersionID); - if (row != 1) { - throw new DSSErrorException(67456, "已经有别的用户对此project进行了更新，不能进行此操作！"); - } - } - - - @Override - public int getOrder() { - return 3; - } -} diff --git a/dss-server/src/main/java/com/webank/wedatasphere/dss/server/lock/LockEnum.java b/dss-server/src/main/java/com/webank/wedatasphere/dss/server/lock/LockEnum.java deleted file mode 100644 index 3b532d460c..0000000000 --- a/dss-server/src/main/java/com/webank/wedatasphere/dss/server/lock/LockEnum.java +++ /dev/null @@ -1,24 +0,0 @@ -/* - * Copyright 2019 WeBank - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package com.webank.wedatasphere.dss.server.lock; - - -public interface LockEnum { - String ADD = "+"; - String EQUAL = "="; -} diff --git a/dss-server/src/main/java/com/webank/wedatasphere/dss/server/operate/AppJointNodeOperate.java b/dss-server/src/main/java/com/webank/wedatasphere/dss/server/operate/AppJointNodeOperate.java deleted file mode 100644 index 276e99d3e2..0000000000 --- a/dss-server/src/main/java/com/webank/wedatasphere/dss/server/operate/AppJointNodeOperate.java +++ /dev/null @@ -1,66 +0,0 @@ -/* - * Copyright 2019 WeBank - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package com.webank.wedatasphere.dss.server.operate; - - -import com.webank.wedatasphere.dss.appjoint.exception.AppJointErrorException; -import com.webank.wedatasphere.dss.appjoint.execution.core.CommonAppJointNode; -import com.webank.wedatasphere.dss.server.function.FunctionInvoker; -import com.webank.wedatasphere.dss.server.function.FunctionPool; -import com.webank.wedatasphere.dss.server.function.NodeServiceFunction; -import com.webank.wedatasphere.dss.server.service.DSSFlowService; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.stereotype.Component; - - -@Component -public class AppJointNodeOperate implements Operate { - - @Autowired - private FunctionInvoker functionInvoker; - - @Override - public boolean canInvokeOperate(Op op) { - return !"workflow.subflow".equals(op.getNodeType()); - } - - @Override - public void add(DSSFlowService dssFlowService, Op op) throws AppJointErrorException { - invokeNodeServiceFunction(op,FunctionPool.createNode); - } - - @Override - public void update(DSSFlowService dssFlowService, Op op) throws AppJointErrorException { - invokeNodeServiceFunction(op,FunctionPool.updateNode); - } - - @Override - public void delete(DSSFlowService dssFlowService, Op op) throws AppJointErrorException { - invokeNodeServiceFunction(op, FunctionPool.deleteNode); - } - - private void invokeNodeServiceFunction(Op op,NodeServiceFunction function) throws AppJointErrorException { - String userName = op.getParams().get("userName").toString(); - Long projectID = Long.valueOf(op.getParams().get("projectID").toString()); - op.getParams().put("id",op.getId()); - CommonAppJointNode node = new CommonAppJointNode(); - node.setProjectId(projectID); - node.setNodeType(op.getNodeType()); - functionInvoker.nodeServiceFunction(userName,op.getParams(),node,function); - } -} diff --git a/dss-server/src/main/java/com/webank/wedatasphere/dss/server/operate/Op.java b/dss-server/src/main/java/com/webank/wedatasphere/dss/server/operate/Op.java deleted file mode 100644 index 6ee9c2dd58..0000000000 --- a/dss-server/src/main/java/com/webank/wedatasphere/dss/server/operate/Op.java +++ /dev/null @@ -1,60 +0,0 @@ -/* - * Copyright 2019 WeBank - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package com.webank.wedatasphere.dss.server.operate; - -import java.util.Map; - - -public class Op { - private Long id; - private String op; - private String nodeType; - private Map params; - - public Long getId() { - return id; - } - - public void setId(Long id) { - this.id = id; - } - - public String getNodeType() { - return nodeType; - } - - public void setNodeType(String nodeType) { - this.nodeType = nodeType; - } - - public String getOp() { - return op; - } - - public void setOp(String op) { - this.op = op; - } - - public Map getParams() { - return params; - } - - public void setParams(Map params) { - this.params = params; - } -} diff --git a/dss-server/src/main/java/com/webank/wedatasphere/dss/server/operate/Operate.java b/dss-server/src/main/java/com/webank/wedatasphere/dss/server/operate/Operate.java deleted file mode 100644 index ec44935c8d..0000000000 --- a/dss-server/src/main/java/com/webank/wedatasphere/dss/server/operate/Operate.java +++ /dev/null @@ -1,36 +0,0 @@ -/* - * Copyright 2019 WeBank - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package com.webank.wedatasphere.dss.server.operate; - - - -import com.webank.wedatasphere.dss.appjoint.exception.AppJointErrorException; -import com.webank.wedatasphere.dss.server.service.DSSFlowService; -import com.webank.wedatasphere.dss.common.exception.DSSErrorException; - - -public interface Operate { - - boolean canInvokeOperate(Op op); - - void add(DSSFlowService dssFlowService, Op op) throws DSSErrorException, AppJointErrorException; - - void update(DSSFlowService dssFlowService, Op op) throws DSSErrorException, AppJointErrorException; - - void delete(DSSFlowService dssFlowService, Op op) throws DSSErrorException, AppJointErrorException; -} diff --git a/dss-server/src/main/java/com/webank/wedatasphere/dss/server/operate/SubFlowOperate.java b/dss-server/src/main/java/com/webank/wedatasphere/dss/server/operate/SubFlowOperate.java deleted file mode 100644 index 8f3d28661a..0000000000 --- a/dss-server/src/main/java/com/webank/wedatasphere/dss/server/operate/SubFlowOperate.java +++ /dev/null @@ -1,73 +0,0 @@ -/* - * Copyright 2019 WeBank - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package com.webank.wedatasphere.dss.server.operate; - - -import com.webank.wedatasphere.dss.common.entity.flow.DSSFlow; -import com.webank.wedatasphere.dss.server.service.DSSFlowService; -import com.webank.wedatasphere.dss.common.exception.DSSErrorException; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.springframework.stereotype.Component; - -import java.util.Arrays; - - -@Component -public class SubFlowOperate implements Operate { - - private Logger logger = LoggerFactory.getLogger(this.getClass()); - - private String nodeType = "workflow.subflow"; - - @Override - public boolean canInvokeOperate(Op op) { - return this.nodeType.equals(op.getNodeType()); - } - - @Override - public void add(DSSFlowService dssFlowService, Op op)throws DSSErrorException { - afterOperateSubFlow(dssFlowService,op); - } - - @Override - public void update(DSSFlowService dssFlowService, Op op) throws DSSErrorException { - logger.info("name:{},description:{}",op.getParams().get("name"),op.getParams().get("description")); - DSSFlow dssFlow = new DSSFlow(); - dssFlow.setId(op.getId()); - dssFlow.setName(op.getParams().get("name").toString()); - dssFlow.setDescription(op.getParams().get("description").toString()); - dssFlowService.updateFlowBaseInfo(dssFlow, Long.valueOf(op.getParams().get("projectVersionID").toString()), null); - afterOperateSubFlow(dssFlowService,op); - } - - @Override - public void delete(DSSFlowService dssFlowService, Op op) throws DSSErrorException { - logger.info("delete subFlow{}",op.getId()); - dssFlowService.batchDeleteFlow(Arrays.asList(op.getId()), Long.valueOf(op.getParams().get("projectVersionID").toString())); - afterOperateSubFlow(dssFlowService,op); - } - - private void afterOperateSubFlow(DSSFlowService dssFlowService, Op op) throws DSSErrorException { - //更新工作流基本信息 - DSSFlow dssFlow = new DSSFlow(); - dssFlow.setId(op.getId()); - dssFlow.setHasSaved(true); - dssFlowService.updateFlowBaseInfo(dssFlow, Long.valueOf(op.getParams().get("projectVersionID").toString()), null); - } -} diff --git a/dss-server/src/main/java/com/webank/wedatasphere/dss/server/publish/PublishJob.java b/dss-server/src/main/java/com/webank/wedatasphere/dss/server/publish/PublishJob.java deleted file mode 100644 index d324b6fa34..0000000000 --- a/dss-server/src/main/java/com/webank/wedatasphere/dss/server/publish/PublishJob.java +++ /dev/null @@ -1,45 +0,0 @@ -/* - * Copyright 2019 WeBank - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package com.webank.wedatasphere.dss.server.publish; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - - -public abstract class PublishJob implements Runnable { - protected Logger logger = LoggerFactory.getLogger(this.getClass()); - protected String user; - protected Long projectVersionID; - protected PublishListner publishListner; - - public void setPublishListner(PublishListner publishListner) { - this.publishListner = publishListner; - } - - public PublishListner getPublishListner() { - return publishListner; - } - - public void setProjectVersionID(Long projectVersionID) { - this.projectVersionID = projectVersionID; - } - - public Long getProjectVersionID() { - return projectVersionID; - } -} diff --git a/dss-server/src/main/java/com/webank/wedatasphere/dss/server/publish/PublishJobEnum.java b/dss-server/src/main/java/com/webank/wedatasphere/dss/server/publish/PublishJobEnum.java deleted file mode 100644 index c971b600e1..0000000000 --- a/dss-server/src/main/java/com/webank/wedatasphere/dss/server/publish/PublishJobEnum.java +++ /dev/null @@ -1,23 +0,0 @@ -/* - * Copyright 2019 WeBank - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package com.webank.wedatasphere.dss.server.publish; - - -public enum PublishJobEnum { - Inited,Running,Succeed,Failed -} diff --git a/dss-server/src/main/java/com/webank/wedatasphere/dss/server/publish/PublishJobFactory.java b/dss-server/src/main/java/com/webank/wedatasphere/dss/server/publish/PublishJobFactory.java deleted file mode 100644 index fff4859104..0000000000 --- a/dss-server/src/main/java/com/webank/wedatasphere/dss/server/publish/PublishJobFactory.java +++ /dev/null @@ -1,47 +0,0 @@ -/* - * Copyright 2019 WeBank - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package com.webank.wedatasphere.dss.server.publish; - - -import com.webank.wedatasphere.dss.server.conf.DSSServerConf; -import com.webank.wedatasphere.dss.server.service.DSSProjectService; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.stereotype.Component; - -import java.util.concurrent.Future; - - -@Component -public class PublishJobFactory { - @Autowired - private PublishManager publishManager; - @Autowired - private DSSProjectService projectService; - - public PublishSubmitJob createSubmitPublishJob(Long projectVersionID, String user, String comment){ - PublishSubmitJob job = new PublishSubmitJob(projectService, user, comment, projectVersionID); - job.setPublishListner(publishManager); - return job; - } - - public PublishSubmitJobDeamon createSubmitPublishJobDeamon(Future future,PublishSubmitJob publishSubmitJob){ - PublishSubmitJobDeamon job = new PublishSubmitJobDeamon(future, (int) DSSServerConf.PUBLISH_TIMEOUT.getValue()); - job.setProjectVersionID(publishSubmitJob.getProjectVersionID()); - return job; - } -} diff --git a/dss-server/src/main/java/com/webank/wedatasphere/dss/server/publish/PublishListner.java b/dss-server/src/main/java/com/webank/wedatasphere/dss/server/publish/PublishListner.java deleted file mode 100644 index 8a8cc447a6..0000000000 --- a/dss-server/src/main/java/com/webank/wedatasphere/dss/server/publish/PublishListner.java +++ /dev/null @@ -1,28 +0,0 @@ -/* - * Copyright 2019 WeBank - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package com.webank.wedatasphere.dss.server.publish; - - -public interface PublishListner { - - void onPublishSucceed(Long projectVersionID, String infoMsg); - - void onPublishFailed(Long projectVersionID, String errorMsg); - - void onPublishRunning(Long projectVersionID, String infoMsg); -} diff --git a/dss-server/src/main/java/com/webank/wedatasphere/dss/server/publish/PublishManager.java b/dss-server/src/main/java/com/webank/wedatasphere/dss/server/publish/PublishManager.java deleted file mode 100644 index c1f29c4e29..0000000000 --- a/dss-server/src/main/java/com/webank/wedatasphere/dss/server/publish/PublishManager.java +++ /dev/null @@ -1,166 +0,0 @@ -/* - * Copyright 2019 WeBank - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package com.webank.wedatasphere.dss.server.publish; - - -import com.webank.wedatasphere.dss.server.service.DSSProjectService; -import com.webank.wedatasphere.dss.common.entity.project.DSSProjectPublishHistory; -import com.webank.wedatasphere.dss.common.exception.DSSErrorException; -import com.webank.wedatasphere.linkis.common.utils.Utils; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.stereotype.Component; - -import javax.annotation.PostConstruct; -import java.util.Date; -import java.util.Map; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.TimeUnit; - - -@Component -public class PublishManager implements PublishListner { - - private Logger logger = LoggerFactory.getLogger(this.getClass()); - - @Autowired - private DSSProjectService projectService; - - private Map cacheMap = new ConcurrentHashMap<>(); - - @PostConstruct - public void init() { - logger.info("定时线程开启..."); - Utils.defaultScheduler().scheduleAtFixedRate(new Runnable() { - @Override - public void run() { - synchronized (cacheMap) { - cacheMap.entrySet().stream() - .filter(f -> (f.getValue().getStatus() == PublishJobEnum.Succeed || f.getValue().getStatus() == PublishJobEnum.Failed) && f.getValue().timeout()) - .forEach(f -> {logger.info("开始remove过期记录：{},更新时间{}",f.getKey(),f.getValue().getUpdateTime());cacheMap.remove(f.getKey());}); - } - } - }, 1, 5, TimeUnit.MINUTES); - Utils.defaultScheduler().scheduleAtFixedRate(new Runnable() { - @Override - public void run() { - synchronized (cacheMap) { - cacheMap.forEach((k, v) -> { - logger.info("cache中projectVersionID为：{}状态为：{}更新时间为：{}",k,v.getStatus(),v.getUpdateTime()); - }); - } - } - }, 1, 10, TimeUnit.MINUTES); - } - - @Override - public void onPublishSucceed(Long projectVersionID, String infoMsg) { - // TODO: 2019/6/21 目前不用加锁 - logger.info(infoMsg); - Date updateTime = new Date(); - PublishProjectCache cache = cacheMap.get(projectVersionID); - cache.setUpdateTime(updateTime); - cache.setStatus(PublishJobEnum.Succeed); - projectService.updatePublishHistory(projectVersionID, PublishJobEnum.Succeed.ordinal(), updateTime); - } - - @Override - public void onPublishFailed(Long projectVersionID, String errorMsg) { - logger.info(errorMsg); - Date updateTime = new Date(); - PublishProjectCache cache = cacheMap.get(projectVersionID); - cache.setUpdateTime(updateTime); - cache.setStatus(PublishJobEnum.Failed); - cache.setMsg(errorMsg); - projectService.updatePublishHistory(projectVersionID, PublishJobEnum.Failed.ordinal(), updateTime); - } - - @Override - public void onPublishRunning(Long projectVersionID, String infoMsg) { - logger.info(infoMsg); - Date updateTime = new Date(); - PublishProjectCache cache = cacheMap.get(projectVersionID); - cache.setUpdateTime(updateTime); - cache.setStatus(PublishJobEnum.Running); - projectService.updatePublishHistory(projectVersionID, PublishJobEnum.Running.ordinal(), updateTime); - } - - public void addPublishCache(Long projectVersionID, Long creatorID, String comment) throws DSSErrorException { - if (cacheMap.get(projectVersionID) == null) { - synchronized (cacheMap) { - if (cacheMap.get(projectVersionID) == null) { - DSSProjectPublishHistory history = projectService.getPublishHistoryByID(projectVersionID); - if (history == null) { - logger.info("创建一个发布job" + projectVersionID); - projectService.createPublishHistory(comment, creatorID, projectVersionID); - } else if (PublishJobEnum.Succeed.ordinal() == history.getState()) { - throw new DSSErrorException(90008, "该工程版本已经发布成功，请不要重复发布"); - } else if (PublishJobEnum.Failed.ordinal() == history.getState()) { - logger.info("数据库中的工程发布状态为失败，运行重新发布" + projectVersionID); - projectService.updatePublishHistory(projectVersionID, PublishJobEnum.Inited.ordinal(), new Date()); - } - // TODO: 2019/6/21 数据库中running或则inited的 - PublishProjectCache newCache = new PublishProjectCache(); - newCache.setStatus(PublishJobEnum.Inited); - newCache.setCreateTime(new Date()); - newCache.setUpdateTime(new Date()); - newCache.setProjectVersionID(projectVersionID); - cacheMap.put(projectVersionID, newCache); - } else { - throw new DSSErrorException(90006, "该工程版本已经发布过，请不要重复发布"); - } - } - } else { - PublishProjectCache cache = cacheMap.get(projectVersionID); - switch (cache.getStatus()) { - case Inited: - case Running: - case Succeed: - logger.info("当前cache状态为：" + cache.getStatus().toString()); - throw new DSSErrorException(90006, "该工程版本已经发布过，请不要重复发布"); - case Failed: - synchronized (cache) { - if (cache.getStatus() == PublishJobEnum.Failed) { - logger.info("当前cache状态为：Failed，准备转换为Inited"); - cache.setStatus(PublishJobEnum.Inited); - cache.setMsg(null); - cache.setUpdateTime(new Date()); - } else { - throw new DSSErrorException(90006, "该工程版本已经发布过，请不要重复发布"); - } - } - break; - default: - logger.error("unsupport status"); - } - } - } - - public PublishProjectCache getPublishCache(Long projectVersionID) { - return cacheMap.get(projectVersionID); - } - - public void checkeIsPublishing(Long projectVersionID) throws DSSErrorException { - PublishProjectCache publishCache = getPublishCache(projectVersionID); - if(publishCache != null && publishCache.isPublishing()){ - logger.info("工程正在发布中,projectVersionID:{}，状态为{}",projectVersionID,publishCache.getStatus()); - throw new DSSErrorException(90014,"工程正在发布中，不允许执行当前操作"); - } - } -} diff --git a/dss-server/src/main/java/com/webank/wedatasphere/dss/server/publish/PublishProjectCache.java b/dss-server/src/main/java/com/webank/wedatasphere/dss/server/publish/PublishProjectCache.java deleted file mode 100644 index 3072d58225..0000000000 --- a/dss-server/src/main/java/com/webank/wedatasphere/dss/server/publish/PublishProjectCache.java +++ /dev/null @@ -1,81 +0,0 @@ -/* - * Copyright 2019 WeBank - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package com.webank.wedatasphere.dss.server.publish; - - -import com.webank.wedatasphere.dss.server.conf.DSSServerConf; - -import java.util.Date; - - -public class PublishProjectCache { - - private Long projectVersionID; - private PublishJobEnum status; - private Date createTime; - private Date updateTime; - private String msg; - - public Boolean timeout(){ - return System.currentTimeMillis() - updateTime.getTime() > (int) DSSServerConf.CACHE_TIMEOUT.getValue(); - } - - public Boolean isPublishing() { - return status != null && (status == PublishJobEnum.Inited || status == PublishJobEnum.Running); - } - - public String getMsg() { - return msg; - } - - public void setMsg(String msg) { - this.msg = msg; - } - - public PublishJobEnum getStatus() { - return status; - } - - public void setStatus(PublishJobEnum status) { - this.status = status; - } - - public Long getProjectVersionID() { - return projectVersionID; - } - - public void setProjectVersionID(Long projectVersionID) { - this.projectVersionID = projectVersionID; - } - - public Date getCreateTime() { - return createTime; - } - - public void setCreateTime(Date createTime) { - this.createTime = createTime; - } - - public Date getUpdateTime() { - return updateTime; - } - - public void setUpdateTime(Date updateTime) { - this.updateTime = updateTime; - } -} diff --git a/dss-server/src/main/java/com/webank/wedatasphere/dss/server/publish/PublishSubmitJob.java b/dss-server/src/main/java/com/webank/wedatasphere/dss/server/publish/PublishSubmitJob.java deleted file mode 100644 index af04335f5e..0000000000 --- a/dss-server/src/main/java/com/webank/wedatasphere/dss/server/publish/PublishSubmitJob.java +++ /dev/null @@ -1,50 +0,0 @@ -/* - * Copyright 2019 WeBank - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package com.webank.wedatasphere.dss.server.publish; - - -import com.webank.wedatasphere.dss.server.service.DSSProjectService; - - -public class PublishSubmitJob extends PublishJob { - - private DSSProjectService projectService; - - private String comment; - - public PublishSubmitJob(DSSProjectService projectService, String user, String comment, Long projectVersionID) { - this.projectService = projectService; - this.user = user; - this.comment = comment; - this.projectVersionID = projectVersionID; - } - - @Override - public void run() { - try { - publishListner.onPublishRunning(projectVersionID,"开始发布工程："+projectVersionID); - projectService.publish(projectVersionID, user, comment); - publishListner.onPublishSucceed(projectVersionID,"工程发布成功："+ projectVersionID); - } catch (Exception e) { - logger.info("发布失败，原因：",e); - Throwable cause = e.getCause(); - String errorMsg = cause == null?e.getMessage():cause.getMessage(); - publishListner.onPublishFailed(projectVersionID,"工程发布失败："+ errorMsg); - } - } -} diff --git a/dss-server/src/main/java/com/webank/wedatasphere/dss/server/publish/PublishSubmitJobDeamon.java b/dss-server/src/main/java/com/webank/wedatasphere/dss/server/publish/PublishSubmitJobDeamon.java deleted file mode 100644 index 1cd79736b5..0000000000 --- a/dss-server/src/main/java/com/webank/wedatasphere/dss/server/publish/PublishSubmitJobDeamon.java +++ /dev/null @@ -1,48 +0,0 @@ -/* - * Copyright 2019 WeBank - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package com.webank.wedatasphere.dss.server.publish; - - -import java.util.concurrent.Future; - - -public class PublishSubmitJobDeamon extends PublishJob { - - private Future future; - - private int time; - - public PublishSubmitJobDeamon(Future future, int time) { - this.future = future; - this.time = time; - } - - @Override - public void run() { - logger.info("projectVersionID:{}的发布守护线程启动", projectVersionID); - try { - Thread.sleep(1000L * time); - } catch (InterruptedException e) { - logger.error("deamon被打断了",e); - Thread.currentThread().interrupt(); - } - logger.info("开始取消projectVersionID:{}的发布线程", projectVersionID); - future.cancel(true); - logger.info("成功取消projectVersionID:{}的发布线程", projectVersionID); - } -} diff --git a/dss-server/src/main/java/com/webank/wedatasphere/dss/server/publish/PublishThreadPool.java b/dss-server/src/main/java/com/webank/wedatasphere/dss/server/publish/PublishThreadPool.java deleted file mode 100644 index 9cf143d6b3..0000000000 --- a/dss-server/src/main/java/com/webank/wedatasphere/dss/server/publish/PublishThreadPool.java +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright 2019 WeBank - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package com.webank.wedatasphere.dss.server.publish; - - - -import com.webank.wedatasphere.linkis.common.utils.Utils; - -import java.util.concurrent.ExecutorService; - - -public class PublishThreadPool { - - private static ExecutorService executorService = Utils.newFixedThreadPool(1000, "project-publish", false); - private static ExecutorService executorServiceDeamon = Utils.newFixedThreadPool(1000, "project-publish-deamon", true); - public static ExecutorService get(){ - return executorService; - } - public static ExecutorService getDeamon(){ - return executorServiceDeamon; - } -} diff --git a/dss-server/src/main/java/com/webank/wedatasphere/dss/server/restful/FlowRestfulApi.java b/dss-server/src/main/java/com/webank/wedatasphere/dss/server/restful/FlowRestfulApi.java deleted file mode 100644 index 102518263b..0000000000 --- a/dss-server/src/main/java/com/webank/wedatasphere/dss/server/restful/FlowRestfulApi.java +++ /dev/null @@ -1,180 +0,0 @@ -/* - * Copyright 2019 WeBank - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package com.webank.wedatasphere.dss.server.restful; - - -import com.webank.wedatasphere.dss.appjoint.exception.AppJointErrorException; -import com.webank.wedatasphere.dss.common.entity.flow.DSSFlow; -import com.webank.wedatasphere.dss.server.service.DSSFlowService; -import com.webank.wedatasphere.dss.server.service.DSSProjectService; -import com.webank.wedatasphere.dss.server.service.DSSUserService; -import com.webank.wedatasphere.dss.common.entity.flow.DSSFlowVersion; -import com.webank.wedatasphere.dss.common.exception.DSSErrorException; -import com.webank.wedatasphere.dss.server.operate.Op; -import com.webank.wedatasphere.dss.server.publish.PublishManager; -import com.webank.wedatasphere.linkis.server.Message; -import com.webank.wedatasphere.linkis.server.security.SecurityFilter; -import org.codehaus.jackson.JsonNode; -import org.codehaus.jackson.map.ObjectMapper; -import org.codehaus.jackson.type.TypeReference; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.stereotype.Component; -import org.springframework.util.StringUtils; - -import javax.servlet.http.HttpServletRequest; -import javax.ws.rs.*; -import javax.ws.rs.core.Context; -import javax.ws.rs.core.MediaType; -import javax.ws.rs.core.Response; -import java.io.IOException; -import java.util.Arrays; -import java.util.Date; -import java.util.List; - - -@Component -@Path("/dss") -@Produces(MediaType.APPLICATION_JSON) -@Consumes(MediaType.APPLICATION_JSON) -public class FlowRestfulApi { - - @Autowired - private DSSFlowService flowService; - @Autowired - private DSSUserService dssUserService; - @Autowired - private PublishManager publishManager; - - ObjectMapper mapper = new ObjectMapper(); - - @GET - @Path("/listAllFlowVersions") - public Response listAllVersions(@Context HttpServletRequest req, @QueryParam("id")Long flowID,@QueryParam("projectVersionID")Long projectVersionID) { - List versions = flowService.listAllFlowVersions(flowID,projectVersionID); - return Message.messageToResponse(Message.ok().data("versions",versions)); - } - - @POST - @Path("/addFlow") - public Response addFlow(@Context HttpServletRequest req, JsonNode json) throws DSSErrorException { - //如果是子工作流，那么分类应该是和父类一起的？ - String userName = SecurityFilter.getLoginUsername(req); - // TODO: 2019/5/23 flowName工程名下唯一校验 - String name = json.get("name").getTextValue(); - String description = json.get("description") == null?null:json.get("description").getTextValue(); - Long parentFlowID = json.get("parentFlowID") ==null?null:json.get("parentFlowID").getLongValue(); - Long taxonomyID = json.get("taxonomyID") == null? null:json.get("taxonomyID").getLongValue(); - Long projectVersionID = json.get("projectVersionID").getLongValue(); - String uses = json.get("uses") == null?null:json.get("uses").getTextValue(); - if(taxonomyID == null && parentFlowID == null) throw new DSSErrorException(90009,"请求选择工作流分类"); - publishManager.checkeIsPublishing(projectVersionID); - DSSFlow dssFlow = new DSSFlow(); - dssFlow.setProjectID(projectService.getProjectByProjectVersionID(projectVersionID).getId()); - dssFlow.setName(name); - dssFlow.setDescription(description); - dssFlow.setCreatorID(dssUserService.getUserID(userName)); - dssFlow.setCreateTime(new Date()); - dssFlow.setState(false); - dssFlow.setSource("create by user"); - dssFlow.setUses(uses); - if(parentFlowID == null){ - dssFlow.setRootFlow(true); - dssFlow.setRank(0); - dssFlow.setHasSaved(true); - dssFlow = flowService.addRootFlow(dssFlow,taxonomyID,projectVersionID); - }else { - dssFlow.setRootFlow(false); - Integer rank = flowService.getParentRank(parentFlowID); - // TODO: 2019/6/3 并发问题考虑for update - dssFlow.setRank(rank+1); - dssFlow.setHasSaved(false); - dssFlow = flowService.addSubFlow(dssFlow,parentFlowID,projectVersionID); - } - // TODO: 2019/5/16 空值校验，重复名校验 - return Message.messageToResponse(Message.ok().data("flow", dssFlow)); - } - - @POST - @Path("/updateFlowBaseInfo") - public Response updateFlowBaseInfo(@Context HttpServletRequest req, JsonNode json) throws DSSErrorException { - Long flowID = json.get("id").getLongValue(); - String name = json.get("name")==null?null:json.get("name").getTextValue(); - String description = json.get("description") == null?null:json.get("description").getTextValue(); - Long taxonomyID = json.get("taxonomyID") == null?null:json.get("taxonomyID").getLongValue(); - Long projectVersionID = json.get("projectVersionID").getLongValue(); - String uses = json.get("uses") == null?null:json.get("uses").getTextValue(); - publishManager.checkeIsPublishing(projectVersionID); - // TODO: 2019/6/13 projectVersionID的更新校验 - //这里可以不做事务 - DSSFlow dssFlow = new DSSFlow(); - dssFlow.setId(flowID); - dssFlow.setName(name); - dssFlow.setDescription(description); - dssFlow.setUses(uses); - flowService.updateFlowBaseInfo(dssFlow,projectVersionID,taxonomyID); - return Message.messageToResponse(Message.ok()); - } - - @GET - @Path("/get") - public Response get(@Context HttpServletRequest req, @QueryParam("id")Long flowID,@QueryParam("version")String version,@QueryParam("projectVersionID")Long projectVersionID) throws DSSErrorException { - // TODO: 2019/5/23 id空值判断 - DSSFlow dssFlow; - if (StringUtils.isEmpty(version)){ - dssFlow = flowService.getLatestVersionFlow(flowID,projectVersionID); - dssFlow.setFlowVersions(Arrays.asList(dssFlow.getLatestVersion())); - }else { - dssFlow = flowService.getOneVersionFlow(flowID, version,projectVersionID); - } - return Message.messageToResponse(Message.ok().data("flow", dssFlow)); - } - - @POST - @Path("/deleteFlow") - public Response deleteFlow(@Context HttpServletRequest req, JsonNode json) throws DSSErrorException { - Long flowID = json.get("id").getLongValue(); - Boolean sure = json.get("sure") == null?false:json.get("sure").getBooleanValue(); - Long projectVersionID = json.get("projectVersionID").getLongValue(); - // TODO: 2019/6/13 projectVersionID的更新校验 - //state为true代表曾经发布过 - if(flowService.getFlowByID(flowID).getState() && !sure) { - return Message.messageToResponse(Message.ok().data("warmMsg","该工作流曾经发布过，删除将会将该工作流的所有版本都删除，是否继续？")); - } - publishManager.checkeIsPublishing(projectVersionID); - flowService.batchDeleteFlow(Arrays.asList(flowID),projectVersionID); - return Message.messageToResponse(Message.ok()); - } - - @POST - @Path("/saveFlow") - public Response saveFlow(@Context HttpServletRequest req, JsonNode json) throws DSSErrorException, IOException, AppJointErrorException { - Long flowID = json.get("id").getLongValue(); - String jsonFlow = json.get("json").getTextValue(); - Long projectVersionID = json.get("projectVersionID").getLongValue(); - String userName = SecurityFilter.getLoginUsername(req); - String comment = json.get("comment") == null?"保存更新":json.get("comment").getTextValue(); - List ops = mapper.readValue(json.get("ops"), new TypeReference>(){}); - publishManager.checkeIsPublishing(projectVersionID); - String version =flowService.saveFlow(flowID,jsonFlow,comment,userName,projectVersionID,ops); - return Message.messageToResponse(Message.ok().data("flowVersion",version)); - } - - @Autowired - private DSSProjectService projectService; - -} diff --git a/dss-server/src/main/java/com/webank/wedatasphere/dss/server/restful/FlowTaxonomyRestfulApi.java b/dss-server/src/main/java/com/webank/wedatasphere/dss/server/restful/FlowTaxonomyRestfulApi.java deleted file mode 100644 index bc43c5ab68..0000000000 --- a/dss-server/src/main/java/com/webank/wedatasphere/dss/server/restful/FlowTaxonomyRestfulApi.java +++ /dev/null @@ -1,113 +0,0 @@ -/* - * Copyright 2019 WeBank - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package com.webank.wedatasphere.dss.server.restful; - - -import com.webank.wedatasphere.dss.server.service.DSSFlowTaxonomyService; -import com.webank.wedatasphere.dss.server.service.DSSProjectService; -import com.webank.wedatasphere.dss.server.service.DSSUserService; -import com.webank.wedatasphere.dss.server.entity.DSSFlowTaxonomy; -import com.webank.wedatasphere.dss.common.exception.DSSErrorException; -import com.webank.wedatasphere.dss.server.publish.PublishManager; -import com.webank.wedatasphere.linkis.server.Message; -import com.webank.wedatasphere.linkis.server.security.SecurityFilter; -import org.codehaus.jackson.JsonNode; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.stereotype.Component; - -import javax.servlet.http.HttpServletRequest; -import javax.ws.rs.Consumes; -import javax.ws.rs.POST; -import javax.ws.rs.Path; -import javax.ws.rs.Produces; -import javax.ws.rs.core.Context; -import javax.ws.rs.core.MediaType; -import javax.ws.rs.core.Response; -import java.util.Date; - - -@Component -@Path("/dss") -@Produces(MediaType.APPLICATION_JSON) -@Consumes(MediaType.APPLICATION_JSON) -public class FlowTaxonomyRestfulApi { - - @Autowired - private DSSFlowTaxonomyService flowTaxonomyService; - @Autowired - private DSSUserService dssUserService; - @Autowired - private DSSProjectService projectService; - @Autowired - private PublishManager publishManager; - - @POST - @Path("/addFlowTaxonomy") - public Response addFlowTaxonomy(@Context HttpServletRequest req, JsonNode json) throws DSSErrorException { - String userName = SecurityFilter.getLoginUsername(req); - String name = json.get("name").getTextValue(); - String description = json.get("description").getTextValue(); - Long projectVersionID = json.get("projectVersionID").getLongValue(); - publishManager.checkeIsPublishing(projectVersionID); - // TODO: 2019/5/16 空值校验，重复名校验 - DSSFlowTaxonomy dssFlowTaxonomy = new DSSFlowTaxonomy(); - Date date = new Date(); - dssFlowTaxonomy.setName(name); - dssFlowTaxonomy.setDescription(description); - dssFlowTaxonomy.setCreatorID(dssUserService.getUserID(userName)); - dssFlowTaxonomy.setCreateTime(date); - dssFlowTaxonomy.setUpdateTime(date); - dssFlowTaxonomy.setProjectID(projectService.getProjectByProjectVersionID(projectVersionID).getId()); - flowTaxonomyService.addFlowTaxonomy(dssFlowTaxonomy,projectVersionID); - return Message.messageToResponse(Message.ok()); - } - @POST - @Path("/updateFlowTaxonomy") - public Response updateFlowTaxonomy(@Context HttpServletRequest req, JsonNode json) throws DSSErrorException { - String name = json.get("name")==null?null:json.get("name").getTextValue(); - String description = json.get("description") == null?null:json.get("description").getTextValue(); - Long id = json.get("id").getLongValue(); - Long projectVersionID = json.get("projectVersionID").getLongValue(); - publishManager.checkeIsPublishing(projectVersionID); - // TODO: 2019/6/13 projectVersionID的更新校验 - // TODO: 2019/5/16 空值校验，重复名校验 - DSSFlowTaxonomy dssFlowTaxonomy = new DSSFlowTaxonomy(); - dssFlowTaxonomy.setId(id); - dssFlowTaxonomy.setName(name); - dssFlowTaxonomy.setDescription(description); - dssFlowTaxonomy.setUpdateTime(new Date()); - flowTaxonomyService.updateFlowTaxonomy(dssFlowTaxonomy,projectVersionID); - return Message.messageToResponse(Message.ok()); - } - - @POST - @Path("/deleteFlowTaxonomy") - public Response deleteFlowTaxonomy(@Context HttpServletRequest req, JsonNode json) throws DSSErrorException { - Long id = json.get("id").getLongValue(); - // TODO: 2019/5/16 工程分类下工程是不是空的校验 - // TODO: 2019/5/16 如果删除的是我的工程，我参与的工程 - Long projectVersionID = json.get("projectVersionID").getLongValue(); - publishManager.checkeIsPublishing(projectVersionID); - // TODO: 2019/6/13 projectVersionID的更新校验 - if(flowTaxonomyService.hasFlows(id)){ - throw new DSSErrorException(90001,"该工作流分类下还有工作流，不能删除该分类！"); - } - flowTaxonomyService.deleteFlowTaxonomy(id,projectVersionID); - return Message.messageToResponse(Message.ok()); - } -} diff --git a/dss-server/src/main/java/com/webank/wedatasphere/dss/server/restful/NodeRestfulApi.java b/dss-server/src/main/java/com/webank/wedatasphere/dss/server/restful/NodeRestfulApi.java deleted file mode 100644 index ec1c43a228..0000000000 --- a/dss-server/src/main/java/com/webank/wedatasphere/dss/server/restful/NodeRestfulApi.java +++ /dev/null @@ -1,115 +0,0 @@ -/* - * Copyright 2019 WeBank - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package com.webank.wedatasphere.dss.server.restful; - - -import com.webank.wedatasphere.dss.appjoint.exception.AppJointErrorException; -import com.webank.wedatasphere.dss.appjoint.execution.core.CommonAppJointNode; -import com.webank.wedatasphere.dss.application.entity.Application; -import com.webank.wedatasphere.dss.application.service.ApplicationService; -import com.webank.wedatasphere.dss.application.util.ApplicationUtils; -import com.webank.wedatasphere.dss.server.entity.NodeInfo; -import com.webank.wedatasphere.dss.server.function.FunctionInvoker; -import com.webank.wedatasphere.dss.server.function.FunctionPool; -import com.webank.wedatasphere.dss.server.function.NodeServiceFunction; -import com.webank.wedatasphere.dss.server.service.DSSNodeInfoService; -import com.webank.wedatasphere.linkis.server.Message; -import com.webank.wedatasphere.linkis.server.security.SecurityFilter; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.stereotype.Component; - -import javax.servlet.http.HttpServletRequest; -import javax.ws.rs.*; -import javax.ws.rs.core.Context; -import javax.ws.rs.core.MediaType; -import javax.ws.rs.core.Response; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - - -@Component -@Path("/dss") -@Produces(MediaType.APPLICATION_JSON) -@Consumes(MediaType.APPLICATION_JSON) -public class NodeRestfulApi { - private Logger logger = LoggerFactory.getLogger(this.getClass()); - - @Autowired - private DSSNodeInfoService nodeInfoService; - - @Autowired - private ApplicationService applicationService; - - @Autowired - private FunctionInvoker functionInvoker; - - @GET - @Path("/listNodeType") - public Response listNodeType(@Context HttpServletRequest req, @QueryParam("applicationName")List names) { - HashMap> nodeTypes = new HashMap<>(); - if(names == null || names.isEmpty()){ - names = applicationService.listApplicationNames(); - } - names.forEach(f ->{ - //如果redirecturl不为ull的话，返回固定格式，并且进行url加密 - Application application = applicationService.getApplication(f); - List nodeInfo = nodeInfoService.getNodeType(f); - String redirectUrl = application.getRedirectUrl(); - if(redirectUrl != null){ - nodeInfo.forEach(n ->n.setJumpUrl(ApplicationUtils.redirectUrlFormat(redirectUrl,n.getJumpUrl()))); - } - nodeTypes.put(f,nodeInfo); - }); - return Message.messageToResponse(Message.ok().data("nodeTypes",nodeTypes)); - } - - @POST - @Path("/createAppjointNode") - public Response createExternalNode(@Context HttpServletRequest req,Map json) throws AppJointErrorException { - return createNodeResponse(req,json, FunctionPool.createNode); - } - - private Response createNodeResponse(HttpServletRequest req, Map json, NodeServiceFunction function) throws AppJointErrorException { - String userName = SecurityFilter.getLoginUsername(req); - Long projectID = Long.parseLong(json.get("id").toString()); - String nodeType = json.get("nodeType").toString(); - Map params = (Map)json.get("params"); - logger.info("CreateExternalNode request params is "+ params+",nodeType:"+nodeType); - CommonAppJointNode node = new CommonAppJointNode(); - node.setProjectId(projectID); - node.setNodeType(nodeType); - Map jobContent = functionInvoker.nodeServiceFunction(userName,params,node,function); - return Message.messageToResponse(Message.ok().data("result",jobContent)); - } - - @POST - @Path("/updateAppjointNode") - public Response updateExternalNode(@Context HttpServletRequest req,Map json) throws AppJointErrorException { - return createNodeResponse(req,json, FunctionPool.updateNode); - } - - @POST - @Path("/deleteAppjointNode") - public Response deleteExternalNode(@Context HttpServletRequest req,Map json) throws AppJointErrorException { - return createNodeResponse(req,json, FunctionPool.deleteNode); - } -} - diff --git a/dss-server/src/main/java/com/webank/wedatasphere/dss/server/restful/ProjectRestfulApi.java b/dss-server/src/main/java/com/webank/wedatasphere/dss/server/restful/ProjectRestfulApi.java deleted file mode 100644 index be22ec7582..0000000000 --- a/dss-server/src/main/java/com/webank/wedatasphere/dss/server/restful/ProjectRestfulApi.java +++ /dev/null @@ -1,201 +0,0 @@ -/* - * Copyright 2019 WeBank - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package com.webank.wedatasphere.dss.server.restful; - - -import com.webank.wedatasphere.dss.appjoint.exception.AppJointErrorException; -import com.webank.wedatasphere.dss.common.entity.project.DSSProject; -import com.webank.wedatasphere.dss.common.entity.project.DSSProjectVersion; -import com.webank.wedatasphere.dss.common.exception.DSSErrorException; -import com.webank.wedatasphere.dss.server.dao.DSSUserMapper; -import com.webank.wedatasphere.dss.server.dao.ProjectMapper; -import com.webank.wedatasphere.dss.server.entity.ApplicationArea; -import com.webank.wedatasphere.dss.server.publish.*; -import com.webank.wedatasphere.dss.server.service.DSSProjectService; -import com.webank.wedatasphere.dss.server.service.DSSProjectTaxonomyService; -import com.webank.wedatasphere.dss.server.service.DSSUserService; -import com.webank.wedatasphere.linkis.server.Message; -import com.webank.wedatasphere.linkis.server.security.SecurityFilter; -import org.codehaus.jackson.JsonNode; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.stereotype.Component; - -import javax.servlet.http.HttpServletRequest; -import javax.ws.rs.*; -import javax.ws.rs.core.Context; -import javax.ws.rs.core.MediaType; -import javax.ws.rs.core.Response; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; -import java.util.concurrent.Future; - - -@Component -@Path("/dss") -@Produces(MediaType.APPLICATION_JSON) -@Consumes(MediaType.APPLICATION_JSON) -public class ProjectRestfulApi { - - @Autowired - private DSSProjectTaxonomyService projectTaxonomyService; - @Autowired - private DSSProjectService projectService; - @Autowired - private DSSUserService dssUserService; - @Autowired - private ProjectMapper projectMapper; - @Autowired - private PublishJobFactory publishJobFactory; - @Autowired - private PublishManager publishManager; - - @Autowired - private DSSUserMapper dssUserMapper; - - @GET - @Path("/listAllProjectVersions") - public Response listAllVersions(@Context HttpServletRequest req, @QueryParam("id") Long projectID) { - List versions = projectService.listAllProjectVersions(projectID); - return Message.messageToResponse(Message.ok().data("versions", versions)); - } - - @GET - @Path("/listApplicationAreas") - public Response listApplicationAreas(@Context HttpServletRequest req) { - String header = req.getHeader("Content-language").trim(); - ApplicationArea[] applicationAreas = ApplicationArea.values(); - List areas = new ArrayList<>(); - Arrays.stream(applicationAreas).forEach(item ->{ - if ("zh-CN".equals(header)){ - areas.add( item.getName()); - }else { - areas.add(item.getEnName()); - } - }); - return Message.messageToResponse(Message.ok().data("applicationAreas", areas)); - } - - @POST - @Path("/addProject") - public Response addProject(@Context HttpServletRequest req, JsonNode json) throws DSSErrorException,AppJointErrorException { - String userName = SecurityFilter.getLoginUsername(req); - String name = json.get("name").getTextValue(); - String description = json.get("description").getTextValue(); - Long taxonomyID = json.get("taxonomyID") == null? null:json.get("taxonomyID").getLongValue(); - String product = json.get("product").getTextValue(); - Integer applicationArea = json.get("applicationArea").getIntValue(); - String business = json.get("business").getTextValue(); - Long workspaceId = json.get("workspaceId") == null ? 1 : json.get("workspaceId").getLongValue(); - // TODO: 2019/5/16 空值校验，重复名校验 - projectService.addProject(userName, name, description, taxonomyID,product,applicationArea,business, workspaceId); - return Message.messageToResponse(Message.ok()); - } - - @POST - @Path("/updateProject") - public Response updateProject(@Context HttpServletRequest req, JsonNode json) throws AppJointErrorException { - String userName = SecurityFilter.getLoginUsername(req); - String name = json.get("name") == null ? null : json.get("name").getTextValue(); - String description = json.get("description") == null ? null : json.get("description").getTextValue(); - long projectID = json.get("id").getLongValue(); - Long taxonomyID = json.get("taxonomyID") == null ? null : json.get("taxonomyID").getLongValue(); - String product = json.get("product").getTextValue(); - Integer applicationArea = json.get("applicationArea").getIntValue(); - String business = json.get("business").getTextValue(); - //这里可以不做事务 - projectService.updateProject(projectID, name, description, userName , product ,applicationArea ,business); - projectTaxonomyService.updateProjectTaxonomyRelation(projectID, taxonomyID); - return Message.messageToResponse(Message.ok()); - } - - @POST - @Path("/deleteProject") - public Response deleteProject(@Context HttpServletRequest req, JsonNode json) throws DSSErrorException { - String userName = SecurityFilter.getLoginUsername(req); - Long projectID = json.get("id").getLongValue(); - Boolean sure = json.get("sure") != null&&json.get("sure").getBooleanValue(); - Boolean ifDelScheduler = json.get("ifDelScheduler") != null &&json.get("ifDelScheduler").getBooleanValue(); - if (projectService.isPublished(projectID) && !sure) { - return Message.messageToResponse(Message.ok().data("warmMsg", "该工程发布过，是否继续？")); - } - // TODO: 2019/5/23 阻止projectID为-1 - projectService.deleteProject(projectID, userName,ifDelScheduler); - return Message.messageToResponse(Message.ok()); - } - - @POST - @Path("/copyProjectAndFlow") - public Response copyProject(@Context HttpServletRequest req, JsonNode json) throws InterruptedException, DSSErrorException, AppJointErrorException { - String userName = SecurityFilter.getLoginUsername(req); - Long projectID = json.get("projectID").getLongValue(); - String projectName = json.get("projectName") == null ? null : json.get("projectName").getTextValue(); - DSSProjectVersion maxVersion = projectMapper.selectLatestVersionByProjectID(projectID); - projectService.copyProject( maxVersion.getId(),projectID, projectName, userName); - return Message.messageToResponse(Message.ok()); - } - - @POST - @Path("/copyVersionAndFlow") - public Response copyProjectVersion(@Context HttpServletRequest req, JsonNode json) throws InterruptedException, DSSErrorException { - String userName = SecurityFilter.getLoginUsername(req); - Long copyprojectVersionID = json.get("projectVersionID").getLongValue(); - DSSProjectVersion currentVersion = projectMapper.selectProjectVersionByID(copyprojectVersionID); - DSSProjectVersion maxVersion = projectMapper.selectLatestVersionByProjectID(currentVersion.getProjectID()); - - projectService.copyProjectVersionMax( maxVersion.getId(), maxVersion,currentVersion,userName,null); - return Message.messageToResponse(Message.ok()); - } - - @POST - @Path("/publish") - public Response publish(@Context HttpServletRequest req, JsonNode json) throws DSSErrorException { - String userName = SecurityFilter.getLoginUsername(req); - Long projectID = json.get("id").getLongValue(); - String comment = json.get("comment").getTextValue(); - DSSProject latestVersionProject = projectService.getLatestVersionProject(projectID); - publishManager.addPublishCache(latestVersionProject.getLatestVersion().getId(), dssUserService.getUserID(userName),comment); - PublishSubmitJob job = publishJobFactory.createSubmitPublishJob(latestVersionProject.getLatestVersion().getId(), userName, comment); - Future submit = PublishThreadPool.get().submit(job); - PublishSubmitJobDeamon deamon = publishJobFactory.createSubmitPublishJobDeamon(submit, job); - PublishThreadPool.getDeamon().execute(deamon); - return Message.messageToResponse(Message.ok()); - } - - @GET - @Path("/publishQuery") - public Response publishQuery(@Context HttpServletRequest req, @QueryParam("projectVersionID") Long projectVersionID) { - PublishProjectCache cache = publishManager.getPublishCache(projectVersionID); - return Message.messageToResponse(Message.ok().data("info",cache)); - } - - @GET - @Path("/getAppJointProjectID") - public Response getAppjointProjectID(@Context HttpServletRequest req, @QueryParam("projectID") Long projectID,@QueryParam("nodeType") String nodeType) { - Long appJointProjectID = projectService.getAppjointProjectID(projectID,nodeType); - return Message.messageToResponse(Message.ok().data("appJointProjectID",appJointProjectID)); - } - - @GET - @Path("/getAppjointProjectIDByApplicationName") - public Response listAppJointProjectID(@Context HttpServletRequest req, @QueryParam("projectID") Long projectID,@QueryParam("applicationName") String applicationName) { - Long appJointProjectID = projectService.getAppjointProjectIDByApplicationName(projectID,applicationName); - return Message.messageToResponse(Message.ok().data("appJointProjectID",appJointProjectID)); - } - -} diff --git a/dss-server/src/main/java/com/webank/wedatasphere/dss/server/restful/ProjectTaxonomyRestfulApi.java b/dss-server/src/main/java/com/webank/wedatasphere/dss/server/restful/ProjectTaxonomyRestfulApi.java deleted file mode 100644 index 4852c0fe9b..0000000000 --- a/dss-server/src/main/java/com/webank/wedatasphere/dss/server/restful/ProjectTaxonomyRestfulApi.java +++ /dev/null @@ -1,98 +0,0 @@ -/* - * Copyright 2019 WeBank - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package com.webank.wedatasphere.dss.server.restful; - - -import com.webank.wedatasphere.dss.server.service.DSSProjectTaxonomyService; -import com.webank.wedatasphere.dss.server.service.DSSUserService; -import com.webank.wedatasphere.dss.server.entity.DSSProjectTaxonomy; -import com.webank.wedatasphere.dss.common.exception.DSSErrorException; -import com.webank.wedatasphere.linkis.server.Message; -import com.webank.wedatasphere.linkis.server.security.SecurityFilter; -import org.codehaus.jackson.JsonNode; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.stereotype.Component; - -import javax.servlet.http.HttpServletRequest; -import javax.ws.rs.Consumes; -import javax.ws.rs.POST; -import javax.ws.rs.Path; -import javax.ws.rs.Produces; -import javax.ws.rs.core.Context; -import javax.ws.rs.core.MediaType; -import javax.ws.rs.core.Response; -import java.util.Date; - - -@Component -@Path("/dss") -@Produces(MediaType.APPLICATION_JSON) -@Consumes(MediaType.APPLICATION_JSON) -public class ProjectTaxonomyRestfulApi { - - @Autowired - private DSSProjectTaxonomyService projectTaxonomyService; - @Autowired - private DSSUserService dssUserService; - - @POST - @Path("/addProjectTaxonomy") - public Response addProjectTaxonomy(@Context HttpServletRequest req, JsonNode json) throws DSSErrorException { - String userName = SecurityFilter.getLoginUsername(req); - String name = json.get("name").getTextValue(); - String description = json.get("description").getTextValue(); - // TODO: 2019/5/16 空值校验，重复名校验 - DSSProjectTaxonomy dssProjectTaxonomy = new DSSProjectTaxonomy(); - Date date = new Date(); - dssProjectTaxonomy.setName(name); - dssProjectTaxonomy.setDescription(description); - dssProjectTaxonomy.setCreatorID(dssUserService.getUserID(userName)); - dssProjectTaxonomy.setCreateTime(date); - dssProjectTaxonomy.setUpdateTime(date); - projectTaxonomyService.addProjectTaxonomy(dssProjectTaxonomy); - return Message.messageToResponse(Message.ok()); - } - @POST - @Path("/updateProjectTaxonomy") - public Response updateProjectTaxonomy(@Context HttpServletRequest req, JsonNode json) throws DSSErrorException { - String name = json.get("name")==null?null:json.get("name").getTextValue(); - String description = json.get("description") == null?null:json.get("description").getTextValue(); - Long id = json.get("id").getLongValue(); - // TODO: 2019/5/16 空值校验，重复名校验 - DSSProjectTaxonomy dssProjectTaxonomy = new DSSProjectTaxonomy(); - dssProjectTaxonomy.setId(id); - dssProjectTaxonomy.setName(name); - dssProjectTaxonomy.setDescription(description); - dssProjectTaxonomy.setUpdateTime(new Date()); - projectTaxonomyService.updateProjectTaxonomy(dssProjectTaxonomy); - return Message.messageToResponse(Message.ok()); - } - - @POST - @Path("/deleteProjectTaxonomy") - public Response deleteProjectTaxonomy(@Context HttpServletRequest req, JsonNode json) throws DSSErrorException { - Long id = json.get("id").getLongValue(); - // TODO: 2019/5/16 工程分类下工程是不是空的校验 - // TODO: 2019/5/16 如果删除的是我的工程，我参与的工程 - if(projectTaxonomyService.hasProjects(id)){ - throw new DSSErrorException(60000,"该工程分类下还有工程，不能删除该分类！"); - } - projectTaxonomyService.deleteProjectTaxonomy(id); - return Message.messageToResponse(Message.ok()); - } -} diff --git a/dss-server/src/main/java/com/webank/wedatasphere/dss/server/restful/TreeRestfulApi.java b/dss-server/src/main/java/com/webank/wedatasphere/dss/server/restful/TreeRestfulApi.java deleted file mode 100644 index fde36296e8..0000000000 --- a/dss-server/src/main/java/com/webank/wedatasphere/dss/server/restful/TreeRestfulApi.java +++ /dev/null @@ -1,59 +0,0 @@ -/* - * Copyright 2019 WeBank - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package com.webank.wedatasphere.dss.server.restful; - - - -import com.webank.wedatasphere.dss.server.crumb.CrumbFactory; -import com.webank.wedatasphere.dss.server.crumb.QuerParamsParser; -import com.webank.wedatasphere.dss.server.entity.Crumb; -import com.webank.wedatasphere.linkis.server.Message; -import com.webank.wedatasphere.linkis.server.security.SecurityFilter; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.stereotype.Component; -import org.springframework.web.bind.annotation.RequestParam; -import scala.Enumeration; - -import javax.servlet.http.HttpServletRequest; -import javax.ws.rs.*; -import javax.ws.rs.core.Context; -import javax.ws.rs.core.MediaType; -import javax.ws.rs.core.Response; - - -@Component -@Path("/dss") -@Produces(MediaType.APPLICATION_JSON) -@Consumes(MediaType.APPLICATION_JSON) -public class TreeRestfulApi { - @Autowired - private CrumbFactory crumbFactory; - - @GET - @Path("/tree") - public Response Crumb(@Context HttpServletRequest req) { - String userName = SecurityFilter.getLoginUsername(req); - String queryString = req.getQueryString() == null ? "" : req.getQueryString(); - String[] params = new String[]{} ; - if (!"".equals(queryString)) params = QuerParamsParser.toArray(queryString); - Enumeration.Value crumbType = QuerParamsParser.getCrumbType(queryString); - Crumb[] crumbs = crumbFactory.createCrumbs(crumbType, params); - Object crumbData = crumbFactory.createCrumbData(crumbType, QuerParamsParser.toMap(params),userName); - return Message.messageToResponse(Message.ok().data("crumbs", crumbs).data("data",crumbData)); - } -} diff --git a/dss-server/src/main/java/com/webank/wedatasphere/dss/server/restful/UserManagerApi.java b/dss-server/src/main/java/com/webank/wedatasphere/dss/server/restful/UserManagerApi.java deleted file mode 100644 index a249e7ae3b..0000000000 --- a/dss-server/src/main/java/com/webank/wedatasphere/dss/server/restful/UserManagerApi.java +++ /dev/null @@ -1,92 +0,0 @@ -package com.webank.wedatasphere.dss.server.restful; - -import com.webank.wedatasphere.dss.appjoint.exception.AppJointErrorException; -import com.webank.wedatasphere.dss.appjoint.scheduler.SchedulerAppJoint; -import com.webank.wedatasphere.dss.application.conf.ApplicationConf; -import com.webank.wedatasphere.dss.application.service.ApplicationService; -import com.webank.wedatasphere.dss.server.constant.DSSServerConstant; -import com.webank.wedatasphere.linkis.server.Message; -import com.webank.wedatasphere.linkis.server.security.SecurityFilter; -import com.webank.wedatasphpere.dss.user.dto.request.AuthorizationBody; -import com.webank.wedatasphpere.dss.user.service.AbsCommand; -import com.webank.wedatasphpere.dss.user.service.impl.UserAuthorizationClient; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.stereotype.Component; - -import javax.servlet.http.HttpServletRequest; -import javax.ws.rs.*; -import javax.ws.rs.core.Context; -import javax.ws.rs.core.MediaType; -import javax.ws.rs.core.Response; - -/** - * @program: user-authorization - * @description: 鲁班对外交互的接口 包括施工 注册用户 - * @create: 2020-08-12 14:24 - **/ - -@Component -@Path("/dss") -@Produces(MediaType.APPLICATION_JSON) -@Consumes(MediaType.APPLICATION_JSON) -public class UserManagerApi { - - private UserAuthorizationClient client = new UserAuthorizationClient(); - private Logger logger = LoggerFactory.getLogger(this.getClass()); - - @Autowired - private ApplicationService applicationService; - - private SchedulerAppJoint schedulerAppJoint; - - @POST - @Path("/user") - public Response createUser(@Context HttpServletRequest req, AuthorizationBody body) { - String username = SecurityFilter.getLoginUsername(req); - String superUserName = ApplicationConf.SUPER_USER_NAME; - if(!username.equals(superUserName)){ - return Message.messageToResponse(Message.error(DSSServerConstant.SUPER_USER_LOGIN_ERROR)); - } - - try { - String result = client.authorization(body); - - if(result.equals(AbsCommand.SUCCESS)){ - schedulerAppJoint = getSchedulerAppJoint(); - if(schedulerAppJoint != null){ - try{ - schedulerAppJoint.getSecurityService().reloadToken(); - }catch (Throwable throwable){ - logger.warn("choose schedulies,don not care"); - } - - } - return Message.messageToResponse(Message.ok()); - }else { - return Message.messageToResponse(Message.error(AbsCommand.ERROR)); - } - } catch (Exception e) { - return Message.messageToResponse(Message.error(e.getMessage())); - } - - - - - } - - private SchedulerAppJoint getSchedulerAppJoint(){ - - if(schedulerAppJoint == null){ - try { - schedulerAppJoint = (SchedulerAppJoint)applicationService.getAppjoint("schedulis"); - } catch (AppJointErrorException e) { - logger.error("Schedule system init failed!", e); - } - } - return schedulerAppJoint; - } - - -} diff --git a/dss-server/src/main/java/com/webank/wedatasphere/dss/server/restful/WorkspaceRestfulApi.java b/dss-server/src/main/java/com/webank/wedatasphere/dss/server/restful/WorkspaceRestfulApi.java deleted file mode 100644 index 4f85a55a4d..0000000000 --- a/dss-server/src/main/java/com/webank/wedatasphere/dss/server/restful/WorkspaceRestfulApi.java +++ /dev/null @@ -1,172 +0,0 @@ -/* - * Copyright 2019 WeBank - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package com.webank.wedatasphere.dss.server.restful; - -import com.webank.wedatasphere.dss.server.dto.response.HomepageDemoMenuVo; -import com.webank.wedatasphere.dss.server.dto.response.HomepageVideoVo; -import com.webank.wedatasphere.dss.server.dto.response.OnestopMenuVo; -import com.webank.wedatasphere.dss.server.entity.DSSWorkspace; -import com.webank.wedatasphere.dss.server.dto.response.WorkspaceDepartmentVo; -import com.webank.wedatasphere.dss.server.dto.response.*; -import com.webank.wedatasphere.dss.server.service.DSSUserService; -import com.webank.wedatasphere.dss.server.service.DSSWorkspaceService; -import com.webank.wedatasphere.linkis.server.Message; -import com.webank.wedatasphere.linkis.server.security.SecurityFilter; -import org.codehaus.jackson.JsonNode; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.stereotype.Component; - -import javax.servlet.http.HttpServletRequest; -import javax.ws.rs.*; -import javax.ws.rs.core.Context; -import javax.ws.rs.core.MediaType; -import javax.ws.rs.core.Response; -import java.util.List; - -/** - * Created by schumiyi on 2020/6/19 - */ -@Component -@Path("/dss") -@Produces(MediaType.APPLICATION_JSON) -@Consumes(MediaType.APPLICATION_JSON) -public class WorkspaceRestfulApi { - - @Autowired - private DSSWorkspaceService dssWorkspaceService; - - @Autowired - private DSSUserService dssUserService; - - @GET - @Path("/workspaces") - public Response getAllWorkspaces(@Context HttpServletRequest req) { - // TODO: Order By time - List workspaces = dssWorkspaceService.getWorkspaces(); - return Message.messageToResponse(Message.ok().data("workspaces", workspaces)); - } - - @GET - @Path("/workspaces/{id}") - public Response getWorkspacesById(@Context HttpServletRequest req, @PathParam("id") Long id) { - DSSWorkspace workspace = dssWorkspaceService.getWorkspacesById(id); - return Message.messageToResponse(Message.ok().data("workspace", workspace)); - } - - @GET - @Path("/workspaces/departments") - public Response getAllWorkspaceDepartments(@Context HttpServletRequest req) { - List departments = dssWorkspaceService.getWorkSpaceDepartments(); - return Message.messageToResponse(Message.ok().data("departments", departments)); - } - - @GET - @Path("/workspaces/exists") - public Response getUsernameExistence(@Context HttpServletRequest req, @QueryParam("name") String name) { - boolean exists = dssWorkspaceService.existWorkspaceName(name); - return Message.messageToResponse(Message.ok().data("workspaceNameExists", exists)); - } - - @POST - @Path("/workspaces") - public Response addWorkspace(@Context HttpServletRequest req, JsonNode json) { - String userName = SecurityFilter.getLoginUsername(req); - String name = json.get("name").getTextValue(); - if (dssWorkspaceService.existWorkspaceName(name)) { - return Message.messageToResponse(Message.error("工作空间名重复")); - } - String department = json.get("department").getTextValue(); - String label = json.get("label").getTextValue(); - String description = json.get("description").getTextValue(); - Long workspaceId = dssWorkspaceService.addWorkspace(userName, name, department, label, description); - return Message.messageToResponse(Message.ok().data("workspaceId", workspaceId)); - } - - @GET - @Path("/workspaces/demos") - public Response getAllHomepageDemos(@Context HttpServletRequest req) { - String header = req.getHeader("Content-language").trim(); - boolean isChinese = "zh-CN".equals(header); - List homepageDemos = dssWorkspaceService.getHomepageDemos(isChinese); - return Message.messageToResponse(Message.ok().data("demos", homepageDemos)); - } - - @GET - @Path("/workspaces/videos") - public Response getAllVideos(@Context HttpServletRequest req) { - String header = req.getHeader("Content-language").trim(); - boolean isChinese = "zh-CN".equals(header); - List homepageVideos = dssWorkspaceService.getHomepageVideos(isChinese); - return Message.messageToResponse(Message.ok().data("videos", homepageVideos)); - } - - @GET - @Path("workspaces/{workspaceId}/managements") - public Response getWorkspaceManagements(@Context HttpServletRequest req, @PathParam("workspaceId") Long workspaceId) { - String header = req.getHeader("Content-language").trim(); - boolean isChinese = "zh-CN".equals(header); - String username = SecurityFilter.getLoginUsername(req); - - List managements = dssWorkspaceService.getWorkspaceManagements(workspaceId, username, isChinese); - return Message.messageToResponse(Message.ok().data("managements", managements)); - } - - @GET - @Path("workspaces/{workspaceId}/applications") - public Response getWorkspaceApplications(@Context HttpServletRequest req, @PathParam("workspaceId") Long workspaceId) { - String header = req.getHeader("Content-language").trim(); - boolean isChinese = "zh-CN".equals(header); - String username = SecurityFilter.getLoginUsername(req); - List applications = dssWorkspaceService.getWorkspaceApplications(workspaceId, username, isChinese); - return Message.messageToResponse(Message.ok().data("applications", applications)); - } - - @GET - @Path("/workspaces/{workspaceId}/favorites") - public Response getWorkspaceFavorites(@Context HttpServletRequest req, @PathParam("workspaceId") Long workspaceId) { - String header = req.getHeader("Content-language").trim(); - boolean isChinese = "zh-CN".equals(header); - String username = SecurityFilter.getLoginUsername(req); - List favorites = dssWorkspaceService.getWorkspaceFavorites(workspaceId, username, isChinese); - return Message.messageToResponse(Message.ok().data("favorites", favorites)); - } - - /** - * 应用加入收藏，返回收藏后id - * - * @param req - * @param json - * @return - */ - @POST - @Path("/workspaces/{workspaceId}/favorites") - public Response addFavorite(@Context HttpServletRequest req, @PathParam("workspaceId") Long workspaceId, JsonNode json) { - String username = SecurityFilter.getLoginUsername(req); - Long menuApplicationId = json.get("menuApplicationId").getLongValue(); - Long favoriteId = dssWorkspaceService.addFavorite(username, workspaceId, menuApplicationId); - return Message.messageToResponse(Message.ok().data("favoriteId", favoriteId)); - } - - @DELETE - @Path("/workspaces/{workspaceId}/favorites/{favouritesId}") - public Response deleteFavorite(@Context HttpServletRequest req, @PathParam("workspaceId") Long workspaceId, @PathParam("favouritesId") Long favouritesId) { - String username = SecurityFilter.getLoginUsername(req); - Long favoriteId = dssWorkspaceService.deleteFavorite(username, favouritesId); - return Message.messageToResponse(Message.ok().data("favoriteId", favoriteId)); - } -} diff --git a/dss-server/src/main/java/com/webank/wedatasphere/dss/server/service/DSSFlowService.java b/dss-server/src/main/java/com/webank/wedatasphere/dss/server/service/DSSFlowService.java deleted file mode 100644 index 12d0f57709..0000000000 --- a/dss-server/src/main/java/com/webank/wedatasphere/dss/server/service/DSSFlowService.java +++ /dev/null @@ -1,74 +0,0 @@ -/* - * Copyright 2019 WeBank - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package com.webank.wedatasphere.dss.server.service; - -import com.webank.wedatasphere.dss.appjoint.exception.AppJointErrorException; -import com.webank.wedatasphere.dss.common.entity.flow.DSSFlow; -import com.webank.wedatasphere.dss.common.entity.flow.DSSFlowVersion; -import com.webank.wedatasphere.dss.common.exception.DSSErrorException; -import com.webank.wedatasphere.dss.server.operate.Op; - -import java.util.List; - - -public interface DSSFlowService { - DSSFlow getFlowByID(Long id); - - List listAllFlowVersions(Long flowID, Long projectVersionID); - - DSSFlow addRootFlow(DSSFlow dssFlow, Long taxonomyID, Long projectVersionID) throws DSSErrorException; - - DSSFlow addSubFlow(DSSFlow dssFlow, Long parentFlowID, Long projectVersionID) throws DSSErrorException; - - /** - * 通过flowID获取最新版本的DSSFlow，版本信息在latestVersion - * @param flowID - * @return - */ - DSSFlow getLatestVersionFlow(Long flowID, Long projectVersionID) throws DSSErrorException; - - /** - * 通过flowID和某个版本号，获取一个DSSFlow，版本信息在versions数组中的第一个元素 - * @param flowID - * @return - */ - DSSFlow getOneVersionFlow(Long flowID, String version, Long projectVersionID); - - /** - * 通过DSSFlow对象拿到最新的json，其实这里只要个flowID应该就可以了 - * @param dssFlow - * @return - */ -/* String getLatestJsonByFlow(dssFlow dssFlow); - - dssFlow getLatestVersionFlow(Long ProjectID,String flowName);*/ - - void updateFlowBaseInfo(DSSFlow dssFlow, Long projectVersionID, Long taxonomyID) throws DSSErrorException; - - void updateFlowTaxonomyRelation(Long flowID, Long taxonomyID) throws DSSErrorException; - - void batchDeleteFlow(List flowIDlist, Long projectVersionID); - - String saveFlow(Long flowID, String jsonFlow, String comment, String userName, Long projectVersionID, List ops) throws DSSErrorException, AppJointErrorException; - - Integer getParentRank(Long flowID); - - DSSFlowVersion getLatestVersionByFlowIDAndProjectVersionID(Long flowID, Long projectVersionID); - - Long getParentFlowID(Long id); -} diff --git a/dss-server/src/main/java/com/webank/wedatasphere/dss/server/service/DSSFlowTaxonomyService.java b/dss-server/src/main/java/com/webank/wedatasphere/dss/server/service/DSSFlowTaxonomyService.java deleted file mode 100644 index f105779a35..0000000000 --- a/dss-server/src/main/java/com/webank/wedatasphere/dss/server/service/DSSFlowTaxonomyService.java +++ /dev/null @@ -1,41 +0,0 @@ -/* - * Copyright 2019 WeBank - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package com.webank.wedatasphere.dss.server.service; - - -import com.webank.wedatasphere.dss.server.entity.DSSFlowTaxonomy; -import com.webank.wedatasphere.dss.common.exception.DSSErrorException; - -import java.util.List; - - -public interface DSSFlowTaxonomyService { - DSSFlowTaxonomy getFlowTaxonomyByID(Long id); - - //--------- - List listAllFlowTaxonomy(Long projectVersionID, Boolean isRootFlow); - List listFlowTaxonomy(Long projectVersionID, Long flowTaxonomyID, Boolean isRootFlow); - - void addFlowTaxonomy(DSSFlowTaxonomy dssFlowTaxonomy, Long projectVersionID) throws DSSErrorException; - - void updateFlowTaxonomy(DSSFlowTaxonomy dssFlowTaxonomy, Long projectVersionID) throws DSSErrorException; - - boolean hasFlows(Long flowTaxonomyID); - - void deleteFlowTaxonomy(Long flowTaxonomyID, Long projectVersionID); -} diff --git a/dss-server/src/main/java/com/webank/wedatasphere/dss/server/service/DSSNodeInfoService.java b/dss-server/src/main/java/com/webank/wedatasphere/dss/server/service/DSSNodeInfoService.java deleted file mode 100644 index dc7303f52f..0000000000 --- a/dss-server/src/main/java/com/webank/wedatasphere/dss/server/service/DSSNodeInfoService.java +++ /dev/null @@ -1,33 +0,0 @@ -/* - * Copyright 2019 WeBank - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package com.webank.wedatasphere.dss.server.service; - - - - -import com.webank.wedatasphere.dss.application.entity.Application; -import com.webank.wedatasphere.dss.server.entity.NodeInfo; - -import java.util.List; - - -public interface DSSNodeInfoService { - - List getNodeType(String f); - -} diff --git a/dss-server/src/main/java/com/webank/wedatasphere/dss/server/service/DSSProjectService.java b/dss-server/src/main/java/com/webank/wedatasphere/dss/server/service/DSSProjectService.java deleted file mode 100644 index 1cad4fd681..0000000000 --- a/dss-server/src/main/java/com/webank/wedatasphere/dss/server/service/DSSProjectService.java +++ /dev/null @@ -1,67 +0,0 @@ -/* - * Copyright 2019 WeBank - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package com.webank.wedatasphere.dss.server.service; - -import com.webank.wedatasphere.dss.appjoint.exception.AppJointErrorException; -import com.webank.wedatasphere.dss.common.entity.project.DSSProject; -import com.webank.wedatasphere.dss.common.entity.project.DSSProjectPublishHistory; -import com.webank.wedatasphere.dss.common.entity.project.DSSProjectVersion; -import com.webank.wedatasphere.dss.common.exception.DSSErrorException; -import com.webank.wedatasphere.dss.common.protocol.RequestDSSProject; - -import java.util.Date; -import java.util.List; - - -public interface DSSProjectService { - - DSSProject getProjectByID(Long id); - - Long addProject(String userName, String name, String description, Long taxonomyID,String product,Integer applicationArea,String business, Long workspaceId) throws DSSErrorException, AppJointErrorException; - - void updateProject(long projectID, String name, String description, String userName , String product ,Integer applicationArea ,String business) throws AppJointErrorException; - - void deleteProject(long projectID, String userName, Boolean ifDelScheduler) throws DSSErrorException; - - DSSProject getLatestVersionProject(Long projectID); - - DSSProject getProjectByProjectVersionID(Long projectVersionID); - - Boolean isPublished(Long projectID); - - List listAllProjectVersions(Long projectID); - - Long copyProject(Long projectVersionID, Long projectID, String projectName, String userName) throws DSSErrorException, InterruptedException, AppJointErrorException; - - void copyProjectVersionMax(Long projectVersionID, DSSProjectVersion maxVersion, DSSProjectVersion copyVersion, String userName, Long WTSSprojectID) throws DSSErrorException, InterruptedException; - - void publish(Long projectVersionID, String userName, String comment) throws DSSErrorException, InterruptedException, AppJointErrorException; - - Long createPublishHistory(String comment, Long creatorID, Long projectVersionID) throws DSSErrorException; - - void updatePublishHistory(Long projectVersionID, Integer status, Date updateTime); - - DSSProjectPublishHistory getPublishHistoryByID(Long projectVersionID); - - DSSProject getExecutionDSSProject(RequestDSSProject requestDSSProject) throws DSSErrorException; - - Long getAppjointProjectID(Long projectID, String nodeType); - - Long getAppjointProjectIDByApplicationName(Long projectID, String applicationName); -} - diff --git a/dss-server/src/main/java/com/webank/wedatasphere/dss/server/service/DSSProjectTaxonomyService.java b/dss-server/src/main/java/com/webank/wedatasphere/dss/server/service/DSSProjectTaxonomyService.java deleted file mode 100644 index cebabbf63b..0000000000 --- a/dss-server/src/main/java/com/webank/wedatasphere/dss/server/service/DSSProjectTaxonomyService.java +++ /dev/null @@ -1,46 +0,0 @@ -/* - * Copyright 2019 WeBank - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package com.webank.wedatasphere.dss.server.service; - -import com.webank.wedatasphere.dss.server.entity.DSSProjectTaxonomy; -import com.webank.wedatasphere.dss.common.exception.DSSErrorException; - -import java.util.List; - - -public interface DSSProjectTaxonomyService { - - DSSProjectTaxonomy getProjectTaxonomyByID(Long id); - - List listProjectTaxonomyByUser(String userName); - - //---------------------- - List listAllProjectTaxonomy(String userName, Long workspaceId); - - List listProjectTaxonomy(Long taxonomyID, String userName); - - void addProjectTaxonomy(DSSProjectTaxonomy dssProjectTaxonomy) throws DSSErrorException; - - void updateProjectTaxonomy(DSSProjectTaxonomy dssProjectTaxonomy) throws DSSErrorException; - - boolean hasProjects(Long projectTaxonomyID); - - void deleteProjectTaxonomy(Long projectTaxonomyID); - - void updateProjectTaxonomyRelation(Long projectID, Long taxonomyID); -} diff --git a/dss-server/src/main/java/com/webank/wedatasphere/dss/server/service/DSSUserService.java b/dss-server/src/main/java/com/webank/wedatasphere/dss/server/service/DSSUserService.java deleted file mode 100644 index 2c2760297c..0000000000 --- a/dss-server/src/main/java/com/webank/wedatasphere/dss/server/service/DSSUserService.java +++ /dev/null @@ -1,23 +0,0 @@ -/* - * Copyright 2019 WeBank - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package com.webank.wedatasphere.dss.server.service; - - -public interface DSSUserService { - Long getUserID(String userName); -} diff --git a/dss-server/src/main/java/com/webank/wedatasphere/dss/server/service/DSSWorkspaceService.java b/dss-server/src/main/java/com/webank/wedatasphere/dss/server/service/DSSWorkspaceService.java deleted file mode 100644 index f88c221ea2..0000000000 --- a/dss-server/src/main/java/com/webank/wedatasphere/dss/server/service/DSSWorkspaceService.java +++ /dev/null @@ -1,63 +0,0 @@ -/* - * Copyright 2019 WeBank - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package com.webank.wedatasphere.dss.server.service; - -import com.webank.wedatasphere.dss.server.dto.response.HomepageDemoMenuVo; -import com.webank.wedatasphere.dss.server.dto.response.HomepageVideoVo; -import com.webank.wedatasphere.dss.server.dto.response.OnestopMenuVo; -import com.webank.wedatasphere.dss.server.entity.DSSWorkspace; -import com.webank.wedatasphere.dss.server.dto.response.WorkspaceDepartmentVo; -import com.webank.wedatasphere.dss.server.dto.response.*; - -import java.util.List; - -/** - * Created by schumiyi on 2020/6/22 - */ -public interface DSSWorkspaceService { - List getWorkspaces(); - - Long addWorkspace(String userName, String name, String department, String label, String description); - - boolean existWorkspaceName(String name); - - List getWorkSpaceDepartments(); - - List getHomepageDemos(boolean isChinese); - - List getHomepageVideos(boolean isChinese); - - List getWorkspaceManagements(Long workspaceId, String username, boolean isChinese); - - List getWorkspaceApplications(Long workspaceId, String username, boolean isChinese); - - DSSWorkspace getWorkspacesById(Long id); - - /** - * 查询用户收藏的应用，如果是新用户，就在数据库给它插入默认两个收藏：脚本开发与工作流 workflow scriptis - * @param workspaceId - * @param username - * @param isChinese - * @return - */ - List getWorkspaceFavorites(Long workspaceId, String username, boolean isChinese); - - Long addFavorite(String username, Long workspaceId, Long menuApplicationId); - - Long deleteFavorite(String username, Long favouritesId); -} diff --git a/dss-server/src/main/java/com/webank/wedatasphere/dss/server/service/impl/DSSFlowServiceImpl.java b/dss-server/src/main/java/com/webank/wedatasphere/dss/server/service/impl/DSSFlowServiceImpl.java deleted file mode 100644 index 218aacc449..0000000000 --- a/dss-server/src/main/java/com/webank/wedatasphere/dss/server/service/impl/DSSFlowServiceImpl.java +++ /dev/null @@ -1,282 +0,0 @@ -/* - * Copyright 2019 WeBank - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package com.webank.wedatasphere.dss.server.service.impl; - - -import com.webank.wedatasphere.dss.appjoint.exception.AppJointErrorException; -import com.webank.wedatasphere.dss.common.entity.flow.DSSFlow; -import com.webank.wedatasphere.dss.common.entity.flow.DSSFlowVersion; -import com.webank.wedatasphere.dss.common.exception.DSSErrorException; -import com.webank.wedatasphere.dss.server.dao.DSSUserMapper; -import com.webank.wedatasphere.dss.server.dao.FlowMapper; -import com.webank.wedatasphere.dss.server.dao.FlowTaxonomyMapper; -import com.webank.wedatasphere.dss.server.lock.Lock; -import com.webank.wedatasphere.dss.server.operate.Op; -import com.webank.wedatasphere.dss.server.operate.Operate; -import com.webank.wedatasphere.dss.server.service.BMLService; -import com.webank.wedatasphere.dss.server.service.DSSFlowService; -import com.webank.wedatasphere.dss.server.service.DSSProjectService; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.dao.DuplicateKeyException; -import org.springframework.stereotype.Service; -import org.springframework.transaction.annotation.Transactional; - -import java.util.*; -import java.util.stream.Collectors; - - -@Service -public class DSSFlowServiceImpl implements DSSFlowService { - - private Logger logger = LoggerFactory.getLogger(this.getClass()); - - @Autowired - private FlowMapper flowMapper; - @Autowired - private FlowTaxonomyMapper flowTaxonomyMapper; - @Autowired - private DSSUserMapper dssUserMapper; - - @Autowired - private DSSProjectService projectService; - @Autowired - private Operate[] operates; - @Autowired - private BMLService bmlService; - - @Override - public DSSFlow getFlowByID(Long id) { - return flowMapper.selectFlowByID(id); - } - - @Override - public List listAllFlowVersions(Long flowID, Long projectVersionID) { - List versions = flowMapper.listFlowVersionsByFlowID(flowID, projectVersionID).stream().sorted((v1, v2) -> { - return v1.compareTo(v2); - }).collect(Collectors.toList()); - return versions; - } - - @Lock - @Transactional(rollbackFor = DSSErrorException.class) - @Override - public DSSFlow addRootFlow(DSSFlow dssFlow, Long taxonomyID, Long projectVersionID) throws DSSErrorException { - try { - flowMapper.insertFlow(dssFlow); - } catch (DuplicateKeyException e) { - logger.info(e.getMessage()); - throw new DSSErrorException(90003, "工作流名不能重复"); - } - //通过resource上传空文件，获取resourceId(jsonPath)和version - Map bmlReturnMap = bmlService.upload(dssUserMapper.getuserName(dssFlow.getCreatorID()), "", UUID.randomUUID().toString() + ".json"); - //数据库中插入版本信息 - DSSFlowVersion version = new DSSFlowVersion(); - version.setFlowID(dssFlow.getId()); - version.setSource("create by user"); - version.setJsonPath(bmlReturnMap.get("resourceId").toString()); - version.setVersion(bmlReturnMap.get("version").toString()); - version.setUpdateTime(new Date()); - version.setUpdatorID(dssFlow.getCreatorID()); - // TODO: 2019/6/12 这里应该由前台传入 - version.setProjectVersionID(projectVersionID); - flowMapper.insertFlowVersion(version); - //数据库中插入分类关联信息 - flowTaxonomyMapper.insertFlowTaxonomyRelation(taxonomyID, dssFlow.getId()); - return dssFlow; - } - - @Lock - @Transactional(rollbackFor = DSSErrorException.class) - @Override - public DSSFlow addSubFlow(DSSFlow dssFlow, Long parentFlowID, Long projectVersionID) throws DSSErrorException { - Long taxonomyID = flowTaxonomyMapper.selectTaxonomyIDByFlowID(parentFlowID); - DSSFlow parentFlow = flowMapper.selectFlowByID(parentFlowID); - dssFlow.setProjectID(parentFlow.getProjectID()); - DSSFlow subFlow = addRootFlow(dssFlow, taxonomyID, projectVersionID); - //数据库中插入关联信息 - flowMapper.insertFlowRelation(subFlow.getId(), parentFlowID); - return subFlow; - } - - @Override - public DSSFlow getLatestVersionFlow(Long flowID, Long projectVersionID) throws DSSErrorException { - DSSFlow dssFlow = getFlowByID(flowID); - DSSFlowVersion dssFlowVersion = getLatestVersionByFlowIDAndProjectVersionID(flowID, projectVersionID); - if (dssFlowVersion == null) throw new DSSErrorException(90011, "该工作流已经被删除，请重新创建"); - String userName = dssUserMapper.getuserName(dssFlowVersion.getUpdatorID()); - Map query = bmlService.query(userName, dssFlowVersion.getJsonPath(), dssFlowVersion.getVersion()); - dssFlowVersion.setJson(query.get("string").toString()); - dssFlow.setLatestVersion(dssFlowVersion); - return dssFlow; - } - - @Override - public DSSFlow getOneVersionFlow(Long flowID, String version, Long projectVersionID) { - DSSFlow dssFlow = getFlowByID(flowID); - DSSFlowVersion dssFlowVersion = flowMapper.selectVersionByFlowID(flowID, version, projectVersionID); - String userName = dssUserMapper.getuserName(dssFlowVersion.getUpdatorID()); - Map query = bmlService.query(userName, dssFlowVersion.getJsonPath(), dssFlowVersion.getVersion()); - dssFlowVersion.setJson(query.get("string").toString()); - dssFlow.setFlowVersions(Arrays.asList(dssFlowVersion)); - dssFlow.setLatestVersion(dssFlowVersion); - return dssFlow; - } - -/* @Override - public String getLatestJsonByFlow(DSSFlow DSSFlow) { - DSSFlow latestVersionFlow = getLatestVersionFlow(DSSFlow.getId()); - return latestVersionFlow.getLatestVersion().getJson(); - } - - @Override - public DSSFlow getLatestVersionFlow(Long ProjectID, String flowName) { - Long flowID = flowMapper.selectFlowIDByProjectIDAndFlowName(ProjectID, flowName); - return getLatestVersionFlow(flowID); - }*/ - - @Lock - @Transactional(rollbackFor = DSSErrorException.class) - @Override - public void updateFlowBaseInfo(DSSFlow dssFlow, Long projectVersionID, Long taxonomyID) throws DSSErrorException { - try { - flowMapper.updateFlowBaseInfo(dssFlow); - } catch (DuplicateKeyException e) { - logger.info(e.getMessage()); - throw new DSSErrorException(90003, "工作流名不能重复"); - } - if (taxonomyID != null) updateFlowTaxonomyRelation(dssFlow.getId(), taxonomyID); - } - - @Override - public void updateFlowTaxonomyRelation(Long flowID, Long taxonomyID) throws DSSErrorException { - DSSFlow dssFlow = getFlowByID(flowID); - Long oldTaxonomyID = flowTaxonomyMapper.selectTaxonomyIDByFlowID(flowID); - if (!dssFlow.getRootFlow() && (!oldTaxonomyID.equals(taxonomyID))) throw new DSSErrorException(90010, "子工作流不允许更新分类id"); - if (!dssFlow.getRootFlow() && (oldTaxonomyID.equals(taxonomyID))) return; - //这里也要同时更新子工作流的分类id - List subFlowIDList = flowMapper.selectSubFlowIDByParentFlowID(flowID); - subFlowIDList.add(flowID); - flowTaxonomyMapper.updateFlowTaxonomyRelation(subFlowIDList, taxonomyID); - } - - @Lock - @Transactional(rollbackFor = DSSErrorException.class) - @Override - public void batchDeleteFlow(List flowIDlist, Long projectVersionID) { - flowIDlist.stream().forEach(f -> { - deleteFlow(f, projectVersionID); - }); - } - - @Lock - @Transactional(rollbackFor = {DSSErrorException.class,AppJointErrorException.class}) - @Override - public String saveFlow(Long flowID, String jsonFlow, String comment, String userName, Long projectVersionID, List ops) throws DSSErrorException, AppJointErrorException { - for (Op op : ops) { - op.getParams().put("projectVersionID",projectVersionID); - op.getParams().put("userName",userName); - logger.info("{}保存工作流，操作:{}id:{}nodetype{},projectVersionID:{}",userName,op.getOp(),op.getId(),op.getNodeType(),projectVersionID); - Optional operate = Arrays.stream(operates).filter(f -> f.canInvokeOperate(op)).findFirst(); - // TODO: 2019/10/29 optinal 判断 and throws exception - switch (op.getOp()) { - case "add": - operate.get().add(this,op); - break; - case "update": - operate.get().update(this,op); - break; - case "delete": - operate.get().delete(this,op); - break; - default: - logger.error("other operation:unable to occur"); - } - } - //获取rsourceId，就是jsonPaht - String resourceId = getLatestVersionByFlowIDAndProjectVersionID(flowID, projectVersionID).getJsonPath(); - //上传文件获取resourceId和version save应该是已经有 - Map bmlReturnMap = bmlService.update(userName, resourceId, jsonFlow); - DSSFlowVersion dssFlowVersion = new DSSFlowVersion(); - dssFlowVersion.setUpdatorID(dssUserMapper.getUserID(userName)); - dssFlowVersion.setUpdateTime(new Date()); - dssFlowVersion.setVersion(bmlReturnMap.get("version").toString()); - dssFlowVersion.setJsonPath(resourceId); - dssFlowVersion.setComment(comment); - dssFlowVersion.setFlowID(flowID); - dssFlowVersion.setSource("保存更新"); - dssFlowVersion.setProjectVersionID(projectVersionID); - //version表中插入数据 - flowMapper.insertFlowVersion(dssFlowVersion); - return bmlReturnMap.get("version").toString(); - } - - @Override - public Integer getParentRank(Long parentFlowID) { - return getFlowByID(parentFlowID).getRank(); - } - - @Override - public DSSFlowVersion getLatestVersionByFlowIDAndProjectVersionID(Long flowID, Long projectVersionID) { - List versions = flowMapper.listVersionByFlowIDAndProjectVersionID(flowID, projectVersionID) - .stream().sorted((v1, v2) -> { - return v1.compareTo(v2); - }).collect(Collectors.toList()); - return versions.isEmpty() ? null : versions.get(0); - } - - @Override - public Long getParentFlowID(Long flowID) { - return flowMapper.getParentFlowID(flowID); - } - - @Deprecated - private Integer getParentFlowIDByFlowID(Long flowID, Integer rank) { - Long parentFlowID = flowMapper.selectParentFlowIDByFlowID(flowID); - if (parentFlowID != null) { - rank++; - getParentFlowIDByFlowID(parentFlowID, rank); - } - return rank; - } - - public void deleteFlow(Long flowId, Long projectVersionID) { - List subFlowIDs = flowMapper.selectSubFlowIDByParentFlowID(flowId); - for (Long subFlowID : subFlowIDs) { - deleteFlow(subFlowID, projectVersionID); - } - for (Long subFlowID : subFlowIDs) { - deleteDSSDB(subFlowID, projectVersionID); - // TODO: 2019/6/5 wtss发布过的工作流的删除？ - // TODO: 2019/6/5 json中资源的删除 - // TODO: 2019/6/5 事务的保证 - } - deleteDSSDB(flowId, projectVersionID); - } - - private void deleteDSSDB(Long flowID, Long projectVersionID) { - flowMapper.deleteFlowVersions(flowID, projectVersionID); - if (projectVersionID == null || (flowMapper.noVersions(flowID) != null && flowMapper.noVersions(flowID))) { - flowMapper.deleteFlowBaseInfo(flowID); - flowMapper.deleteFlowRelation(flowID); - flowTaxonomyMapper.deleteFlowTaxonomyRelation(flowID); - } - //第一期没有工作流的发布，所以不需要删除DSS工作流的发布表 - } -} diff --git a/dss-server/src/main/java/com/webank/wedatasphere/dss/server/service/impl/DSSFlowTaxonomyServiceImpl.java b/dss-server/src/main/java/com/webank/wedatasphere/dss/server/service/impl/DSSFlowTaxonomyServiceImpl.java deleted file mode 100644 index 0cbba745b2..0000000000 --- a/dss-server/src/main/java/com/webank/wedatasphere/dss/server/service/impl/DSSFlowTaxonomyServiceImpl.java +++ /dev/null @@ -1,138 +0,0 @@ -/* - * Copyright 2019 WeBank - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package com.webank.wedatasphere.dss.server.service.impl; - -import com.webank.wedatasphere.dss.server.service.DSSFlowService; -import com.webank.wedatasphere.dss.server.service.DSSFlowTaxonomyService; -import com.webank.wedatasphere.dss.server.service.DSSProjectService; -import com.webank.wedatasphere.dss.server.dao.FlowMapper; -import com.webank.wedatasphere.dss.server.dao.FlowTaxonomyMapper; -import com.webank.wedatasphere.dss.server.entity.DSSFlowTaxonomy; -import com.webank.wedatasphere.dss.common.entity.flow.DSSFlow; -import com.webank.wedatasphere.dss.common.entity.flow.DSSFlowVersion; -import com.webank.wedatasphere.dss.common.entity.project.DSSProject; -import com.webank.wedatasphere.dss.common.exception.DSSErrorException; -import com.webank.wedatasphere.dss.server.lock.Lock; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.dao.DuplicateKeyException; -import org.springframework.stereotype.Service; -import org.springframework.transaction.annotation.Transactional; - -import java.util.Arrays; -import java.util.List; -import java.util.stream.Collectors; - - -@Service -public class DSSFlowTaxonomyServiceImpl implements DSSFlowTaxonomyService { - - private Logger logger = LoggerFactory.getLogger(this.getClass()); - @Autowired - private FlowTaxonomyMapper flowTaxonomyMapper; - - @Autowired - private FlowMapper flowMapper; - - @Autowired - private DSSProjectService projectService; - - @Autowired - private DSSFlowService flowService; - - @Override - public DSSFlowTaxonomy getFlowTaxonomyByID(Long id) { - return flowTaxonomyMapper.selectFlowTaxonomyByID(id); - } - - private List listFlowByTaxonomyID(Long projectID, Long taxonomyID, Boolean isRootFlow){ - return flowMapper.listFlowByTaxonomyID(projectID,taxonomyID,isRootFlow); - } - - // TODO: 2019/5/16 级联查询返回json序列化好像有问题，暂时只能循环查 - @Override - public List listAllFlowTaxonomy(Long projectVersionID, Boolean isRootFlow) { - DSSProject dssProject = projectService.getProjectByProjectVersionID(projectVersionID); - List dwsFlowTaxonomies = listFlowTaxonomyByProjectID(dssProject.getId()); - for (DSSFlowTaxonomy dssFlowTaxonomy : dwsFlowTaxonomies) { - List dssFlowList = listFlowByTaxonomyID(dssProject.getId(), dssFlowTaxonomy.getId(),isRootFlow); - for (DSSFlow dssFlow : dssFlowList) { - DSSFlowVersion version = flowService.getLatestVersionByFlowIDAndProjectVersionID(dssFlow.getId(),projectVersionID); - dssFlow.setLatestVersion(version); - } - dssFlowTaxonomy.setDssFlowList(dssFlowList.stream().filter(f ->f.getLatestVersion() !=null).collect(Collectors.toList())); - } - return dwsFlowTaxonomies; - } - - private List listFlowTaxonomyByProjectID(Long projectID) { - return flowTaxonomyMapper.listFlowTaxonomyByProjectID(projectID); - } - - //有projectID应该可以同时过滤不同用户中的，我的工作流分类下的工作流 - @Override - public List listFlowTaxonomy(Long projectVersionID, Long flowTaxonomyID, Boolean isRootFlow) { - DSSFlowTaxonomy flowTaxonomy = getFlowTaxonomyByID(flowTaxonomyID); - DSSProject dssProject = projectService.getProjectByProjectVersionID(projectVersionID); - List dssFlowList = listFlowByTaxonomyID(dssProject.getId(),flowTaxonomyID,isRootFlow); - for (DSSFlow dssFlow : dssFlowList) { - DSSFlowVersion version = flowService.getLatestVersionByFlowIDAndProjectVersionID(dssFlow.getId(),projectVersionID); - dssFlow.setLatestVersion(version); - } - flowTaxonomy.setDssFlowList(dssFlowList.stream().filter(f ->f.getLatestVersion() !=null).collect(Collectors.toList())); - return Arrays.asList(flowTaxonomy); - } - - @Lock - @Transactional(rollbackFor = DSSErrorException.class) - @Override - public void addFlowTaxonomy(DSSFlowTaxonomy dssFlowTaxonomy, Long projectVersionID) throws DSSErrorException { - try { - flowTaxonomyMapper.insertFlowTaxonomy(dssFlowTaxonomy); - }catch (DuplicateKeyException e){ - logger.info(e.getMessage()); - throw new DSSErrorException(90005,"工作流名分类名不能重复"); - } - } - - @Lock - @Transactional(rollbackFor = DSSErrorException.class) - @Override - public void updateFlowTaxonomy(DSSFlowTaxonomy dssFlowTaxonomy, Long projectVersionID) throws DSSErrorException { - try { - flowTaxonomyMapper.updateFlowTaxonomy(dssFlowTaxonomy); - }catch (DuplicateKeyException e){ - logger.info(e.getMessage()); - throw new DSSErrorException(90005,"工作流名分类名不能重复"); - } - } - - @Override - public boolean hasFlows(Long flowTaxonomyID) { - Long count = flowTaxonomyMapper.hasFlows(flowTaxonomyID); - return count != 0; - } - - @Lock - @Transactional(rollbackFor = DSSErrorException.class) - @Override - public void deleteFlowTaxonomy(Long flowTaxonomyID,Long projectVersionID) { - flowTaxonomyMapper.deleteFlowTaxonomy(flowTaxonomyID); - } -} diff --git a/dss-server/src/main/java/com/webank/wedatasphere/dss/server/service/impl/DSSNodeInfoServiceImpl.java b/dss-server/src/main/java/com/webank/wedatasphere/dss/server/service/impl/DSSNodeInfoServiceImpl.java deleted file mode 100644 index 0b9571f265..0000000000 --- a/dss-server/src/main/java/com/webank/wedatasphere/dss/server/service/impl/DSSNodeInfoServiceImpl.java +++ /dev/null @@ -1,40 +0,0 @@ -/* - * Copyright 2019 WeBank - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package com.webank.wedatasphere.dss.server.service.impl; - -import com.webank.wedatasphere.dss.server.dao.NodeInfoMapper; -import com.webank.wedatasphere.dss.server.entity.NodeInfo; -import com.webank.wedatasphere.dss.server.service.DSSNodeInfoService; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.stereotype.Service; - -import java.util.List; - - -@Service -public class DSSNodeInfoServiceImpl implements DSSNodeInfoService { - - @Autowired - private NodeInfoMapper nodeInfoMapper; - - @Override - public List getNodeType(String developProject) { - return nodeInfoMapper.selectNodeInfo(developProject); - } - -} diff --git a/dss-server/src/main/java/com/webank/wedatasphere/dss/server/service/impl/DSSProjectServiceImpl.java b/dss-server/src/main/java/com/webank/wedatasphere/dss/server/service/impl/DSSProjectServiceImpl.java deleted file mode 100644 index 9d007cebc5..0000000000 --- a/dss-server/src/main/java/com/webank/wedatasphere/dss/server/service/impl/DSSProjectServiceImpl.java +++ /dev/null @@ -1,609 +0,0 @@ -/* - * Copyright 2019 WeBank - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package com.webank.wedatasphere.dss.server.service.impl; - - -import com.webank.wedatasphere.dss.appjoint.exception.AppJointErrorException; -import com.webank.wedatasphere.dss.appjoint.scheduler.SchedulerAppJoint; -import com.webank.wedatasphere.dss.appjoint.scheduler.entity.SchedulerProject; -import com.webank.wedatasphere.dss.appjoint.scheduler.hooks.ProjectPublishHook; -import com.webank.wedatasphere.dss.appjoint.scheduler.parser.ProjectParser; -import com.webank.wedatasphere.dss.appjoint.scheduler.tuning.ProjectTuning; -import com.webank.wedatasphere.dss.appjoint.service.ProjectService; -import com.webank.wedatasphere.dss.application.entity.Application; -import com.webank.wedatasphere.dss.application.service.ApplicationService; -import com.webank.wedatasphere.dss.common.entity.flow.DSSFlow; -import com.webank.wedatasphere.dss.common.entity.flow.DSSFlowVersion; -import com.webank.wedatasphere.dss.common.entity.project.DSSProject; -import com.webank.wedatasphere.dss.common.entity.project.DSSProjectPublishHistory; -import com.webank.wedatasphere.dss.common.entity.project.DSSProjectVersion; -import com.webank.wedatasphere.dss.common.entity.project.Project; -import com.webank.wedatasphere.dss.common.exception.DSSErrorException; -import com.webank.wedatasphere.dss.common.protocol.RequestDSSProject; -import com.webank.wedatasphere.dss.common.utils.DSSExceptionUtils; -import com.webank.wedatasphere.dss.server.constant.DSSServerConstant; -import com.webank.wedatasphere.dss.server.dao.*; -import com.webank.wedatasphere.dss.server.entity.DSSFlowTaxonomy; -import com.webank.wedatasphere.dss.server.entity.DSSProjectTaxonomyRelation; -import com.webank.wedatasphere.dss.server.function.FunctionInvoker; -import com.webank.wedatasphere.dss.server.lock.Lock; -import com.webank.wedatasphere.dss.server.lock.LockEnum; -import com.webank.wedatasphere.dss.server.service.BMLService; -import com.webank.wedatasphere.dss.server.service.DSSFlowService; -import com.webank.wedatasphere.dss.server.service.DSSProjectService; -import com.webank.wedatasphere.dss.server.util.ThreadPoolTool; -import org.apache.commons.lang.StringUtils; -import org.apache.commons.math3.util.Pair; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.stereotype.Service; -import org.springframework.transaction.annotation.Transactional; - -import java.io.*; -import java.util.*; -import java.util.concurrent.ConcurrentHashMap; -import java.util.stream.Collectors; -import java.util.stream.Stream; - - -@Service -public class DSSProjectServiceImpl implements DSSProjectService { - - private Logger logger = LoggerFactory.getLogger(this.getClass()); - - @Autowired - private ProjectTaxonomyMapper projectTaxonomyMapper; - @Autowired - private DSSUserMapper dssUserMapper; - @Autowired - private FlowMapper flowMapper; - @Autowired - private FlowTaxonomyMapper flowTaxonomyMapper; - @Autowired - private DSSFlowService flowService; - @Autowired - private ProjectMapper projectMapper; - - @Autowired - private BMLService bmlService; - @Autowired - private ApplicationService applicationService; - @Autowired - private FunctionInvoker functionInvoker; - - private SchedulerAppJoint schedulerAppJoint=null; - - @Override - public DSSProject getProjectByID(Long id) { - /*JdbcProjectImpl instance = wtssdbConnector.getInjector().getInstance(JdbcProjectImpl.class); - Project project = instance.fetchProjectById(id.intValue()); - DSSProject DSSProject = EntityUtils.Project2DSSProject(project);*/ - return projectMapper.selectProjectByID(id); - } - - @Transactional(rollbackFor = {DSSErrorException.class,AppJointErrorException.class}) - @Override - public Long addProject(String userName, String name, String description, Long taxonomyID,String product,Integer applicationArea,String business, Long workspaceId) throws DSSErrorException, AppJointErrorException { - DSSProject dssProject = new DSSProject(); - dssProject.setUserName(userName); - dssProject.setName(name); - dssProject.setDescription(description); - //创建scheduler的project - if(existSchesulis()){ - createSchedulerProject(dssProject); - } - //创建appjoint的project - Map appjointProjectIDAndAppID = createAppjointProject(dssProject); - Long userID = dssUserMapper.getUserID(userName); - //创建dss自己的project - Pair pair = addDSSProject(userID, name, description,product,applicationArea,business, workspaceId); - //添加关联 - projectTaxonomyMapper.addProjectTaxonomyRelation(pair.getFirst(), taxonomyID, userID); - if(!appjointProjectIDAndAppID.isEmpty())projectMapper.addAccessProjectRelation(appjointProjectIDAndAppID,pair.getFirst()); - return pair.getSecond(); - } - - - - private Map createAppjointProject(DSSProject project) throws DSSErrorException, AppJointErrorException { - Map applicationProjectIDs = new HashMap(); - List> pairs = functionInvoker.projectServiceAddFunction(project, ProjectService::createProject, applicationService.listAppjoint()); - for (Pair pair : pairs) { - if(pair.getFirst().getId() != null){ - applicationProjectIDs.put(applicationService.getApplication(pair.getSecond()).getId(),pair.getFirst().getId()); - } - } - return applicationProjectIDs; - } - - private Pair addDSSProject(Long userID, String name, String description, String product, Integer applicationArea, String business, Long workspaceId) { - DSSProject dssProject = new DSSProject(); - dssProject.setUserID(userID); - dssProject.setName(name); - dssProject.setDescription(description); - dssProject.setSource(DSSServerConstant.DSS_PROJECT_SOURCE); - dssProject.setCreateTime(new Date()); - dssProject.setCreateBy(userID); - dssProject.setProduct(product); - dssProject.setApplicationArea(applicationArea); - dssProject.setBusiness(business); - dssProject.setWorkspaceId(workspaceId); - projectMapper.addProject(dssProject); - DSSProjectVersion dssProjectVersion = new DSSProjectVersion(); - dssProjectVersion.setComment(DSSServerConstant.DSS_PROJECT_FIRST_VERSION_COMMENT); - dssProjectVersion.setProjectID(dssProject.getId()); - dssProjectVersion.setUpdateTime(new Date()); - dssProjectVersion.setUpdatorID(userID); - dssProjectVersion.setVersion(DSSServerConstant.DSS_PROJECT_FIRST_VERSION); - dssProjectVersion.setLock(0); - projectMapper.addProjectVersion(dssProjectVersion); - return new Pair(dssProject.getId(), dssProjectVersion.getId()); - } - - private void createSchedulerProject(DSSProject dssProject) throws DSSErrorException { - try { - if(getSchedulerAppJoint() != null) { - functionInvoker.projectServiceAddFunction(dssProject, ProjectService::createProject, Arrays.asList(getSchedulerAppJoint())); - }else{ - logger.error("Add scheduler project failed for scheduler appjoint is null"); - } - } catch (Exception e) { - logger.error("add scheduler project failed,", e); - throw new DSSErrorException(90002, "add scheduler project failed" + e.getMessage()); - } - } - - @Override - public void updateProject(long projectID, String name, String description, String userName, String product ,Integer applicationArea ,String business) throws AppJointErrorException { - // TODO: 2019/9/19 appjoint的update - //无法修改名字 - //更新wtssProject中的description - if(!StringUtils.isBlank(description)){ - DSSProject project = getProjectByID(projectID); - project.setUserName(userName); - project.setDescription(description); - if(existSchesulis()){ - if(getSchedulerAppJoint() != null) { - functionInvoker.projectServiceFunction(project,ProjectService::updateProject,Arrays.asList(getSchedulerAppJoint())); - }else{ - logger.error("Update scheduler project failed for scheduler appjoint is null"); - } - } - functionInvoker.projectServiceFunction(project,ProjectService::updateProject,applicationService.listAppjoint()); - } - projectMapper.updateDescription(projectID, description, product ,applicationArea ,business); - } - - @Transactional(rollbackFor = DSSErrorException.class) - @Override - public void deleteProject(long projectID, String userName, Boolean ifDelScheduler) throws DSSErrorException { - try { - DSSProject project = getProjectByID(projectID); - project.setUserName(userName); - if(ifDelScheduler){ - if(existSchesulis()){ - if(getSchedulerAppJoint() != null) { - functionInvoker.projectServiceFunction(project, ProjectService::deleteProject, Arrays.asList(getSchedulerAppJoint())); - }else{ - logger.error("Delete scheduler project failed for scheduler appjoint is null"); - } - } - } - functionInvoker.projectServiceFunction(project,ProjectService::deleteProject,applicationService.listAppjoint()); - // TODO: 2019/11/15 删除relations表 - } catch (Exception e) { - logger.info("删除工程失败，原因：", e); - String errorMsg = e.getCause() == null ? e.getMessage() : e.getCause().getMessage(); - throw new DSSErrorException(90012, errorMsg); - } - flowTaxonomyMapper.deleteFlowTaxonomyByProjectID(projectID); - List dssFlowList = flowMapper.listFlowByProjectID(projectID); - flowService.batchDeleteFlow(dssFlowList.stream().map(f -> f.getId()).distinct().collect(Collectors.toList()), null); - projectMapper.deleteProjectVersions(projectID); - projectMapper.deleteProjectBaseInfo(projectID); - projectTaxonomyMapper.deleteProjectTaxonomyRelationByProjectID(projectID); - } - - @Override - public DSSProject getLatestVersionProject(Long projectID) { - DSSProject dssProject = getProjectByID(projectID); - DSSProjectVersion dssProjectVersion = projectMapper.selectLatestVersionByProjectID(projectID); - dssProject.setLatestVersion(dssProjectVersion); - return dssProject; - } - - /** - * 通过projectVersionID获取一个projrct对象，不会返回最新的版本信息 - * - * @param projectVersionID - * @return - */ - @Override - public DSSProject getProjectByProjectVersionID(Long projectVersionID) { - DSSProject dssProject = projectMapper.selectProjectByVersionID(projectVersionID); - return dssProject; - } - - @Override - public Boolean isPublished(Long projectID) { - return !projectMapper.noPublished(projectID); - } - - @Override - public List listAllProjectVersions(Long projectID) { - List dssProjectVersions = projectMapper.listProjectVersionsByProjectID(projectID); - for (DSSProjectVersion dssProjectVersion : dssProjectVersions) { - DSSProjectPublishHistory publishHistory = projectMapper.selectProjectPublishHistoryByProjectVersionID(dssProjectVersion.getId()); - dssProjectVersion.setPublishHistory(publishHistory); - } - return dssProjectVersions; - } - - - @Lock(type = LockEnum.ADD) - @Transactional(rollbackFor = {DSSErrorException.class, InterruptedException.class,AppJointErrorException.class}) - @Override - public void publish(Long projectVersionID, String userName, String comment) throws DSSErrorException, InterruptedException, AppJointErrorException { - - SchedulerAppJoint schedulerAppJoint = getSchedulerAppJoint(); - if(schedulerAppJoint != null) { - ProjectParser projectParser = schedulerAppJoint.getProjectParser(); - ProjectTuning projectTuning = schedulerAppJoint.getProjectTuning(); - ProjectPublishHook[] projectPublishHooks = schedulerAppJoint.getProjectPublishHooks(); - // TODO: 2019/9/24 try catch 下载json要挪到parser去 - //1.封装DSSProject - DSSProject dssProject = projectMapper.selectProjectByVersionID(projectVersionID); - dssProject.setUserName(dssUserMapper.getuserName(dssProject.getUserID())); - logger.info(userName + "-开始发布工程：" + dssProject.getName() + "版本ID为：" + projectVersionID); - ArrayList dssFlows = new ArrayList<>(); - List dssFlowVersionList = flowMapper.listLatestRootFlowVersionByProjectVersionID(projectVersionID); - for (DSSFlowVersion dssFlowVersion : dssFlowVersionList) { - DSSFlow dssFlow = flowMapper.selectFlowByID(dssFlowVersion.getFlowID()); - String json = (String) bmlService.query(userName, dssFlowVersion.getJsonPath(), dssFlowVersion.getVersion()).get("string"); - if (!dssFlow.getHasSaved()) { - logger.info("工作流{}从未保存过，忽略", dssFlow.getName()); - } else if (StringUtils.isNotBlank(json)) { - dssFlowVersion.setJson(json); - dssFlow.setLatestVersion(dssFlowVersion); - createPublishProject(userName, dssFlowVersion.getFlowID(), dssFlow, projectVersionID); - dssFlows.add(dssFlow); - } else { - String warnMsg = String.format(DSSServerConstant.PUBLISH_FLOW_REPORT_FORMATE, dssFlow.getName(), dssFlowVersion.getVersion()); - logger.info(warnMsg); - throw new DSSErrorException(90013, warnMsg); - } - } - if (dssFlows.isEmpty()) throw new DSSErrorException(90007, "该工程没有可以发布的工作流,请检查工作流是否都为空"); - dssProject.setFlows(dssFlows); - //2.封装DSSProject完成，开始发布 - SchedulerProject schedulerProject = projectParser.parseProject(dssProject); - projectTuning.tuningSchedulerProject(schedulerProject); - Stream.of(projectPublishHooks).forEach(DSSExceptionUtils.handling(hook -> hook.prePublish(schedulerProject))); - (schedulerAppJoint.getProjectService()).publishProject(schedulerProject, schedulerAppJoint.getSecurityService().login(userName)); - Stream.of(projectPublishHooks).forEach(DSSExceptionUtils.handling(hook -> hook.postPublish(schedulerProject))); - //3.发布完成后复制工程 - DSSProjectVersion dssProjectVersion = projectMapper.selectProjectVersionByID(projectVersionID); - copyProjectVersionMax(projectVersionID, dssProjectVersion, dssProjectVersion, userName, null); - }else { - logger.error("SchedulerAppJoint is null"); - throw new DSSErrorException(90014, "SchedulerAppJoint is null"); - } - } - - @Override - public Long createPublishHistory(String comment, Long creatorID, Long projectVersionID) { - DSSProjectPublishHistory dssProjectPublishHistory = new DSSProjectPublishHistory(); - dssProjectPublishHistory.setComment(comment); - dssProjectPublishHistory.setCreateID(creatorID); - dssProjectPublishHistory.setCreateTime(new Date()); - dssProjectPublishHistory.setUpdateTime(new Date()); - dssProjectPublishHistory.setProjectVersionID(projectVersionID); - dssProjectPublishHistory.setState(0); - projectMapper.insertPublishHistory(dssProjectPublishHistory); - return dssProjectPublishHistory.getId(); - } - - @Override - public void updatePublishHistory(Long projectVersionID, Integer status, Date updateTime) { - projectMapper.updatePublishHistoryState(projectVersionID, status); - } - - @Override - public DSSProjectPublishHistory getPublishHistoryByID(Long projectVersionID) { - return projectMapper.selectProjectPublishHistoryByProjectVersionID(projectVersionID); - } - - @Override - public DSSProject getExecutionDSSProject(RequestDSSProject requestDSSProject) throws DSSErrorException { - DSSFlow dssFlow = flowService.getOneVersionFlow(requestDSSProject.flowId(), requestDSSProject.version(), requestDSSProject.projectVersionId()); - DSSProject dssProject = projectMapper.selectProjectByVersionID(requestDSSProject.projectVersionId()); - dssProject.setUserName(dssUserMapper.getuserName(dssProject.getUserID())); - DSSFlow returnFlow = recursiveGenerateParentFlow(dssFlow, requestDSSProject); - dssProject.setFlows(Arrays.asList(returnFlow)); - return dssProject; - } - - @Override - public Long getAppjointProjectID(Long projectID, String nodeType) { - // TODO: 2019/11/15 判断工程在dss中存在 - Application applicationbyNodeType = applicationService.getApplicationbyNodeType(nodeType); - Long appjointProjectID = projectMapper.getAppjointProjectID(projectID, applicationbyNodeType.getId()); - return appjointProjectID == null? projectID:appjointProjectID; - } - - @Override - public Long getAppjointProjectIDByApplicationName(Long projectID, String applicationName) { - Long appjointProjectID = projectMapper.getAppjointProjectID(projectID, applicationService.getApplication(applicationName).getId()); - return appjointProjectID == null? projectID:appjointProjectID; - } - - private DSSFlow recursiveGenerateParentFlow(DSSFlow dssFlow, RequestDSSProject requestDSSProject) throws DSSErrorException { - DSSFlow returnFlow = null; - Long parentFlowID = flowService.getParentFlowID(dssFlow.getId()); - if(parentFlowID != null){ - //对于当前执行的工作流的父工作流，直接找其最新的版本 - DSSFlow parentFlow = flowService.getLatestVersionFlow(parentFlowID, requestDSSProject.projectVersionId()); - parentFlow.setChildren(Arrays.asList(dssFlow)); - returnFlow = recursiveGenerateParentFlow(parentFlow, requestDSSProject); - }else { - returnFlow = dssFlow; - } - return returnFlow; - } - - private void createPublishProject(String userName, Long parentFlowID, DSSFlow dssFlowParent, Long projectVersionID) throws DSSErrorException { - List subFlowIDS = flowMapper.selectSubFlowIDByParentFlowID(parentFlowID); - ArrayList dssFlows = new ArrayList<>(); - for (Long subFlowID : subFlowIDS) { - DSSFlowVersion dssFlowVersion = flowService.getLatestVersionByFlowIDAndProjectVersionID(subFlowID, projectVersionID); - if (dssFlowVersion != null) { //subFlowIDS通过flow关联查出来的，但是有可能最新版本的project对已有的flows做了删除 - DSSFlow dssFlow = flowMapper.selectFlowByID(dssFlowVersion.getFlowID()); - String json = (String) bmlService.query(userName, dssFlowVersion.getJsonPath(), dssFlowVersion.getVersion()).get("string"); - if (!dssFlow.getHasSaved()) { - logger.info("工作流{}从未保存过，忽略", dssFlow.getName()); - }else if (StringUtils.isNotBlank(json)){ - dssFlowVersion.setJson(json); - dssFlow.setLatestVersion(dssFlowVersion); - createPublishProject(userName, subFlowID, dssFlow, projectVersionID); - dssFlows.add(dssFlow); - } else { - String warnMsg = String.format(DSSServerConstant.PUBLISH_FLOW_REPORT_FORMATE, dssFlow.getName(), dssFlowVersion.getVersion()); - logger.info(warnMsg); - throw new DSSErrorException(90013, warnMsg); - } - } - } - dssFlowParent.setChildren(dssFlows); - } - - /* - 复制工程 - */ - @Lock - @Transactional(rollbackFor = {DSSErrorException.class, InterruptedException.class,AppJointErrorException.class}) - @Override - public Long copyProject(Long projectVersionID, Long projectID, String projectName, String userName) throws DSSErrorException, InterruptedException, AppJointErrorException { - DSSProject project = projectMapper.selectProjectByID(projectID); - if (StringUtils.isNotEmpty(projectName)) {project.setName(projectName);} - DSSProjectTaxonomyRelation projectTaxonomyRelation = projectTaxonomyMapper.selectProjectTaxonomyRelationByTaxonomyIdOrProjectId(projectID); - //添加至wtss的project数据库，获取projectID - project.setUserName(userName); - if(existSchesulis()){ - createSchedulerProject(project); - } - Map appjointProjectIDAndAppID = createAppjointProject(project); - Long userID = dssUserMapper.getUserID(userName); - //添加至DSS的project数据库，这里的projectID应该不需要自增 - //目前是相同数据库，需要自增id - project.setUserID(userID); - project.setCreateTime(new Date()); - project.setId(null); - projectMapper.addProject(project); - if(!appjointProjectIDAndAppID.isEmpty())projectMapper.addAccessProjectRelation(appjointProjectIDAndAppID,project.getId()); - projectTaxonomyMapper.addProjectTaxonomyRelation(project.getId(), projectTaxonomyRelation.getTaxonomyId(), userID); - DSSProjectVersion maxVersion = projectMapper.selectLatestVersionByProjectID(projectID); - copyProjectVersionMax(maxVersion.getId(), maxVersion, maxVersion, userName, project.getId()); - return project.getId(); - } - - private boolean existSchesulis(){ - return applicationService.getApplication("schedulis") != null; - } - - - /** - * 复制工程的某个历史版本到最新版本,同时复制工作流 - * - * @param projectVersionID - * @param copyVersion 复制的工程版本 - * @param WTSSprojectID 如果是工程复制,则使用此参数. 指定复制工作流的初始版本号. - * 如果是工程版本复制,则设置为null即可. - */ - @Lock(type = LockEnum.ADD) - @Transactional(rollbackFor = {DSSErrorException.class, InterruptedException.class}) - @Override - public void copyProjectVersionMax(Long projectVersionID, DSSProjectVersion maxVersion, DSSProjectVersion copyVersion, String userName, Long WTSSprojectID) throws DSSErrorException, InterruptedException { -// copy project_version - String maxVersionNum = generateNewVersion(maxVersion.getVersion()); - if (null != WTSSprojectID) { - copyVersion.setVersion(DSSServerConstant.DSS_PROJECT_FIRST_VERSION); - copyVersion.setProjectID(WTSSprojectID); - } else { - copyVersion.setVersion(maxVersionNum); - } - Long userID = dssUserMapper.getUserID(userName); - copyVersion.setUpdatorID(userID); - copyVersion.setUpdateTime(new Date()); - List flowVersions = flowMapper.listLastFlowVersionsByProjectVersionID(copyVersion.getId()) - .stream().sorted((o1, o2) -> Integer.valueOf(o1.getFlowID().toString()) - Integer.valueOf(o2.getFlowID().toString())) - .collect(Collectors.toList()); - Long oldProjectVersionID = copyVersion.getId(); - copyVersion.setId(null); - projectMapper.addProjectVersion(copyVersion); - if (copyVersion.getId() == null) {throw new DSSErrorException(90015, "复制工程版本失败");} - Map subAndParentFlowIDMap = new ConcurrentHashMap<>(); - // copy flow - if (null != WTSSprojectID) { - flowVersions.stream().forEach(f -> { - DSSFlow flow = flowMapper.selectFlowByID(f.getFlowID()); - Long parentFlowID = flowMapper.selectParentFlowIDByFlowID(flow.getId()); - if (parentFlowID != null) {subAndParentFlowIDMap.put(flow.getId(), parentFlowID);} - }); - for (DSSFlowVersion fv : flowVersions) { - // 添加所有父子到map中 - DSSFlow flow = flowMapper.selectFlowByID(fv.getFlowID()); - flow.setCreatorID(userID); - flow.setName(flow.getName()); - flow.setProjectID(copyVersion.getProjectID()); - flow.setCreateTime(new Date()); - - Long taxonomyID = flowTaxonomyMapper.selectTaxonomyIDByFlowID(flow.getId()); - DSSFlowTaxonomy flowTaxonomy = flowTaxonomyMapper.selectFlowTaxonomyByID(taxonomyID); - //新增flow相关数据 - fv.setOldFlowID(flow.getId()); - flow.setId(null); - flowMapper.insertFlow(flow); - if (null == flow.getId()) {throw new DSSErrorException(90016, "复制工作流失败");} - for (Map.Entry entry : subAndParentFlowIDMap.entrySet()) { - if (entry.getValue().equals(fv.getFlowID())){subAndParentFlowIDMap.put(entry.getKey(), flow.getId());} - if (entry.getKey().equals(fv.getFlowID())){subAndParentFlowIDMap.put(flow.getId(), entry.getValue());} - } - if (flowTaxonomy.getProjectID() != -1 && (!flowTaxonomy.getProjectID().equals(copyVersion.getProjectID()))) { - flowTaxonomy.setProjectID(copyVersion.getProjectID()); - flowTaxonomy.setCreateTime(new Date()); - flowTaxonomy.setUpdateTime(new Date()); - flowTaxonomy.setProjectID(copyVersion.getUpdatorID()); - flowTaxonomyMapper.insertFlowTaxonomy(flowTaxonomy); - } - if (null != taxonomyID){flowTaxonomyMapper.insertFlowTaxonomyRelation(flowTaxonomy.getId(), flow.getId());} - fv.setFlowID(flow.getId()); - } - for (DSSFlowVersion fv : flowVersions) { - if (subAndParentFlowIDMap.get(fv.getFlowID()) != null){flowMapper.insertFlowRelation(fv.getFlowID(), subAndParentFlowIDMap.get(fv.getFlowID()));} - } - } -// copy flow_version - if (flowVersions.size() > 0) { - ThreadPoolTool tool = new ThreadPoolTool(5, flowVersions); - tool.setCallBack(new ThreadPoolTool.CallBack() { - @Override - public void method(List flowVersions) { - for (DSSFlowVersion fv : flowVersions) { - // 工作流版本的json文件，都是需要重新上传到bml - Map bmlQueryMap = bmlService.download(dssUserMapper.getuserName(fv.getUpdatorID()), fv.getJsonPath(), fv.getVersion()); - BufferedReader bufferedReader = new BufferedReader(new InputStreamReader((InputStream) bmlQueryMap.get("is"))); - StringBuilder sb = new StringBuilder(); - String s = null; - try { - while ((s = bufferedReader.readLine()) != null) { - sb.append(s); - sb.append("\n"); - } - String json = sb.toString().trim(); - List newflowIDs = getSubNewFlowID(subAndParentFlowIDMap, fv.getFlowID()); - List oldFlowIDs = getOldSubFlowID(subAndParentFlowIDMap, fv.getFlowID()); - if (oldFlowIDs != null && newflowIDs != null) { - logger.info("replace之前:" + json); - if (json != null) { - for (int i = 0; i < newflowIDs.size(); i++) { - json = json.replace(DSSServerConstant.EMVEDDEDFLOWID + oldFlowIDs.get(i), DSSServerConstant.EMVEDDEDFLOWID + newflowIDs.get(i)); - } - } - logger.info("replace之后:" + json); - } - if (json != null) { - InputStream in_nocode = new ByteArrayInputStream(json.getBytes()); - Map bmlReturnMap = bmlService.upload(userName, in_nocode, UUID.randomUUID().toString() + ".json"); - fv.setProjectVersionID(copyVersion.getId()); - fv.setUpdateTime(new Date()); - fv.setUpdatorID(copyVersion.getUpdatorID()); - String oldFlowVersion = fv.getVersion(); - fv.setJsonPath(bmlReturnMap.get("resourceId").toString()); - fv.setVersion(bmlReturnMap.get("version").toString()); - fv.setSource("Copy from Old { ProjectVersionID : " + oldProjectVersionID + " ,FlowID:" + fv.getOldFlowID() + ",Version:" + oldFlowVersion + "} to New { ProjectVersionID:" + fv.getProjectVersionID() + " ,FlowID:" + fv.getFlowID() + ",Version:" + fv.getVersion() + "}"); - bufferedReader.close(); - in_nocode.close(); - } - } catch (IOException e) { - logger.error("工作流复制IO异常", e); - } - - } - } - }); - tool.excute(); - } - if (flowVersions != null && flowVersions.size() > 0) { - flowVersions.stream().forEach(f -> { - logger.info("jsonPaht:{},oldeFlowID:{},newFlowID", f.getJsonPath(), f.getOldFlowID(), f.getFlowID()); - }); - flowMapper.batchInsertFlowVersion(flowVersions); - } - } - - private String generateNewVersion(String version){ - int next = Integer.parseInt(version.substring(1, version.length())) + 1; - return DSSServerConstant.VERSION_PREFIX + String.format(DSSServerConstant.VERSION_FORMAT, next); - } - - - private List getOldSubFlowID(Map subAndParentFlowIDMap, Long flowID) { - List> collect = subAndParentFlowIDMap.entrySet().stream().filter(f -> f.getValue().equals(flowID)).collect(Collectors.toList()); - if (collect.isEmpty()) return null; - int size = collect.size() / 2; - ArrayList longs = new ArrayList<>(); - List> sortedCollections = collect.stream().sorted((o, n) -> { - return Integer.valueOf(o.getKey().toString()) - Integer.valueOf(n.getKey().toString()); - }).collect(Collectors.toList()); - sortedCollections.subList(0, size).stream().forEach(p -> longs.add(p.getKey())); - return longs; - /*if(collect.isEmpty()){ - return UUID.randomUUID().toString(); - }else { - return collect.get(0).getKey() - collect.get(1).getKey() <0? Constant.EMVEDDEDFLOWID +collect.get(0).getKey():Constant.EMVEDDEDFLOWID +collect.get(1).getKey(); - }*/ - } - - private List getSubNewFlowID(Map subAndParentFlowIDMap, Long flowID) { - List> collect = subAndParentFlowIDMap.entrySet().stream().filter(f -> f.getValue().equals(flowID)).collect(Collectors.toList()); - if (collect.isEmpty()) return null; - int size = collect.size() / 2; - ArrayList longs = new ArrayList<>(); - List> sortedCollections = collect.stream().sorted((o, n) -> { - return Integer.valueOf(n.getKey().toString()) - Integer.valueOf(o.getKey().toString()); - }).collect(Collectors.toList()); - sortedCollections.subList(0, size).stream().forEach(p -> longs.add(p.getKey())); - return longs.stream().sorted((o1, o2) -> { - return Integer.valueOf(o1.toString()) - Integer.valueOf(o2.toString()); - }).collect(Collectors.toList()); - } - - - private SchedulerAppJoint getSchedulerAppJoint(){ - if(schedulerAppJoint == null){ - try { - schedulerAppJoint = (SchedulerAppJoint)applicationService.getAppjoint("schedulis"); - } catch (AppJointErrorException e) { - logger.error("Schedule system init failed!", e); - } - } - return schedulerAppJoint; - } - -} diff --git a/dss-server/src/main/java/com/webank/wedatasphere/dss/server/service/impl/DSSProjectTaxonomyServiceImpl.java b/dss-server/src/main/java/com/webank/wedatasphere/dss/server/service/impl/DSSProjectTaxonomyServiceImpl.java deleted file mode 100644 index 2151151679..0000000000 --- a/dss-server/src/main/java/com/webank/wedatasphere/dss/server/service/impl/DSSProjectTaxonomyServiceImpl.java +++ /dev/null @@ -1,132 +0,0 @@ -/* - * Copyright 2019 WeBank - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package com.webank.wedatasphere.dss.server.service.impl; - - -import com.webank.wedatasphere.dss.common.entity.project.DSSProject; -import com.webank.wedatasphere.dss.server.dao.ProjectTaxonomyMapper; -import com.webank.wedatasphere.dss.server.entity.DSSProjectTaxonomy; -import com.webank.wedatasphere.dss.server.service.DSSProjectService; -import com.webank.wedatasphere.dss.server.service.DSSProjectTaxonomyService; -import com.webank.wedatasphere.dss.common.exception.DSSErrorException; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.dao.DuplicateKeyException; -import org.springframework.stereotype.Service; - -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; - - -@Service -public class DSSProjectTaxonomyServiceImpl implements DSSProjectTaxonomyService { - - private Logger logger = LoggerFactory.getLogger(this.getClass()); - - @Autowired - private ProjectTaxonomyMapper projectTaxonomyMapper; - @Autowired - private DSSProjectService projectService; - - @Override - public DSSProjectTaxonomy getProjectTaxonomyByID(Long id) { - return projectTaxonomyMapper.selectProjectTaxonomyByID(id); - } - - @Override - public List listProjectTaxonomyByUser(String userName) { - return projectTaxonomyMapper.listProjectTaxonomyByUser(userName); - } - - - private List listProjectIDByTaxonomyID(Long taxonomyID, String userName) { - return projectTaxonomyMapper.listProjectIDByTaxonomyID(taxonomyID, userName); - } - - - @Override - public List listAllProjectTaxonomy(String userName, Long workspaceId) { - List dssProjectTaxonomies = listProjectTaxonomyByUser(userName); - for (DSSProjectTaxonomy dssProjectTaxonomy : dssProjectTaxonomies) { - List projectIDs = listProjectIDByTaxonomyID(dssProjectTaxonomy.getId(), userName); - ArrayList dssProjectList = new ArrayList<>(); - for (Long projectID : projectIDs) { - DSSProject dssProject = projectService.getLatestVersionProject(projectID); - // 只选择返回属于这个workspace的project，(某些用户拥有多个workspace的不同project) - if(workspaceId.equals(dssProject.getWorkspaceId())) { - dssProjectList.add(dssProject); - } - } - dssProjectTaxonomy.setDssProjectList(dssProjectList); - } - return dssProjectTaxonomies; - } - - @Override - public List listProjectTaxonomy(Long taxonomyID, String userName) { - DSSProjectTaxonomy dssProjectTaxonomy = getProjectTaxonomyByID(taxonomyID); - List projectIDs = listProjectIDByTaxonomyID(dssProjectTaxonomy.getId(), userName); - ArrayList dssProjectList = new ArrayList<>(); - for (Long projectID : projectIDs) { - DSSProject dssProject = projectService.getLatestVersionProject(projectID); - dssProjectList.add(dssProject); - } - dssProjectTaxonomy.setDssProjectList(dssProjectList); - return Arrays.asList(dssProjectTaxonomy); - } - - @Override - public void addProjectTaxonomy(DSSProjectTaxonomy dssProjectTaxonomy) throws DSSErrorException { - try { - projectTaxonomyMapper.insertProjectTaxonomy(dssProjectTaxonomy); - } catch (DuplicateKeyException e) { - logger.info(e.getMessage()); - throw new DSSErrorException(90004, "工程分类名不能重复"); - } - } - - @Override - public void updateProjectTaxonomy(DSSProjectTaxonomy dssProjectTaxonomy) throws DSSErrorException { - try { - projectTaxonomyMapper.updateProjectTaxonomy(dssProjectTaxonomy); - } catch (DuplicateKeyException e) { - logger.info(e.getMessage()); - throw new DSSErrorException(90004, "工程分类名不能重复"); - } - } - - @Override - public boolean hasProjects(Long projectTaxonomyID) { - Long count = projectTaxonomyMapper.hasProjects(projectTaxonomyID); - return count != 0; - } - - @Override - public void deleteProjectTaxonomy(Long projectTaxonomyID) { - projectTaxonomyMapper.deleteProjectTaxonomy(projectTaxonomyID); - } - - @Override - public void updateProjectTaxonomyRelation(Long projectID, Long taxonomyID) { - if (taxonomyID != null) { - projectTaxonomyMapper.updateProjectTaxonomyRelation(projectID, taxonomyID); - } - } -} diff --git a/dss-server/src/main/java/com/webank/wedatasphere/dss/server/service/impl/DSSUserServiceImpl.java b/dss-server/src/main/java/com/webank/wedatasphere/dss/server/service/impl/DSSUserServiceImpl.java deleted file mode 100644 index 60973c98f6..0000000000 --- a/dss-server/src/main/java/com/webank/wedatasphere/dss/server/service/impl/DSSUserServiceImpl.java +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright 2019 WeBank - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package com.webank.wedatasphere.dss.server.service.impl; - - -import com.webank.wedatasphere.dss.application.entity.DSSUser; -import com.webank.wedatasphere.dss.server.dao.DSSUserMapper; -import com.webank.wedatasphere.dss.server.service.DSSUserService; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.stereotype.Service; - - -@Service -public class DSSUserServiceImpl implements DSSUserService { - @Autowired - private DSSUserMapper dssUserMapper; - - @Override - public Long getUserID(String userName) { - return dssUserMapper.getUserID(userName); - } -} diff --git a/dss-server/src/main/java/com/webank/wedatasphere/dss/server/service/impl/DSSWorkspaceServiceImpl.java b/dss-server/src/main/java/com/webank/wedatasphere/dss/server/service/impl/DSSWorkspaceServiceImpl.java deleted file mode 100644 index 19a27befb1..0000000000 --- a/dss-server/src/main/java/com/webank/wedatasphere/dss/server/service/impl/DSSWorkspaceServiceImpl.java +++ /dev/null @@ -1,155 +0,0 @@ -/* - * Copyright 2019 WeBank - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package com.webank.wedatasphere.dss.server.service.impl; - -import com.webank.wedatasphere.dss.server.constant.DSSServerConstant; -import com.webank.wedatasphere.dss.server.dao.WorkspaceMapper; -import com.webank.wedatasphere.dss.server.dto.response.*; -import com.webank.wedatasphere.dss.server.entity.*; -import com.webank.wedatasphere.dss.server.service.DSSWorkspaceService; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.stereotype.Service; - -import java.util.ArrayList; -import java.util.List; - -/** - * Created by schumiyi on 2020/6/22 - */ -@Service -public class DSSWorkspaceServiceImpl implements DSSWorkspaceService { - - @Autowired - private WorkspaceMapper workspaceMapper; - - @Override - public List getWorkspaces() { - - return workspaceMapper.getWorkspaces(); - } - - @Override - public DSSWorkspace getWorkspacesById(Long id) { - return workspaceMapper.getWorkspaceById(id); - } - - - @Override - public Long addWorkspace(String userName, String name, String department, String label, String description) { - DSSWorkspace dssWorkspace = new DSSWorkspace(); - dssWorkspace.setName(name); - dssWorkspace.setDepartment(department); - dssWorkspace.setDescription(description); - dssWorkspace.setLabel(label); - dssWorkspace.setCreateBy(userName); - dssWorkspace.setSource(DSSServerConstant.DSS_WORKSPACE_SOURCE); - dssWorkspace.setLastUpdateUser(userName); - workspaceMapper.addWorkSpace(dssWorkspace); - return dssWorkspace.getId(); - } - - @Override - public boolean existWorkspaceName(String name) { - return !workspaceMapper.findByWorkspaceName(name).isEmpty(); - } - - @Override - public List getWorkSpaceDepartments() { - // TODO: service层和dao层完善 - WorkspaceDepartmentVo dp = new WorkspaceDepartmentVo(); - dp.setId(1L); - dp.setName("应用开发组"); - WorkspaceDepartmentVo di = new WorkspaceDepartmentVo(); - di.setId(2L); - di.setName("平台研发组"); - List departments = new ArrayList<>(); - departments.add(dp); - departments.add(di); - return departments; - } - - @Override - public List getHomepageDemos(boolean isChinese) { - List demoMenuVos = isChinese ? workspaceMapper.getHomepageDemoMenusCn() : workspaceMapper.getHomepageDemoMenusEn(); - for (HomepageDemoMenuVo demoMenuVo : demoMenuVos) { - Long menuId = demoMenuVo.getId(); - List demoInstanceVos = isChinese ? workspaceMapper.getHomepageInstancesByMenuIdCn(menuId) : workspaceMapper.getHomepageInstancesByMenuIdEn(menuId); - demoMenuVo.setDemoInstances(demoInstanceVos); - } - return demoMenuVos; - } - - @Override - public List getHomepageVideos(boolean isChinese) { - return isChinese ? workspaceMapper.getHomepageVideosCn() : workspaceMapper.getHomepageVideosEn(); - } - - @Override - public List getWorkspaceManagements(Long workspaceId, String username, boolean isChinese) { - if (!isAdminUser(workspaceId, username)) { - return new ArrayList<>(); - } - List managementMenuVos = isChinese ? workspaceMapper.getManagementMenuCn() : workspaceMapper.getManagementMenuEn(); - return getMenuAppInstances(managementMenuVos, isChinese); - } - - private List getMenuAppInstances(List menuVos, boolean isChinese) { - for (OnestopMenuVo menuVo : menuVos) { - Long menuId = menuVo.getId(); - List menuAppInstanceVos = isChinese ? workspaceMapper.getMenuAppInstancesCn(menuId) : workspaceMapper.getMenuAppInstancesEn(menuId); - menuVo.setAppInstances(menuAppInstanceVos); - } - return menuVos; - } - - @Override - public List getWorkspaceApplications(Long workspaceId, String username, boolean isChinese) { - List applicationMenuVos = isChinese ? workspaceMapper.getApplicationMenuCn() : workspaceMapper.getApplicationMenuEn(); - return getMenuAppInstances(applicationMenuVos, isChinese); - } - - @Override - public List getWorkspaceFavorites(Long workspaceId, String username, boolean isChinese) { - return isChinese ? workspaceMapper.getWorkspaceFavoritesCn(username, workspaceId) : workspaceMapper.getWorkspaceFavoritesEn(username, workspaceId); - } - - @Override - public Long addFavorite(String username, Long workspaceId, Long menuApplicationId) { - DSSFavorite dssFavorite = new DSSFavorite(); - dssFavorite.setUsername(username); - dssFavorite.setWorkspaceId(workspaceId); - dssFavorite.setMenuApplicationId(menuApplicationId); - // todo: order will from the front end - dssFavorite.setOrder(1); - dssFavorite.setCreateBy(username); - dssFavorite.setLastUpdateUser(username); - workspaceMapper.addFavorite(dssFavorite); - return dssFavorite.getId(); - } - - @Override - public Long deleteFavorite(String username, Long favouritesId) { - workspaceMapper.deleteFavorite(favouritesId); - return favouritesId; - } - - private boolean isAdminUser(Long workspaceId, String username) { - DSSWorkspace workspace = workspaceMapper.getWorkspaceById(workspaceId); - return username != null && workspace != null && username.equals(workspace.getCreateBy()); - } -} diff --git a/dss-server/src/main/java/com/webank/wedatasphere/dss/server/util/ThreadPoolTool.java b/dss-server/src/main/java/com/webank/wedatasphere/dss/server/util/ThreadPoolTool.java deleted file mode 100644 index fe0f40fc8c..0000000000 --- a/dss-server/src/main/java/com/webank/wedatasphere/dss/server/util/ThreadPoolTool.java +++ /dev/null @@ -1,132 +0,0 @@ -/* - * Copyright 2019 WeBank - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package com.webank.wedatasphere.dss.server.util; - - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - - -import java.util.List; -import java.util.concurrent.CountDownLatch; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; - -public class ThreadPoolTool { - - private Logger logger = LoggerFactory.getLogger(this.getClass()); - - //单个线程处理的数据量 - private int singleCount; - //处理的总数据量 - private int listSize; - //开启的线程数 - private int runSize; - //操作的数据集 - private List list; - //计数器 - private CountDownLatch begin,end; - //线程池 - private ExecutorService executorService; - //回调 - private CallBack callBack; - - - public void setCallBack(CallBack callBack) { - this.callBack = callBack; - } - - public ThreadPoolTool(int singleCount, List list){ - this.singleCount = singleCount; - this.list = list; - if (list != null){ - this.listSize = list.size(); - this.runSize = (this.listSize/this.singleCount) + 1; - } - } - - public void excute() throws InterruptedException { - executorService = Executors.newFixedThreadPool(runSize); - begin = new CountDownLatch(1); - end = new CountDownLatch(runSize); - //创建线程 - int startIndex = 0; - int endIndex = 0; - List newList = null; - for (int i = 0; i < runSize; i++) { - //计算每个线程对应的数据 - if (i < (runSize - 1)){ - startIndex = i * singleCount; - endIndex = (i + 1) * singleCount; - newList = list.subList(startIndex,endIndex); - }else { - startIndex = i * singleCount; - endIndex = listSize; - newList = list.subList(startIndex,endIndex); - } - //创建线程类处理数据 - MyThread myThread = new MyThread(newList, begin, end) { - @Override - public void method(List list) { - callBack.method(list); - } - }; - //执行线程 - executorService.execute(myThread); - } - //计数器减一 - begin.countDown(); - end.await(); - //关闭线程池 - executorService.shutdown(); - } - - //抽象线程类 - public abstract class MyThread implements Runnable{ - - private List list; - private CountDownLatch begin,end; - - public MyThread(List list, CountDownLatch begin, CountDownLatch end){ - this.list = list; - this.begin = begin; - this.end = end; - } - - @Override - public void run() { - try { - //执行程序 - method(list); - begin.await(); - } catch (InterruptedException e) { - logger.error(e.getMessage()); - Thread.currentThread().interrupt(); - }finally { - //计数器减一 - end.countDown(); - } - } - public abstract void method(List list); - } - - //回调接口定义 - public interface CallBack{ - public void method(List list); - } -} diff --git a/dss-server/src/main/resources/application.yml b/dss-server/src/main/resources/application.yml deleted file mode 100644 index 419506c3f8..0000000000 --- a/dss-server/src/main/resources/application.yml +++ /dev/null @@ -1,22 +0,0 @@ -server: - port: 9004 -spring: - application: - name: dss-server - - -eureka: - client: - serviceUrl: - defaultZone: http://127.0.0.1:20303/eureka/ - instance: - metadata-map: - test: wedatasphere - -management: - endpoints: - web: - exposure: - include: refresh,info -logging: - config: classpath:log4j2.xml diff --git a/dss-server/src/main/resources/default/AddschedulerUser.sh b/dss-server/src/main/resources/default/AddschedulerUser.sh deleted file mode 100644 index 116c2d96b8..0000000000 --- a/dss-server/src/main/resources/default/AddschedulerUser.sh +++ /dev/null @@ -1,13 +0,0 @@ -#!/bin/bash - -user=$1 -password=$2 -installDir=$3 - - -if grep -i "^${user}=" $installDir/token.properties; - then - sed -i '' "s/^$user=.*/$user=$password/" $installDir/token.properties -else - echo "$user=$password" >> $installDir/token.properties -fi diff --git a/dss-server/src/main/resources/default/CreateLdapAccount.sh b/dss-server/src/main/resources/default/CreateLdapAccount.sh deleted file mode 100644 index 1318ab16b7..0000000000 --- a/dss-server/src/main/resources/default/CreateLdapAccount.sh +++ /dev/null @@ -1,18 +0,0 @@ -#!/bin/bash -source /etc/profile -server_host=$1 -server_login_user=$2 -server_login_password=$3 -server_python_path=$4 -server_ldap_source_path=$5 -ldap_user=$6 -ldap_password=$7 -echo "$server_login_password ssh $server_login_user@$server_host sudo python $server_python_path add_with_pw $ldap_user -p $ldap_password" -sshpass -p $server_login_password ssh $server_login_user@$server_host "sudo python $server_python_path add_with_pw $ldap_user -p $ldap_password" -#sshpass -p $server_login_password ssh $server_login_user@$server_host "sudo su - root -c 'source /etc/profile && source $server_ldap_source_path && sudo python $server_python_path add_with_pw $ldap_user -p $ldap_password && deactivate'" - -echo "******************LDAP USER CREATED***********************" - - - - diff --git a/dss-server/src/main/resources/default/CreateLinuxUser.sh b/dss-server/src/main/resources/default/CreateLinuxUser.sh deleted file mode 100644 index fabcac1c5a..0000000000 --- a/dss-server/src/main/resources/default/CreateLinuxUser.sh +++ /dev/null @@ -1,35 +0,0 @@ -#!/bin/bash - -source /etc/profile -server_user_strs=$1 -echo $server_user_strs -add_user_name=$2 -add_user_password=$3 -server_user_array=(${server_user_strs//,/ }) -for server_user_str in ${server_user_array[@]} -do - server_user_info=(${server_user_str//#/ }) - server_host=${server_user_info[0]} - server_user_name=${server_user_info[1]} - server_user_password=${server_user_info[2]} - echo "${server_host},${server_user_name},${server_user_password}" - - sudo sshpass -p $server_user_password ssh -o ConnectTimeout=1 $server_user_name@$server_host "echo success" - [ $? -ne 0 ] && echo "登录主机${server_host}失败" && exit 254 -done - -echo "************服务器网络校验通过,开始创建用户*****************" - -for server_user_str in ${server_user_array[@]} -do - - server_user_info=(${server_user_str//#/ }) - server_host=${server_user_info[0]} - server_user_name=${server_user_info[1]} - server_user_password=${server_user_info[2]} - - #sshpass -p $server_user_password ssh $server_user_name@$server_host "sudo useradd $add_user_name && echo $add_user_password |sudo -i passwd --stdin $add_user_name" - sshpass -p $server_user_password ssh $server_user_name@$server_host "sudo useradd $add_user_name -s /sbin/nologin" - - [ $? -ne 0 ] && echo "创建用户失败：${host}失败" && exit 254 -done diff --git a/dss-server/src/main/resources/default/HdfsPath.sh b/dss-server/src/main/resources/default/HdfsPath.sh deleted file mode 100644 index cd765486c5..0000000000 --- a/dss-server/src/main/resources/default/HdfsPath.sh +++ /dev/null @@ -1,6 +0,0 @@ -#!/bin/bash - -user=$1 -dir=$2 -hdfs dfs -mkdir -p $dir -hdfs dfs -chown $user:$user $dir \ No newline at end of file diff --git a/dss-server/src/main/resources/default/LinuxPath.sh b/dss-server/src/main/resources/default/LinuxPath.sh deleted file mode 100644 index bc55860b10..0000000000 --- a/dss-server/src/main/resources/default/LinuxPath.sh +++ /dev/null @@ -1,7 +0,0 @@ -#!/bin/bash - -user=$1 -dir=$2 -echo $1 $2; -sudo mkdir -p $dir -sudo chown $user:$user $dir \ No newline at end of file diff --git a/dss-server/src/main/resources/linkis.properties b/dss-server/src/main/resources/linkis.properties deleted file mode 100644 index bd39846ef3..0000000000 --- a/dss-server/src/main/resources/linkis.properties +++ /dev/null @@ -1,71 +0,0 @@ -# -# Copyright 2019 WeBank -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# - -wds.linkis.test.mode=true - -wds.linkis.server.mybatis.datasource.url=jdbc:mysql://0.0.0.1:3306/linkis?characterEncoding=UTF-8 - -wds.linkis.server.mybatis.datasource.username= - -wds.linkis.server.mybatis.datasource.password= - - -wds.linkis.log.clear=true -wds.linkis.server.version=v1 - -##restful -wds.linkis.server.restful.scan.packages=com.webank.wedatasphere.dss.server.restful,com.webank.wedatasphere.dss.application.restful - -##mybatis -wds.linkis.server.mybatis.mapperLocations=classpath:com/webank/wedatasphere/dss/server/dao/impl/*.xml,classpath*:com/webank/wedatasphere/dss/application/dao/impl/*.xml - -wds.linkis.server.mybatis.typeAliasesPackage=com.webank.wedatasphere.dss.server.entity,com.webank.wedatasphere.dss.application.entity - -wds.linkis.server.mybatis.BasePackage=com.webank.wedatasphere.dss.server.dao,com.webank.wedatasphere.dss.application.dao - -##azkaban -wds.dss.appjoint.scheduler.azkaban.address=http://0.0.0.0:8081 - -wds.linkis.gateway.ip=127.0.0.1 -wds.linkis.gateway.port=9001 - -wds.dss.appjoint.scheduler.project.store.dir=file:///appcom/tmp/wds/scheduler -wds.linkis.super.user.name=root -wds.linkis.workspace.user.root.path=file:///tmp/linkis/ -wds.linkis.hdfs.user.root.path=hdfs:///tmp/linkis -wds.linkis.result.set.root.path=hdfs:///tmp/linkis -wds.linkis.scheduler.path=file:///appcom/tmp/wds/scheduler -wds.linkis.user.path=hdfs:///user -wds.linkis.dss.install.dir=/usr/local/dss_linkis/dss/dss-server -wds.linkis.azkaban.install.dir=/usr/local/dss_linkis/azkaban - -wds.linkis.metastore.hive.hdfs.base.path=/user/hive/warehouse -wds.linkis.metastore.script.path=default/Metastore.sh -wds.linkis.metastore.db.tail=_default - -wds.linkis.kerberos.realm= -wds.linkis.kerberos.admin= -wds.linkis.kerberos.enable.switch=0 -wds.linkis.kerberos.script.path=default/Kerberos.sh -wds.linkis.kerberos.keytab.path=/etc/security/keytabs -wds.linkis.kerberos.kdc.node= -wds.linkis.kerberos.kdc.user.name= -wds.linkis.kerberos.kdc.user.password= -wds.linkis.kerberos.ssh.port=22 -wds.dss.deploy.path=/usr/local/dss_linkis -wds.dss.user.account.command.class=com.webank.wedatasphpere.dss.user.service.impl.LinuxUserCommand,com.webank.wedatasphpere.dss.user.service.impl.KerberosCommand,com.webank.wedatasphpere.dss.user.service.impl.LdapCommand,com.webank.wedatasphpere.dss.user.service.impl.WorkspaceCommand,com.webank.wedatasphpere.dss.user.service.impl.MetastoreCommand,com.webank.wedatasphpere.dss.user.service.impl.AzkabanCommand -wds.dss.scheduler.url=/schedule/system \ No newline at end of file diff --git a/dss-server/src/main/resources/log4j.properties b/dss-server/src/main/resources/log4j.properties deleted file mode 100644 index 0807e60877..0000000000 --- a/dss-server/src/main/resources/log4j.properties +++ /dev/null @@ -1,37 +0,0 @@ -# -# Copyright 2019 WeBank -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# - -### set log levels ### - -log4j.rootCategory=INFO,console - -log4j.appender.console=org.apache.log4j.ConsoleAppender -log4j.appender.console.Threshold=INFO -log4j.appender.console.layout=org.apache.log4j.PatternLayout -#log4j.appender.console.layout.ConversionPattern= %d{ISO8601} %-5p (%t) [%F:%M(%L)] - %m%n -log4j.appender.console.layout.ConversionPattern= %d{ISO8601} %-5p (%t) %p %c{1} - %m%n - - -log4j.appender.com.webank.bdp.ide.core=org.apache.log4j.DailyRollingFileAppender -log4j.appender.com.webank.bdp.ide.core.Threshold=INFO -log4j.additivity.com.webank.bdp.ide.core=false -log4j.appender.com.webank.bdp.ide.core.layout=org.apache.log4j.PatternLayout -log4j.appender.com.webank.bdp.ide.core.Append=true -log4j.appender.com.webank.bdp.ide.core.File=logs/linkis.log -log4j.appender.com.webank.bdp.ide.core.layout.ConversionPattern= %d{ISO8601} %-5p (%t) [%F:%M(%L)] - %m%n - -log4j.logger.org.springframework=INFO diff --git a/dss-server/src/main/resources/log4j2.xml b/dss-server/src/main/resources/log4j2.xml deleted file mode 100644 index 3923cd9f39..0000000000 --- a/dss-server/src/main/resources/log4j2.xml +++ /dev/null @@ -1,39 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - diff --git a/dss-server/src/main/resources/token.properties b/dss-server/src/main/resources/token.properties deleted file mode 100644 index 14963db725..0000000000 --- a/dss-server/src/main/resources/token.properties +++ /dev/null @@ -1,2 +0,0 @@ -#${userName}=${password} -hadoop=hadoop diff --git a/dss-server/src/main/scala/com/webank/wedatasphere/dss/server/crumb/CrumbFactory.scala b/dss-server/src/main/scala/com/webank/wedatasphere/dss/server/crumb/CrumbFactory.scala deleted file mode 100644 index 35df203a7a..0000000000 --- a/dss-server/src/main/scala/com/webank/wedatasphere/dss/server/crumb/CrumbFactory.scala +++ /dev/null @@ -1,144 +0,0 @@ -/* - * Copyright 2019 WeBank - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package com.webank.wedatasphere.dss.server.crumb - -import com.webank.wedatasphere.dss.server.entity.CrumbType.CrumbType -import com.webank.wedatasphere.dss.server.entity.{Crumb, CrumbType, DSSFlowTaxonomy, DSSProjectTaxonomy} -import com.webank.wedatasphere.dss.server.service.impl.{DSSFlowServiceImpl, DSSFlowTaxonomyServiceImpl, DSSProjectServiceImpl, DSSProjectTaxonomyServiceImpl} -import org.springframework.beans.factory.annotation.Autowired -import org.springframework.stereotype.Component - -import scala.collection.mutable.ArrayBuffer - - -@Component -class CrumbFactory { - @Autowired - private var projectService: DSSProjectServiceImpl = _ - @Autowired - private var projectTaxonomyService:DSSProjectTaxonomyServiceImpl = _ - @Autowired - private var flowService: DSSFlowServiceImpl = _ - @Autowired - private var flowTaxonomyService:DSSFlowTaxonomyServiceImpl = _ - - def createCrumbs(crumbType: CrumbType, params: Array[String]): Array[Crumb] = { - crumbType match { - case CrumbType.All => Array(createIndexCrumb, createAllCrumb) - case CrumbType.SortProject => Array(createIndexCrumb, createAllCrumb, createSortProjectCrumb(params)) - case CrumbType.Project => Array(createIndexCrumb, createAllCrumb, createSortProjectCrumb(subParams(params,Array(1))), - createProjectCrumb(params)) - case CrumbType.SortFlow => Array(createIndexCrumb, createAllCrumb, createSortProjectCrumb(subParams(params,Array(1))), - createProjectCrumb(subParams(params,Array(1,2,4))), createSortFlowCrumb(params)) - case CrumbType.Flow => Array(createIndexCrumb, createAllCrumb, createSortProjectCrumb(subParams(params,Array(1))), - createProjectCrumb(subParams(params,Array(1,2,5))), createSortFlowCrumb(subParams(params,Array(1,2,3,5))), - createFlowCrumb(params)) - } - } - - private def subParams(params: Array[String], index: Array[Int]): Array[String] = { - val subParams = new ArrayBuffer[String] - for (i <- 1 to params.length) { - if (index.contains(i)) subParams += params(i - 1) - } - subParams.toArray - } - - private def createSortProjectCrumb(params: Array[String]): Crumb = { - val paramsMap = QuerParamsParser.toMap(params) - val projectTaxonomy = projectTaxonomyService.getProjectTaxonomyByID(paramsMap.get("projectTaxonomyID").toLong) - new Crumb(CrumbType.SortProject, QuerParamsParser.toQueryString(params), projectTaxonomy.getName, null) - } - - private def createProjectCrumb(params: Array[String]): Crumb = { - val paramsMap = QuerParamsParser.toMap(params) - val project = projectService.getProjectByProjectVersionID(paramsMap.get("projectVersionID").toLong) - new Crumb(CrumbType.Project, QuerParamsParser.toQueryString(params), project.getName, null) - } - - private def createSortFlowCrumb(params: Array[String]): Crumb = { - val paramsMap = QuerParamsParser.toMap(params) - val flowTaxonomy = flowTaxonomyService.getFlowTaxonomyByID(paramsMap.get("flowTaxonomyID").toLong) - new Crumb(CrumbType.SortFlow, QuerParamsParser.toQueryString(params), flowTaxonomy.getName, null) - } - - private def createFlowCrumb(params: Array[String]): Crumb = { - val paramsMap = QuerParamsParser.toMap(params) - val flow = flowService.getFlowByID(paramsMap.get("flowID").toLong) - new Crumb(CrumbType.Flow, QuerParamsParser.toQueryString(params), flow.getName, null) - } - - private def createIndexCrumb: Crumb = { - new Crumb(CrumbType.Index, "", "首页", null) - } - - private def createAllCrumb: Crumb = { - new Crumb(CrumbType.All, "", "工程开发", null) - } - - def createCrumbTree(crumbType: CrumbType, params: java.util.Map[String, String]): Crumb = { - crumbType match { - case CrumbType.All => - case CrumbType.SortProject => - case CrumbType.Project => - case CrumbType.SortFlow => - case CrumbType.Flow => - } - null - } - - def createCrumbData(crumbType: CrumbType, params: java.util.Map[String, String],userName:String): Any = { - crumbType match { - case CrumbType.All =>createAllData(userName, params.get("workspaceId").toLong) - case CrumbType.SortProject =>createSortProjectData(params.get("projectTaxonomyID").toLong,userName) - case CrumbType.Project =>createProjectData(params.get("projectVersionID").toLong,params.get("isRootFlow").toBoolean) - case CrumbType.SortFlow =>createSortFlowData(params.get("projectVersionID").toLong,params.get("flowTaxonomyID").toLong,params.get("isRootFlow").toBoolean) - case CrumbType.Flow =>null - } - } - - private def createAllData(userName:String, workspaceId: Long):java.util.List[DSSProjectTaxonomy] ={ - projectTaxonomyService.listAllProjectTaxonomy(userName, workspaceId) - } - - private def createSortProjectData(projectTaxonomyID:Long,userName:String):java.util.List[DSSProjectTaxonomy] ={ - projectTaxonomyService.listProjectTaxonomy(projectTaxonomyID,userName) - } - - private def createProjectData(projectVersionID:Long,isRootFlow:Boolean):java.util.List[DSSFlowTaxonomy] = { - flowTaxonomyService.listAllFlowTaxonomy(projectVersionID,isRootFlow) - } - - private def createSortFlowData(projectVersionID:Long,flowTaxonomyID:Long,isRootFlow:Boolean):java.util.List[DSSFlowTaxonomy] = { - flowTaxonomyService.listFlowTaxonomy(projectVersionID,flowTaxonomyID:Long,isRootFlow:Boolean) - } -} - -/*object Test { - def main(args: Array[String]): Unit = { - import collection.JavaConversions._ - val a = new util.HashMap[String, String]() - a.put("a", "b") - a += "c" -> "d" - a += "e" -> "f" - a += "g" -> "dh" - a += "cerq" -> "dwer" - val b = new CrumbFactory - print(b.removeParams(a, "565")) - } -}*/ diff --git a/dss-server/src/main/scala/com/webank/wedatasphere/dss/server/crumb/QuerParamsParser.scala b/dss-server/src/main/scala/com/webank/wedatasphere/dss/server/crumb/QuerParamsParser.scala deleted file mode 100644 index 5b979f85f7..0000000000 --- a/dss-server/src/main/scala/com/webank/wedatasphere/dss/server/crumb/QuerParamsParser.scala +++ /dev/null @@ -1,80 +0,0 @@ -/* - * Copyright 2019 WeBank - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package com.webank.wedatasphere.dss.server.crumb - - -import com.webank.wedatasphere.dss.server.entity.CrumbType -import com.webank.wedatasphere.dss.server.entity.CrumbType.CrumbType - - - -object QuerParamsParser { - - def getCrumbType(queryParams: String): CrumbType = { - queryParams.split("&").size match { - case 1 => CrumbType.All - case 2 => CrumbType.SortProject - case 4 => CrumbType.Project - case 5 => CrumbType.SortFlow - case 6 => CrumbType.Flow - } - } - - def toArray(queryParams: String): Array[String] = { - queryParams.split("&") - } - - def toMap(queryParams: Array[String]): java.util.Map[String, String] = { - import collection.JavaConverters._ - val regex = ("(.+)=(.+)").r - queryParams.map { - _ match { - case regex(k, v) => k -> v - } - }.toMap.asJava - } - - @Deprecated - def toQueryString(params: java.util.Map[String, String]): String = { - import collection.JavaConversions._ - val queryString = params.foldLeft("")((r, p) => s"${r}&${p._1}=${p._2}") - queryString.substring(1, queryString.length) - } - - - def toQueryString(params:Array[String]):String ={ - val queryString = params.foldLeft("")((r,p) =>s"${r}&${p}") - queryString.substring(1, queryString.length) - } -} - - -/*object Test{ - def main(args: Array[String]): Unit = { - val a = new util.HashMap[String,String]() - a.put("a","b") - a.put("c","d") - a.put("e","f") - val b = QuerParamsParser.toQueryString(a) - print(b.substring(1,b.length)) - } -}*/ - - - - diff --git a/dss-server/src/main/scala/com/webank/wedatasphere/dss/server/entity/Crumb.scala b/dss-server/src/main/scala/com/webank/wedatasphere/dss/server/entity/Crumb.scala deleted file mode 100644 index c222b1961f..0000000000 --- a/dss-server/src/main/scala/com/webank/wedatasphere/dss/server/entity/Crumb.scala +++ /dev/null @@ -1,28 +0,0 @@ -/* - * Copyright 2019 WeBank - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package com.webank.wedatasphere.dss.server.entity - -import CrumbType.CrumbType - -import scala.beans.BeanProperty - - - -class Crumb(var crumbTypeE: CrumbType, @BeanProperty var params: String, @BeanProperty var name: String, @BeanProperty var childrens: List[Crumb]) { - @BeanProperty var crumbType: String = crumbTypeE.toString -} diff --git a/dss-server/src/main/scala/com/webank/wedatasphere/dss/server/entity/CrumbType.scala b/dss-server/src/main/scala/com/webank/wedatasphere/dss/server/entity/CrumbType.scala deleted file mode 100644 index 957a8471d1..0000000000 --- a/dss-server/src/main/scala/com/webank/wedatasphere/dss/server/entity/CrumbType.scala +++ /dev/null @@ -1,24 +0,0 @@ -/* - * Copyright 2019 WeBank - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package com.webank.wedatasphere.dss.server.entity - - -object CrumbType extends Enumeration{ - type CrumbType = Value - val Index,All,SortProject,Project,SortFlow,Flow = Value -} diff --git a/dss-server/src/main/scala/com/webank/wedatasphere/dss/server/receiver/DSSServerReceiver.scala b/dss-server/src/main/scala/com/webank/wedatasphere/dss/server/receiver/DSSServerReceiver.scala deleted file mode 100644 index 951128083a..0000000000 --- a/dss-server/src/main/scala/com/webank/wedatasphere/dss/server/receiver/DSSServerReceiver.scala +++ /dev/null @@ -1,49 +0,0 @@ -/* - * Copyright 2019 WeBank - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package com.webank.wedatasphere.dss.server.receiver - -import com.webank.wedatasphere.dss.application.dao.ApplicationMapper -import com.webank.wedatasphere.dss.common.exception.DSSErrorException -import com.webank.wedatasphere.dss.common.protocol.{RequestDSSApplication, RequestDSSProject} -import com.webank.wedatasphere.dss.server.service.DSSProjectService -import com.webank.wedatasphere.linkis.rpc.{Receiver, Sender} -import org.springframework.beans.factory.annotation.Autowired -import org.springframework.stereotype.Component - -import scala.concurrent.duration.Duration - - -@Component -class DSSServerReceiver extends Receiver{ - - @Autowired - var dwsProjectService:DSSProjectService = _ - - @Autowired - var applicationMapper:ApplicationMapper = _ - - override def receive(message: Any, sender: Sender): Unit = {} - - override def receiveAndReply(message: Any, sender: Sender): Any = message match { - case f:RequestDSSProject => dwsProjectService.getExecutionDSSProject(f) - case RequestDSSApplication(name) => applicationMapper.getApplication(name) - case _ =>throw new DSSErrorException(90000,"") - } - - override def receiveAndReply(message: Any, duration: Duration, sender: Sender): Any = {} -} diff --git a/dss-server/src/main/scala/com/webank/wedatasphere/dss/server/service/BMLService.scala b/dss-server/src/main/scala/com/webank/wedatasphere/dss/server/service/BMLService.scala deleted file mode 100644 index 8e2d380d23..0000000000 --- a/dss-server/src/main/scala/com/webank/wedatasphere/dss/server/service/BMLService.scala +++ /dev/null @@ -1,111 +0,0 @@ -/* - * Copyright 2019 WeBank - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package com.webank.wedatasphere.dss.server.service - -import java.io.{ByteArrayInputStream, InputStream} -import java.util -import java.util.UUID - -import com.webank.wedatasphere.dss.common.exception.DSSErrorException -import com.webank.wedatasphere.linkis.bml.client.{BmlClient, BmlClientFactory} -import com.webank.wedatasphere.linkis.bml.protocol.{BmlDownloadResponse, BmlUpdateResponse, BmlUploadResponse} -import org.springframework.stereotype.Component - -import scala.collection.JavaConversions._ - - -@Component -class BMLService { - - def upload(userName: String, content: String, fileName: String): util.Map[String, Object] = { - val inputStream = new ByteArrayInputStream(content.getBytes("utf-8")) - val client: BmlClient = createBMLClient(userName) - val resource: BmlUploadResponse = client.uploadResource(userName, fileName, inputStream) - if (!resource.isSuccess) throw new DSSErrorException(911113, "上传失败") - val map = new util.HashMap[String, Object] - map += "resourceId" -> resource.resourceId - map += "version" -> resource.version - } - - def upload(userName: String, inputStream: InputStream, fileName: String): util.Map[String, Object] = { - val client: BmlClient = createBMLClient(userName) - val resource: BmlUploadResponse = client.uploadResource(userName, fileName, inputStream) - if (!resource.isSuccess) throw new DSSErrorException(911113, "上传失败") - val map = new util.HashMap[String, Object] - map += "resourceId" -> resource.resourceId - map += "version" -> resource.version - } - - def update(userName: String, resourceId: String, inputStream: InputStream): util.Map[String, Object] = { - val client: BmlClient = createBMLClient(userName) - val resource: BmlUpdateResponse = client.updateResource(userName, resourceId, "", inputStream) - if (!resource.isSuccess) throw new DSSErrorException(911114, "更新失败") - val map = new util.HashMap[String, Object] - map += "resourceId" -> resource.resourceId - map += "version" -> resource.version - } - - def update(userName: String, resourceId: String, content: String): util.Map[String, Object] = { - val inputStream = new ByteArrayInputStream(content.getBytes("utf-8")) - val client: BmlClient = createBMLClient(userName) - val resource: BmlUpdateResponse = client.updateResource(userName, resourceId, UUID.randomUUID().toString+".json", inputStream) - if (!resource.isSuccess) throw new DSSErrorException(911114, "更新失败") - val map = new util.HashMap[String, Object] - map += "resourceId" -> resource.resourceId - map += "version" -> resource.version - } - - def query(userName: String, resourceId: String, version: String): util.Map[String, Object] = { - val client: BmlClient = createBMLClient(userName) - var resource: BmlDownloadResponse = null - if (version == null) { - resource = client.downloadResource(userName, resourceId) - } else { - resource = client.downloadResource(userName, resourceId, version) - } - if (!resource.isSuccess) throw new DSSErrorException(911115, "下载失败") - val map = new util.HashMap[String, Object] - map += "path" -> resource.fullFilePath - map += "string" -> inputstremToString(resource.inputStream) - } - - def download(userName: String, resourceId: String, version: String): util.Map[String, Object] = { - val client: BmlClient = createBMLClient(userName) - var resource: BmlDownloadResponse = null - if (version == null) { - resource = client.downloadResource(userName, resourceId) - } else { - resource = client.downloadResource(userName, resourceId, version) - } - if (!resource.isSuccess) throw new DSSErrorException(911115, "下载失败") - val map = new util.HashMap[String, Object] - map += "path" -> resource.fullFilePath - map += "is" -> resource.inputStream - } - - private def inputstremToString(inputStream: InputStream): String = { - scala.io.Source.fromInputStream(inputStream).mkString - } - - private def createBMLClient(userName: String): BmlClient = { - if (userName == null) - BmlClientFactory.createBmlClient() - else - BmlClientFactory.createBmlClient(userName) - } -} diff --git a/dss-server/src/main/test/HttpTest.java b/dss-server/src/main/test/HttpTest.java deleted file mode 100644 index 0f2c6ac5f3..0000000000 --- a/dss-server/src/main/test/HttpTest.java +++ /dev/null @@ -1,185 +0,0 @@ -import com.webank.wedatasphere.dss.schedulis.exception.SchedulisSchedulerException; -import com.webank.wedatasphere.dss.schedulis.exception.SchedulisServerException; - -import org.apache.commons.io.IOUtils; -import org.apache.commons.lang.StringUtils; -import org.apache.http.Header; -import org.apache.http.HttpEntity; -import org.apache.http.NameValuePair; -import org.apache.http.client.CookieStore; -import org.apache.http.client.entity.UrlEncodedFormEntity; -import org.apache.http.client.methods.CloseableHttpResponse; -import org.apache.http.client.methods.HttpGet; -import org.apache.http.client.methods.HttpPost; -import org.apache.http.client.protocol.HttpClientContext; -import org.apache.http.cookie.Cookie; -import org.apache.http.entity.mime.MultipartEntityBuilder; -import org.apache.http.impl.client.BasicCookieStore; -import org.apache.http.impl.client.CloseableHttpClient; -import org.apache.http.impl.client.HttpClients; -import org.apache.http.message.BasicNameValuePair; -import org.apache.http.protocol.HTTP; -import org.apache.http.util.EntityUtils; -import org.junit.Test; -import scala.Tuple2; - -import java.io.*; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; -import java.util.Optional; - - -public class HttpTest { - - //@Test - public Cookie test01() throws IOException { - HttpPost httpPost = new HttpPost("http://127.0..0.1:8290/checkin"); - List params = new ArrayList<>(); - params.add(new BasicNameValuePair("username", "allenlliu")); - params.add(new BasicNameValuePair("userpwd", "*****")); - params.add(new BasicNameValuePair("action", "login")); - httpPost.setEntity(new UrlEncodedFormEntity(params)); - CookieStore cookieStore = new BasicCookieStore(); - CloseableHttpClient httpClient = null; - CloseableHttpResponse response = null; - HttpClientContext context = null; - try { - httpClient = HttpClients.custom().setDefaultCookieStore(cookieStore).build(); - context = HttpClientContext.create(); - response = httpClient.execute(httpPost, context); - } finally { - IOUtils.closeQuietly(response); - IOUtils.closeQuietly(httpClient); - } - List cookies = context.getCookieStore().getCookies(); - return cookies.get(0); - } - - @Test - public void test02() throws IOException { - List strings = new ArrayList<>(); - strings.add("1"); - strings.add("bbb"); - strings.add("ccc"); - } - - public void print(String string) throws IOException, IllegalAccessException{ - switch (string){ - case "1":throw new IOException("a"); - case "2":throw new IllegalAccessException("b"); - default: - System.out.println(string); - } - } - @Test - public void test03() throws IOException, SchedulisSchedulerException { - Cookie cookie = test01(); - List params = new ArrayList<>(); - params.add(new BasicNameValuePair("ajax","fetchProjectPage")); - params.add(new BasicNameValuePair("start","0")); - params.add(new BasicNameValuePair("length","10")); - params.add(new BasicNameValuePair("projectsType","personal")); - params.add(new BasicNameValuePair("pageNum","1")); - params.add(new BasicNameValuePair("order","orderProjectName")); - CookieStore cookieStore = new BasicCookieStore(); - cookieStore.addCookie(cookie); - HttpClientContext context = HttpClientContext.create(); - CloseableHttpResponse response = null; - CloseableHttpClient httpClient = null; - try { - String finalUrl = "http://127.0.0.1:8088/index" + "?" + EntityUtils.toString(new UrlEncodedFormEntity(params)); - HttpGet httpGet = new HttpGet(finalUrl); - httpGet.addHeader(HTTP.CONTENT_ENCODING, "UTF-8"); - httpClient = HttpClients.custom().setDefaultCookieStore(cookieStore).build(); - response = httpClient.execute(httpGet, context); - /*Header[] allHeaders = context.getRequest().getAllHeaders(); - Optional
header = Arrays.stream(allHeaders).filter(f -> "Cookie".equals(f.getAppJointName())).findFirst(); - header.ifPresent(AzkabanUtils.handlingConsumerWrapper(this::parseCookie));*/ - } catch (Exception e) { - throw new SchedulisSchedulerException(90004, e.getMessage()); - } finally { - IOUtils.closeQuietly(response); - IOUtils.closeQuietly(httpClient); - } - } - - @Test - public void test04(){ -/* A a = new A(); - Tuple2 hello = a.hello(); - Tuple2 stringStringTuple2 = new Tuple2<>("","");*/ - } - - @Test - public void test05(){ - String subFlowPath = "/Users/v_wbjftang/linuxDownloads/project_0926_tjf/flow2/subFlows/subFlow21/subFlows/subFlow211"; - int indexOf = subFlowPath.indexOf("subFlows"); - if(indexOf != -1){ - subFlowPath = subFlowPath.substring(0, indexOf-1); - } - String substring1 = subFlowPath.substring(0, subFlowPath.lastIndexOf("/")); - System.out.println(substring1); - String allPath = "/Users/v_wbjftang/linuxDownloads/project_0926_tjf/flow2/subFlows/subFlow21/subFlows/subFlow211"; - String substring = allPath.substring(substring1.length() +1 ); - System.out.println("\\" + File.separator); - //String s = substring.replaceAll("\\\\subFlows\\\\", "...."); - String s = substring.replaceAll("\\" + File.separator +"subFlows" + "\\" + File.separator, "...."); - System.out.println(s); - } - @Test - public void test06(){ - String nodeType = "spark.sql"; - String[] split = nodeType.split("\\."); - for (int i = 1; i < split.length; i++) { - String s = split[i]; - String replace = s.replace(s.charAt(0), (char) (s.charAt(0) - 32)); - split[i] = replace; - } - System.out.println(this.getClass().getPackage().getName()); - for (int i = 0; i < split.length; i++) { - System.out.println(i); - } - } - - @Test - public void test07() throws IOException { - Cookie cookie = test01(); - String projectName = "create_project0930"; - HttpPost httpPost = new HttpPost("http://127.0..0.1:8290/manager" + "?project=" + projectName); - httpPost.addHeader(HTTP.CONTENT_ENCODING, "UTF-8"); - CloseableHttpResponse response = null; - File file = new File("E:\\appcom\\tmp\\dws\\wtss\\neiljianliu\\2019-09-30\\project_0926_tjf.zip"); - CookieStore cookieStore = new BasicCookieStore(); - cookieStore.addCookie(cookie); - CloseableHttpClient httpClient = null; - InputStream inputStream = null; - try { - httpClient = HttpClients.custom().setDefaultCookieStore(cookieStore).build(); - MultipartEntityBuilder entityBuilder = MultipartEntityBuilder.create(); - entityBuilder.addBinaryBody("file",file); - entityBuilder.addTextBody("ajax", "upload"); - entityBuilder.addTextBody("project", projectName); - httpPost.setEntity(entityBuilder.build()); - response = httpClient.execute(httpPost); - HttpEntity httpEntity = response.getEntity(); - inputStream = httpEntity.getContent(); - String entStr = null; - entStr = IOUtils.toString(inputStream, "utf-8"); - if(response.getStatusLine().getStatusCode() != 200){ - throw new SchedulisServerException(90005, "release project failed, " + entStr); - } - }catch (Exception e){ - System.out.println(e.getMessage()); - } - finally { - IOUtils.closeQuietly(inputStream); - IOUtils.closeQuietly(response); - IOUtils.closeQuietly(httpClient); - } - } - @Test - public void test08(){ - - } -} diff --git a/dss-server/src/main/test/com/webank/Adf.scala b/dss-server/src/main/test/com/webank/Adf.scala deleted file mode 100644 index a102f41477..0000000000 --- a/dss-server/src/main/test/com/webank/Adf.scala +++ /dev/null @@ -1,9 +0,0 @@ -package com.webank - - -object Adf { - def main(args: Array[String]): Unit = { - val s = "10asdfasd" - print(s.stripPrefix("http://")) - } -} diff --git a/dss-server/src/main/test/com/webank/HttpTest.java b/dss-server/src/main/test/com/webank/HttpTest.java deleted file mode 100644 index 888ce006a3..0000000000 --- a/dss-server/src/main/test/com/webank/HttpTest.java +++ /dev/null @@ -1,141 +0,0 @@ -package com.webank; - -import com.webank.wedatasphere.dss.schedulis.exception.SchedulisSchedulerException; -import com.webank.wedatasphere.dss.schedulis.linkisjob.AbstractLinkisAzkabanJob; -import com.webank.wedatasphere.dss.schedulis.linkisjob.LinkisAzkabanJobFactory; -import org.apache.commons.io.IOUtils; -import org.apache.http.NameValuePair; -import org.apache.http.client.CookieStore; -import org.apache.http.client.entity.UrlEncodedFormEntity; -import org.apache.http.client.methods.CloseableHttpResponse; -import org.apache.http.client.methods.HttpGet; -import org.apache.http.client.methods.HttpPost; -import org.apache.http.client.protocol.HttpClientContext; -import org.apache.http.cookie.Cookie; -import org.apache.http.impl.client.BasicCookieStore; -import org.apache.http.impl.client.CloseableHttpClient; -import org.apache.http.impl.client.HttpClients; -import org.apache.http.message.BasicNameValuePair; -import org.apache.http.protocol.HTTP; -import org.apache.http.util.EntityUtils; -import org.junit.Test; - -import java.io.File; -import java.io.IOException; -import java.util.ArrayList; -import java.util.List; - - -public class HttpTest { - - //@Test - public Cookie test01() throws IOException { - HttpPost httpPost = new HttpPost("http://127.0.0.1:8088/checkin"); - List params = new ArrayList<>(); - params.add(new BasicNameValuePair("username", "neiljianliu")); - params.add(new BasicNameValuePair("userpwd", "*****")); - params.add(new BasicNameValuePair("action", "login")); - httpPost.setEntity(new UrlEncodedFormEntity(params)); - CookieStore cookieStore = new BasicCookieStore(); - CloseableHttpClient httpClient = null; - CloseableHttpResponse response = null; - HttpClientContext context = null; - try { - httpClient = HttpClients.custom().setDefaultCookieStore(cookieStore).build(); - context = HttpClientContext.create(); - response = httpClient.execute(httpPost, context); - } finally { - IOUtils.closeQuietly(response); - IOUtils.closeQuietly(httpClient); - } - List cookies = context.getCookieStore().getCookies(); - return cookies.get(0); - } - - @Test - public void test02() throws IOException { - List strings = new ArrayList<>(); - strings.add("1"); - strings.add("bbb"); - strings.add("ccc"); - } - - public void print(String string) throws IOException, IllegalAccessException{ - switch (string){ - case "1":throw new IOException("a"); - case "2":throw new IllegalAccessException("b"); - default: - System.out.println(string); - } - } - @Test - public void test03() throws IOException, SchedulisSchedulerException { - Cookie cookie = test01(); - List params = new ArrayList<>(); - params.add(new BasicNameValuePair("ajax","fetchProjectPage")); - params.add(new BasicNameValuePair("start","0")); - params.add(new BasicNameValuePair("length","10")); - params.add(new BasicNameValuePair("projectsType","personal")); - params.add(new BasicNameValuePair("pageNum","1")); - params.add(new BasicNameValuePair("order","orderProjectName")); - CookieStore cookieStore = new BasicCookieStore(); - cookieStore.addCookie(cookie); - HttpClientContext context = HttpClientContext.create(); - CloseableHttpResponse response = null; - CloseableHttpClient httpClient = null; - try { - String finalUrl = "http://127.0.0.1:8088/index" + "?" + EntityUtils.toString(new UrlEncodedFormEntity(params)); - HttpGet httpGet = new HttpGet(finalUrl); - httpGet.addHeader(HTTP.CONTENT_ENCODING, "UTF-8"); - httpClient = HttpClients.custom().setDefaultCookieStore(cookieStore).build(); - response = httpClient.execute(httpGet, context); - /*Header[] allHeaders = context.getRequest().getAllHeaders(); - Optional
header = Arrays.stream(allHeaders).filter(f -> "Cookie".equals(f.getAppJointName())).findFirst(); - header.ifPresent(AzkabanUtils.handlingConsumerWrapper(this::parseCookie));*/ - } catch (Exception e) { - throw new SchedulisSchedulerException(90002, e.getMessage()); - } finally { - IOUtils.closeQuietly(response); - IOUtils.closeQuietly(httpClient); - } - } - - @Test - public void test04(){ -/* A a = new A(); - Tuple2 hello = a.hello(); - Tuple2 stringStringTuple2 = new Tuple2<>("","");*/ - } - - @Test - public void test05(){ - //String subFlowPath = "C:\\Users\\v_wbjftang\\linuxDownloads\\project_0926_tjf\\flow2\\subFlows\\subFlow21\\subFlows\\subFlow211"; - String subFlowPath = "/Users/v_wbjftang/linuxDownloads/project_0926_tjf/flow2/subFlows/subFlow21/subFlows/subFlow211"; - int indexOf = subFlowPath.indexOf("subFlows"); - if(indexOf != -1){ - subFlowPath = subFlowPath.substring(0, indexOf-1); - } - String substring1 = subFlowPath.substring(0, subFlowPath.lastIndexOf("/")); - System.out.println(substring1); - String allPath = "/Users/v_wbjftang/linuxDownloads/project_0926_tjf/flow2/subFlows/subFlow21/subFlows/subFlow211"; - String substring = allPath.substring(substring1.length() +1 ); - System.out.println("\\" + File.separator); - //String s = substring.replaceAll("\\\\subFlows\\\\", "...."); - String s = substring.replaceAll("\\" + File.separator +"subFlows" + "\\" + File.separator, "...."); - System.out.println(s); - } - @Test - public void test06(){ -/* String nodeType = "spark.sql"; - try { - AbstractLinkisAzkabanJob job = LinkisAzkabanJobFactory.createJob(nodeType); - } catch (ClassNotFoundException e) { - e.printStackTrace(); - } catch (IllegalAccessException e) { - e.printStackTrace(); - } catch (InstantiationException e) { - e.printStackTrace(); - } - System.out.println("sdf");*/ - } -} diff --git a/dss-server/src/main/test/com/webank/JobParam.java b/dss-server/src/main/test/com/webank/JobParam.java deleted file mode 100644 index 482b060a3a..0000000000 --- a/dss-server/src/main/test/com/webank/JobParam.java +++ /dev/null @@ -1,11 +0,0 @@ -package com.webank; - - -public enum JobParam { - driver_memory; - - @Override - public String toString() { - return super.toString(); - } -} diff --git a/dss-server/src/main/test/com/webank/ReqTest.scala b/dss-server/src/main/test/com/webank/ReqTest.scala deleted file mode 100644 index 792e5ab96b..0000000000 --- a/dss-server/src/main/test/com/webank/ReqTest.scala +++ /dev/null @@ -1,9 +0,0 @@ -package com.webank - -import dispatch.Req - - -object ReqTest extends App{ - private val url: Req = dispatch.url("") - print(url == url.setMethod("POST")) -} diff --git a/dss-server/src/main/test/com/webank/TestUnit.java b/dss-server/src/main/test/com/webank/TestUnit.java deleted file mode 100644 index 07c619ae98..0000000000 --- a/dss-server/src/main/test/com/webank/TestUnit.java +++ /dev/null @@ -1,14 +0,0 @@ -package com.webank; - -import com.webank.wedatasphpere.dss.user.service.impl.UserAuthorizationClient; -import org.junit.Test; - -public class TestUnit { - - @Test - public void test() throws Exception { - - UserAuthorizationClient UserAuthorizationClient = new UserAuthorizationClient(); - - } -} diff --git a/dss-user-manager/pom.xml b/dss-user-manager/pom.xml deleted file mode 100644 index ef03b3a667..0000000000 --- a/dss-user-manager/pom.xml +++ /dev/null @@ -1,87 +0,0 @@ - - - - dss - com.webank.wedatasphere.dss - 0.9.1 - - 4.0.0 - - dss-user-manager - jar - - UTF-8 - - - - - com.webank.wedatasphere.linkis - linkis-module - ${linkis.version} - provided - - - scala-xml_2.11 - org.scala-lang.modules - - - guava - com.google.guava - - - - - com.webank.wedatasphere.linkis - linkis-common - 0.9.4 - - - com.github.rholder - guava-retrying - 2.0.0 - provided - - - org.junit.jupiter - junit-jupiter - RELEASE - test - - - dom4j - dom4j - 1.6.1 - - - cn.hutool - hutool-all - 5.3.2 - - - mysql - mysql-connector-java - 5.1.49 - - - com.typesafe - config - 1.4.1 - - - - - - org.apache.maven.plugins - maven-deploy-plugin - - - - org.apache.maven.plugins - maven-jar-plugin - - - - - \ No newline at end of file diff --git a/dss-user-manager/src/main/java/com/webank/wedatasphpere/dss/user/conf/DSSUserManagerConfig.java b/dss-user-manager/src/main/java/com/webank/wedatasphpere/dss/user/conf/DSSUserManagerConfig.java deleted file mode 100644 index 731194cc49..0000000000 --- a/dss-user-manager/src/main/java/com/webank/wedatasphpere/dss/user/conf/DSSUserManagerConfig.java +++ /dev/null @@ -1,60 +0,0 @@ -/* - * Copyright 2019 WeBank - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - - -package com.webank.wedatasphpere.dss.user.conf; - -import com.webank.wedatasphere.linkis.common.conf.CommonVars; - -import java.util.ResourceBundle; - -/** - * @program: dss-appjoint-auth - * @description: 用户模块配置文件 - * - * @create: 2020-12-30 16:26 - **/ - - -public class DSSUserManagerConfig { -// private final static ResourceBundle resource = ResourceBundle.getBundle("linkis"); - public static final String LOCAL_USER_ROOT_PATH = CommonVars.apply("wds.dss.user.root.dir","null").getValue().trim(); - public static final String BDP_SERVER_MYBATIS_DATASOURCE_URL = CommonVars.apply("wds.linkis.server.mybatis.datasource.url", "null").getValue().trim(); - public static final String BDP_SERVER_MYBATIS_DATASOURCE_USERNAME = CommonVars.apply("wds.linkis.server.mybatis.datasource.username", "null").getValue().trim(); - public static final String BDP_SERVER_MYBATIS_DATASOURCE_PASSWORD = CommonVars.apply("wds.linkis.server.mybatis.datasource.password", "null").getValue().trim(); - public static final String SCHEDULER_ADDRESS = CommonVars.apply("wds.dss.appjoint.scheduler.azkaban.address", "null").getValue().trim(); - public static final String USER_ACCOUNT_COMMANDS = CommonVars.apply("wds.dss.user.account.command.class", "null").getValue().trim(); - - public static final String METASTORE_HDFS_PATH = CommonVars.apply("wds.linkis.metastore.hive.hdfs.base.path", "null").getValue().trim(); - public static final String METASTORE_SCRIPT_PAHT = CommonVars.apply("wds.linkis.metastore.script.path", "null").getValue().trim(); - public static final String METASTORE_DB_TAIL = CommonVars.apply("wds.linkis.metastore.db.tail", "_default").getValue().trim(); - - public static final String KERBEROS_REALM = CommonVars.apply("wds.linkis.kerberos.realm", "null").getValue().trim(); - public static final String KERBEROS_ADMIN = CommonVars.apply("wds.linkis.kerberos.admin", "null").getValue().trim(); - public static final String KERBEROS_SCRIPT_PATH = CommonVars.apply("wds.linkis.kerberos.script.path", "null").getValue().trim(); - public static final String KERBEROS_KEYTAB_PATH = CommonVars.apply("wds.linkis.kerberos.keytab.path", "null").getValue().trim(); - public static final String KERBEROS_SSH_PORT = CommonVars.apply("wds.linkis.kerberos.ssh.port", "22").getValue().trim(); - public static final String KERBEROS_KDC_NODE = CommonVars.apply("wds.linkis.kerberos.kdc.node", "null").getValue().trim(); - public static final String KERBEROS_KDC_USER_NAME = CommonVars.apply("wds.linkis.kerberos.kdc.user.name", "null").getValue().trim(); - public static final String KERBEROS_KDC_USER_PASSWORD = CommonVars.apply("wds.linkis.kerberos.kdc.user.password", "null").getValue().trim(); - public static final String KERBEROS_ENABLE_SWITCH = CommonVars.apply("wds.linkis.kerberos.enable.switch", "null").getValue().trim(); - public static final String DSS_DEPLOY_PATH = CommonVars.apply("wds.dss.deploy.path", "null").getValue().trim(); - public static final String DSS_SCHEDULER_URL = CommonVars.apply("wds.dss.scheduler.url", "/schedule/system").getValue().trim(); - - - -} diff --git a/dss-user-manager/src/main/java/com/webank/wedatasphpere/dss/user/dto/request/AuthorizationBody.java b/dss-user-manager/src/main/java/com/webank/wedatasphpere/dss/user/dto/request/AuthorizationBody.java deleted file mode 100644 index 250f66f169..0000000000 --- a/dss-user-manager/src/main/java/com/webank/wedatasphpere/dss/user/dto/request/AuthorizationBody.java +++ /dev/null @@ -1,99 +0,0 @@ -/* - * Copyright 2019 WeBank - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - - -package com.webank.wedatasphpere.dss.user.dto.request; - -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; - -/** - * @program: user-manager - * @description: 施工单数据结构 - * - * @create: 2020-08-12 14:29 - **/ -public class AuthorizationBody { - - private String username; - private String password; - private String dssInstallDir; - private String azkakanDir; - private ArrayList servers; - - - public ArrayList getServers() { - return servers; - } - - public void setServers(ArrayList servers) { - this.servers = servers; - } - - - - - public String getAzkakanDir() { - return azkakanDir; - } - - public void setAzkakanDir(String azkakanDir) { - this.azkakanDir = azkakanDir; - } - - public String getDssInstallDir() { - return dssInstallDir; - } - - public void setDssInstallDir(String installDir) { - this.dssInstallDir = installDir; - } - - public List> getPaths() { - return paths; - } - - public void setPaths(List> paths) { - this.paths = paths; - } - - private List> paths; - - public String getUsername() { - return username; - } - - public void setUsername(String username) { - this.username = username; - } - - public String getPassword() { - return password; - } - - public void setPassword(String password) { - this.password = password; - } - - public String getDatabaseName(){ - return this.username + "_default"; - } - - - -} diff --git a/dss-user-manager/src/main/java/com/webank/wedatasphpere/dss/user/dto/request/LinuxServer.java b/dss-user-manager/src/main/java/com/webank/wedatasphpere/dss/user/dto/request/LinuxServer.java deleted file mode 100644 index 7569612f5c..0000000000 --- a/dss-user-manager/src/main/java/com/webank/wedatasphpere/dss/user/dto/request/LinuxServer.java +++ /dev/null @@ -1,53 +0,0 @@ -/* - * Copyright 2019 WeBank - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - - -package com.webank.wedatasphpere.dss.user.dto.request; - -public class LinuxServer { - - private String linuxHost; //master ip，Comma separated - private String linuxLoginUser; - private String linuxLoginPassword; - - - public String getLinuxHost() { - return linuxHost; - } - - public void setLinuxHost(String linuxHosts) { - this.linuxHost = linuxHosts; - } - - public String getLinuxLoginUser() { - return linuxLoginUser; - } - - public void setLinuxLoginUser(String linuxLoginUser) { - this.linuxLoginUser = linuxLoginUser; - } - - public String getLinuxLoginPassword() { - return linuxLoginPassword; - } - - public void setLinuxLoginPassword(String linuxLoginPassword) { - this.linuxLoginPassword = linuxLoginPassword; - } - - -} diff --git a/dss-user-manager/src/main/java/com/webank/wedatasphpere/dss/user/service/AbsCommand.java b/dss-user-manager/src/main/java/com/webank/wedatasphpere/dss/user/service/AbsCommand.java deleted file mode 100644 index be4bf3332f..0000000000 --- a/dss-user-manager/src/main/java/com/webank/wedatasphpere/dss/user/service/AbsCommand.java +++ /dev/null @@ -1,114 +0,0 @@ -/* - * Copyright 2019 WeBank - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - - -package com.webank.wedatasphpere.dss.user.service; - - -import com.webank.wedatasphere.linkis.server.Message; -import com.webank.wedatasphpere.dss.user.dto.request.AuthorizationBody; -import org.dom4j.DocumentException; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import javax.ws.rs.core.Response; -import java.io.BufferedReader; -import java.io.IOException; -import java.io.InputStreamReader; -import java.net.URL; - -/** - * 各模块的授权 继承这个类 根据需要实现自己的类。 - */ -public abstract class AbsCommand implements Command { - protected final Logger logger = LoggerFactory.getLogger(getClass()); - - @Override - public String capacity(AuthorizationBody body) { - return Command.SUCCESS; - } - - @Override - public String renew(AuthorizationBody body) { - return Command.SUCCESS; - } - - @Override - public String undoAuthorization(AuthorizationBody body) { return Command.SUCCESS; } - - @Override -// public String authorization(AuthorizationBody body) throws DocumentException { return Command.SUCCESS; } - public String authorization(AuthorizationBody body) throws IOException, Exception { return Command.SUCCESS; } - - public String toMessage(String msg) { - return this.getClass().getSimpleName() + "the module starts execution"+ msg; - } - - protected String runShell(String scriptPath, String[] args){ - String bashCommand; - try { - bashCommand = "sh " + scriptPath + " " + String.join(" ", args); - Runtime runtime = Runtime.getRuntime(); - Process process = runtime.exec(bashCommand); - - return this.getString(process); - } - catch (Exception e){ - logger.error(scriptPath, e); - return e.getMessage(); - } - } - - protected String getString(Process process) throws IOException, InterruptedException { - BufferedReader br = new BufferedReader(new InputStreamReader(process.getInputStream())); - - String inline; - while ((inline = br.readLine()) != null) { - if (!inline.equals("")) { - inline = inline.replaceAll("<", "<").replaceAll(">", ">"); - logger.info(inline); - } else { - logger.info("\n"); - } - } - br.close(); - br = new BufferedReader(new InputStreamReader(process.getErrorStream())); //错误信息 - while ((inline = br.readLine()) != null) { - if (!inline.equals("")) - logger.warn(inline); - else - logger.warn("\n"); - } - - int status = process.waitFor(); - if (status != 0){ - logger.error("shell error: "+status); - } - br.close(); - return Command.SUCCESS; - } - - protected String getResource(String path){ - try { - URL url = this.getClass().getClassLoader().getResource(path); - return url.getPath(); - }catch (Exception e){ - logger.error("File does not exist " + path, e); - } - return null; - } -} diff --git a/dss-user-manager/src/main/java/com/webank/wedatasphpere/dss/user/service/Command.java b/dss-user-manager/src/main/java/com/webank/wedatasphpere/dss/user/service/Command.java deleted file mode 100644 index dfed254468..0000000000 --- a/dss-user-manager/src/main/java/com/webank/wedatasphpere/dss/user/service/Command.java +++ /dev/null @@ -1,58 +0,0 @@ -/* - * Copyright 2019 WeBank - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - - -package com.webank.wedatasphpere.dss.user.service; - - -import com.webank.wedatasphpere.dss.user.dto.request.AuthorizationBody; -import org.dom4j.DocumentException; - -import java.io.IOException; - -public interface Command { - - final public static String SUCCESS = "success"; - final public static String ERROR = "error"; - /** - * 授权开通服务 - * @param body - * @return 成功 success 其他失败 - */ - public String authorization(AuthorizationBody body) throws DocumentException, IOException, Exception; - - /** - * 关闭授权 - * @param body - * @return 成功 success 其他失败 - */ - public String undoAuthorization(AuthorizationBody body) throws Exception; - - /** - * 扩容 - * @param body - * @return 成功 success 其他失败 - */ - public String capacity(AuthorizationBody body) throws Exception; - - /** - * 续费 - * @param body - * @return 成功 success 其他失败 - */ - public String renew(AuthorizationBody body) throws Exception; -} diff --git a/dss-user-manager/src/main/java/com/webank/wedatasphpere/dss/user/service/MacroCommand.java b/dss-user-manager/src/main/java/com/webank/wedatasphpere/dss/user/service/MacroCommand.java deleted file mode 100644 index 0d2d3431cf..0000000000 --- a/dss-user-manager/src/main/java/com/webank/wedatasphpere/dss/user/service/MacroCommand.java +++ /dev/null @@ -1,33 +0,0 @@ -/* - * Copyright 2019 WeBank - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - - -package com.webank.wedatasphpere.dss.user.service; - - -/** - * @program: user-authorization - * @description: 开通命令接口 - * - * @create: 2020-08-10 14:24 - **/ -public interface MacroCommand extends Command { - - public void add(AbsCommand command) throws Exception; - - -} diff --git a/dss-user-manager/src/main/java/com/webank/wedatasphpere/dss/user/service/impl/AzkabanCommand.java b/dss-user-manager/src/main/java/com/webank/wedatasphpere/dss/user/service/impl/AzkabanCommand.java deleted file mode 100644 index 2f85ae38b3..0000000000 --- a/dss-user-manager/src/main/java/com/webank/wedatasphpere/dss/user/service/impl/AzkabanCommand.java +++ /dev/null @@ -1,110 +0,0 @@ -/* - * Copyright 2019 WeBank - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - - -package com.webank.wedatasphpere.dss.user.service.impl; - -import com.webank.wedatasphpere.dss.user.dto.request.AuthorizationBody; -import com.webank.wedatasphpere.dss.user.service.AbsCommand; -import org.dom4j.Document; -import org.dom4j.DocumentException; -import org.dom4j.Element; -import org.dom4j.io.OutputFormat; -import org.dom4j.io.SAXReader; -import org.dom4j.io.XMLWriter; - -import java.io.File; -import java.io.FileInputStream; -import java.io.FileOutputStream; -import java.io.IOException; -import java.util.Iterator; -import java.util.List; - -/** - * @program: dss-appjoint-auth - * @description: 开通azkaban账号 - * - * @create: 2021-01-08 15:53 - **/ - -public class AzkabanCommand extends AbsCommand { - @Override - public String authorization(AuthorizationBody body) { - - try{ - this.xmlHandler(body.getAzkakanDir()+"/conf/azkaban-users.xml", body); - String[] args = {body.getUsername(), body.getPassword(), body.getDssInstallDir()+"/conf/"}; - String path = getResource("default/AddschedulerUser.sh"); - return this.runShell(path, args); - }catch (Exception err){ - logger.error("AzkabanCommand auth error:", err); - return err.getMessage(); - } - } - - private void xmlHandler(String azkPath, AuthorizationBody body) throws DocumentException, IOException { - SAXReader reader = new SAXReader(); - - File file = new File(azkPath); - Document document; - FileInputStream fis = null; - try { - fis = new FileInputStream(file); - document = reader.read(fis); - }catch (DocumentException e){ - throw e; - }finally { - if (fis != null) { - fis.close(); - } - } - - Element root = document.getRootElement(); - - Iterator it = root.elementIterator("user"); - Boolean userExists = false; - Element element = null; - while (it.hasNext()) { - element = (Element) it.next(); - - String v = element.attributeValue("username"); - if(v.equals(body.getUsername())){ //修改密码 - userExists = true; - element.attribute("password").setValue(body.getPassword()); - } - } - if(!userExists){ //新增账号 - Element cloneEl = element.createCopy(); - cloneEl.attribute("username").setValue(body.getUsername()); - cloneEl.attribute("password").setValue(body.getPassword()); - - List elements = root.elements("user"); - elements.add(elements.size(), cloneEl); - } - - this.saveXml(document, file); - - } - - private void saveXml(Document document, File file) throws IOException { - FileOutputStream out =new FileOutputStream(file); - OutputFormat format=OutputFormat.createPrettyPrint(); - XMLWriter writer=new XMLWriter(out, format); - writer.write(document); - writer.close(); - } -} diff --git a/dss-user-manager/src/main/java/com/webank/wedatasphpere/dss/user/service/impl/KerberosCommand.java b/dss-user-manager/src/main/java/com/webank/wedatasphpere/dss/user/service/impl/KerberosCommand.java deleted file mode 100644 index ef120f19e7..0000000000 --- a/dss-user-manager/src/main/java/com/webank/wedatasphpere/dss/user/service/impl/KerberosCommand.java +++ /dev/null @@ -1,69 +0,0 @@ -/* - * Copyright 2019 WeBank - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - - -package com.webank.wedatasphpere.dss.user.service.impl; - -import com.typesafe.config.Config; -import com.typesafe.config.ConfigFactory; -import com.webank.wedatasphere.linkis.common.conf.CommonVars; -import com.webank.wedatasphpere.dss.user.conf.DSSUserManagerConfig; -import com.webank.wedatasphpere.dss.user.dto.request.AuthorizationBody; -import com.webank.wedatasphpere.dss.user.service.AbsCommand; -import com.webank.wedatasphpere.dss.user.service.Command; - -import java.io.IOException; -import java.net.InetAddress; - -/** - * @date 2021/1/5 - */ -public class KerberosCommand extends AbsCommand { - - @Override - public String authorization(AuthorizationBody body) throws Exception { - String rst = createKt(body); - return rst != Command.SUCCESS ? rst : Command.SUCCESS; - } - - private String createKt(AuthorizationBody body) throws Exception { - String userName = body.getUsername(); - String hostName = InetAddress.getLocalHost().getHostName(); - String res = null; - if(userName != null){ - res = callShell(DSSUserManagerConfig.KERBEROS_SCRIPT_PATH, userName,hostName, - DSSUserManagerConfig.KERBEROS_KEYTAB_PATH,DSSUserManagerConfig.KERBEROS_SSH_PORT, - DSSUserManagerConfig.KERBEROS_KDC_NODE,DSSUserManagerConfig.KERBEROS_KDC_USER_NAME,DSSUserManagerConfig.KERBEROS_KDC_USER_PASSWORD,DSSUserManagerConfig.KERBEROS_REALM,DSSUserManagerConfig.KERBEROS_ENABLE_SWITCH); - } - return res; - } - - private String callShell(String shellFile, String username, String hostName, String keytabPath, - String sshPort, String kdcNode, String kdcUser,String password, String realm,String enableSwich) throws Exception { - - String bashCommand = getResource(shellFile); - String scriptCmd ; - if(null != hostName){ - scriptCmd = String.format("%s %s %s %s %s %s %s %s %s", username,hostName,keytabPath,sshPort,kdcNode,kdcUser,password,realm,enableSwich); - }else { - scriptCmd = String.format("%s %s %s %s %s %s %s %s", username,keytabPath,sshPort,kdcNode,kdcUser,password,realm,enableSwich); - } - String[] args = scriptCmd.split(" "); - return this.runShell(bashCommand, args); - } - -} diff --git a/dss-user-manager/src/main/java/com/webank/wedatasphpere/dss/user/service/impl/LdapCommand.java b/dss-user-manager/src/main/java/com/webank/wedatasphpere/dss/user/service/impl/LdapCommand.java deleted file mode 100644 index 31c2ecdbbf..0000000000 --- a/dss-user-manager/src/main/java/com/webank/wedatasphpere/dss/user/service/impl/LdapCommand.java +++ /dev/null @@ -1,58 +0,0 @@ -/* - * Copyright 2019 WeBank - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - - -package com.webank.wedatasphpere.dss.user.service.impl; - -import com.webank.wedatasphere.linkis.common.conf.CommonVars; -import com.webank.wedatasphpere.dss.user.conf.DSSUserManagerConfig; -import com.webank.wedatasphpere.dss.user.dto.request.AuthorizationBody; -import com.webank.wedatasphpere.dss.user.service.AbsCommand; -import com.webank.wedatasphpere.dss.user.service.Command; - -import java.io.BufferedReader; -import java.io.InputStreamReader; -import java.util.HashMap; -import java.util.List; - -/** - * @program: user-authorization - * @description: 创建用户空间 - * - * @create: 2020-08-13 13:39 - **/ - -public class LdapCommand extends AbsCommand { - - @Override - public String authorization(AuthorizationBody body) throws Exception { - - String userName = body.getUsername(); - String UserPassword = body.getPassword(); - String dssDeployPath = DSSUserManagerConfig.DSS_DEPLOY_PATH; - - String bashCommand = this.getClass().getClassLoader().getResource("default/CreateLdapAccount.sh").getPath(); - String[] args = { - userName, - UserPassword, - dssDeployPath - }; - - return this.runShell(bashCommand, args); - } - -} diff --git a/dss-user-manager/src/main/java/com/webank/wedatasphpere/dss/user/service/impl/LinuxUserCommand.java b/dss-user-manager/src/main/java/com/webank/wedatasphpere/dss/user/service/impl/LinuxUserCommand.java deleted file mode 100644 index 10a567e216..0000000000 --- a/dss-user-manager/src/main/java/com/webank/wedatasphpere/dss/user/service/impl/LinuxUserCommand.java +++ /dev/null @@ -1,60 +0,0 @@ -/* - * Copyright 2019 WeBank - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - - -package com.webank.wedatasphpere.dss.user.service.impl; - -import com.webank.wedatasphpere.dss.user.dto.request.AuthorizationBody; -import com.webank.wedatasphpere.dss.user.dto.request.LinuxServer; -import com.webank.wedatasphpere.dss.user.service.AbsCommand; - -import java.util.ArrayList; - -/** - * @program: user-authorization - * @description: 创建用户空间 - * - * @create: 2020-08-13 13:39 - **/ - -public class LinuxUserCommand extends AbsCommand { - - @Override - public String authorization(AuthorizationBody body) throws Exception { - -// String hosts = body.getLinuxHosts(); -// String linuxPassword = body.getLinuxLoginPassword(); -// String linuxUserName = body.getLinuxLoginUser(); - ArrayList linuxServers = body.getServers(); - logger.info("服务器ip"+linuxServers.toString()); - StringBuffer stringBuffer = new StringBuffer(); - for(LinuxServer linuxServer:linuxServers){ - String hosts = linuxServer.getLinuxHost(); - String linuxPassword = linuxServer.getLinuxLoginPassword(); - String linuxUserName = linuxServer.getLinuxLoginUser(); - stringBuffer.append(hosts).append("#").append(linuxUserName).append("#").append(linuxPassword).append(","); - } - stringBuffer.deleteCharAt(stringBuffer.lastIndexOf(",")); - String addUserName = body.getUsername(); - String addUserPassword = body.getPassword(); - String bashCommand = this.getClass().getClassLoader().getResource("default/CreateLinuxUser.sh").getPath(); - String[] args = {stringBuffer.toString(),addUserName,addUserPassword}; - - return this.runShell(bashCommand, args); - } - -} diff --git a/dss-user-manager/src/main/java/com/webank/wedatasphpere/dss/user/service/impl/MetastoreCommand.java b/dss-user-manager/src/main/java/com/webank/wedatasphpere/dss/user/service/impl/MetastoreCommand.java deleted file mode 100644 index 6076919192..0000000000 --- a/dss-user-manager/src/main/java/com/webank/wedatasphpere/dss/user/service/impl/MetastoreCommand.java +++ /dev/null @@ -1,56 +0,0 @@ -/* - * Copyright 2019 WeBank - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - - -package com.webank.wedatasphpere.dss.user.service.impl; - -import com.typesafe.config.Config; -import com.typesafe.config.ConfigFactory; -import com.webank.wedatasphere.linkis.common.conf.CommonVars; -import com.webank.wedatasphpere.dss.user.conf.DSSUserManagerConfig; -import com.webank.wedatasphpere.dss.user.dto.request.AuthorizationBody; -import com.webank.wedatasphpere.dss.user.service.AbsCommand; -import com.webank.wedatasphpere.dss.user.service.Command; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** - * @date 2021/1/5 - */ -public class MetastoreCommand extends AbsCommand { - private static final Logger logger = LoggerFactory.getLogger(MetastoreCommand.class); - - @Override - public String authorization(AuthorizationBody body) throws Exception { - String rst = createDb(body); - return rst != Command.SUCCESS ? rst : Command.SUCCESS; - } - - private String createDb(AuthorizationBody body) throws Exception { - String bashCommand = null; - String[] args = null; - String userName = body.getUsername(); - if (userName != null) { - String dbName = userName + DSSUserManagerConfig.METASTORE_DB_TAIL; - String path = DSSUserManagerConfig.METASTORE_HDFS_PATH + "/"+dbName+".db"; - bashCommand = getResource(DSSUserManagerConfig.METASTORE_SCRIPT_PAHT); - args = new String[]{ userName,dbName,path, - DSSUserManagerConfig.KERBEROS_REALM,DSSUserManagerConfig.KERBEROS_ADMIN,DSSUserManagerConfig.KERBEROS_KEYTAB_PATH,DSSUserManagerConfig.KERBEROS_ENABLE_SWITCH}; - } - return this.runShell(bashCommand, args); - } -} diff --git a/dss-user-manager/src/main/java/com/webank/wedatasphpere/dss/user/service/impl/SchedulisCommand.java b/dss-user-manager/src/main/java/com/webank/wedatasphpere/dss/user/service/impl/SchedulisCommand.java deleted file mode 100644 index 73351943b9..0000000000 --- a/dss-user-manager/src/main/java/com/webank/wedatasphpere/dss/user/service/impl/SchedulisCommand.java +++ /dev/null @@ -1,146 +0,0 @@ -/* - * Copyright 2019 WeBank - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - - -package com.webank.wedatasphpere.dss.user.service.impl; - -import com.fasterxml.jackson.databind.JsonNode; -import com.webank.wedatasphpere.dss.user.conf.DSSUserManagerConfig; -import com.webank.wedatasphpere.dss.user.dto.request.AuthorizationBody; -import com.webank.wedatasphpere.dss.user.service.AbsCommand; -import com.webank.wedatasphpere.dss.user.service.Command; - -import java.net.URI; -import java.sql.Connection; -import java.sql.DriverManager; -import java.sql.PreparedStatement; -import java.sql.SQLException; -import java.sql.Statement; -import java.util.Arrays; -import java.util.HashMap; -import java.util.Map; - -import cn.hutool.crypto.SecureUtil; -import cn.hutool.crypto.symmetric.DES; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.springframework.http.HttpStatus; -import org.springframework.http.ResponseEntity; -import org.springframework.util.StringUtils; -import org.springframework.web.client.RestTemplate; -import org.springframework.web.util.UriComponentsBuilder; - -/** - * @date 2021/1/11 - */ -public class SchedulisCommand extends AbsCommand { - - private static final Logger logger = LoggerFactory.getLogger(SchedulisCommand.class); - - @Override - public String authorization(AuthorizationBody body) { - String username = body.getUsername(); - logger.info("开始新增调度用户：{}", username); - String password = body.getPassword(); - String encryptionPwd = getEncryptionPwd(username, password); - Connection connection; - Statement stmt; - try { - logger.info("开始插入ctyun_user表"); - connection = getConnection(); - stmt = connection.createStatement(); - String sql = "INSERT INTO `ctyun_user` (`id`,`name`,`username`,`email`,`password`,`work_order_item_config`) VALUES (?,?,?,?,?,NULL) ON DUPLICATE KEY UPDATE `password` = ?"; - PreparedStatement statement = connection.prepareCall(sql); - statement.setString(1, username); - statement.setString(2, username); - statement.setString(3, username); - statement.setString(4, username); - statement.setString(5, encryptionPwd); - statement.setString(6, encryptionPwd); - statement.executeUpdate(); - stmt.close(); - connection.close(); - logger.info("完成插入ctyun_user表"); - } catch (SQLException e) { - logger.error(e.getMessage()); - } - logger.info("开始调用接口新增schedulis用户"); - addSchedulisUser(username, password); - logger.info("结束调用接口新增schedulis用户"); - return Command.SUCCESS; - } - - private static Connection getConnection() { - try { - //注册数据库的驱动 - Class.forName("com.mysql.jdbc.Driver"); - //获取数据库连接（里面内容依次是：主机名和端口、用户名、密码） - String url = DSSUserManagerConfig.BDP_SERVER_MYBATIS_DATASOURCE_URL; - String user = DSSUserManagerConfig.BDP_SERVER_MYBATIS_DATASOURCE_USERNAME; - String password = DSSUserManagerConfig.BDP_SERVER_MYBATIS_DATASOURCE_PASSWORD; - return DriverManager.getConnection(url, user, password); - } catch (Exception e) { - logger.error(e.getMessage()); - } - return null; - } - - private boolean addSchedulisUser(String username, String password) { - String schedulerUrl = DSSUserManagerConfig.DSS_SCHEDULER_URL; - RestTemplate restTemplate = new RestTemplate(); - String schedulisUrl = DSSUserManagerConfig.SCHEDULER_ADDRESS; - String url = new StringBuilder().append(schedulisUrl) - .append(schedulerUrl) - .toString(); - Map params = new HashMap<>(4); - params.put("userId", username); - params.put("password", password); - params.put("ajax", "addSystemUserViaFastTrackCtyun"); - String fullUrl = addParams(url, params); - UriComponentsBuilder builder = UriComponentsBuilder.fromHttpUrl(fullUrl); - URI uri = builder.build().encode().toUri(); - ResponseEntity responseEntity = restTemplate.getForEntity(uri, JsonNode.class); - return HttpStatus.OK.equals(responseEntity.getStatusCode()); - } - - private static String addParams(String url, Map params) { - if (params.isEmpty()) { - return url; - } - StringBuilder sb = new StringBuilder().append(url).append("?"); - for (Map.Entry entry : params.entrySet()) { - if (StringUtils.hasText(entry.getValue())) { - sb.append(entry.getKey()) - .append("=") - .append(entry.getValue()) - .append("&"); - } - } - return sb.deleteCharAt(sb.length()-1).toString(); - } - - private String getEncryptionPwd(String username, String password) { - int minSize = 8; - while (username.length() < minSize) { - username += username; - } - byte[] keyBytes = username.getBytes(); - Arrays.sort(keyBytes); - DES des = SecureUtil.des(keyBytes); - return des.encryptHex(password); - } -} diff --git a/dss-user-manager/src/main/java/com/webank/wedatasphpere/dss/user/service/impl/UserAuthorizationClient.java b/dss-user-manager/src/main/java/com/webank/wedatasphpere/dss/user/service/impl/UserAuthorizationClient.java deleted file mode 100644 index fd275a0e4b..0000000000 --- a/dss-user-manager/src/main/java/com/webank/wedatasphpere/dss/user/service/impl/UserAuthorizationClient.java +++ /dev/null @@ -1,57 +0,0 @@ -/* - * Copyright 2019 WeBank - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - - -package com.webank.wedatasphpere.dss.user.service.impl; - - -import com.webank.wedatasphpere.dss.user.conf.DSSUserManagerConfig; -import com.webank.wedatasphpere.dss.user.dto.request.AuthorizationBody; -import com.webank.wedatasphpere.dss.user.service.AbsCommand; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** - * @program: user-authorization - * @description: - * - * @create: 2020-08-10 14:24 - **/ -public class UserAuthorizationClient { - - public UserMacroCommand userMacroCommand = new UserMacroCommand(); - protected final Logger logger = LoggerFactory.getLogger(UserAuthorizationClient.class); - - public UserAuthorizationClient() { - - String[] commandPaths = DSSUserManagerConfig.USER_ACCOUNT_COMMANDS.split(","); - for(String classPath: commandPaths){ - try { - userMacroCommand.add((AbsCommand) Class.forName(classPath).newInstance()); - } catch (Exception e) { - logger.info(e.getMessage()); - e.printStackTrace(); - } - } - } - - public String authorization(AuthorizationBody body) throws Exception { - return userMacroCommand.authorization(body); - } - - -} diff --git a/dss-user-manager/src/main/java/com/webank/wedatasphpere/dss/user/service/impl/UserMacroCommand.java b/dss-user-manager/src/main/java/com/webank/wedatasphpere/dss/user/service/impl/UserMacroCommand.java deleted file mode 100644 index c842e666c3..0000000000 --- a/dss-user-manager/src/main/java/com/webank/wedatasphpere/dss/user/service/impl/UserMacroCommand.java +++ /dev/null @@ -1,118 +0,0 @@ -/* - * Copyright 2019 WeBank - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - - -package com.webank.wedatasphpere.dss.user.service.impl; - - -import com.github.rholder.retry.*; -import com.google.common.base.Predicates; -import com.webank.wedatasphpere.dss.user.dto.request.AuthorizationBody; -import com.webank.wedatasphpere.dss.user.service.AbsCommand; -import com.webank.wedatasphpere.dss.user.service.Command; -import com.webank.wedatasphpere.dss.user.service.MacroCommand; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.List; -import java.util.concurrent.Callable; -import java.util.concurrent.ExecutionException; - -/** - * @program: user-authorization - * @description: 开通命令实现 - * - * @create: 2020-08-10 14:24 - **/ -public class UserMacroCommand implements MacroCommand { - - private List commandList = new ArrayList<>(); - - protected final Logger logger = LoggerFactory.getLogger(getClass()); - - private static Integer RETRY_COUNT = 1; - - @Override - public void add(AbsCommand command) { - - commandList.add(command); - } - - @Override - public String authorization(AuthorizationBody body) throws Exception { - - return this.execute("authorization", body); - - } - - @Override - public String undoAuthorization(AuthorizationBody json) throws Exception { - - return this.execute("undoAuthorization", json); - - } - - @Override - public String capacity(AuthorizationBody json) throws Exception { - - return this.execute("capacity", json); - - } - - @Override - public String renew(AuthorizationBody json) throws Exception { - - return this.execute("renew", json); - - } - - - /** - * 授权操作基础方法 - * @param funName 调用的函数名 - * @param body 传入的数据 - * @return - * @throws ClassNotFoundException - * @throws NoSuchMethodException - */ - private String execute(String funName, AuthorizationBody body) throws Exception { - - for (AbsCommand command : commandList) { - - switch (funName) { - case "authorization": - command.authorization(body); - - case "undoAuthorization": - command.undoAuthorization(body); - - case "capacity": - command.capacity(body); - } - - } - - return "success"; - } - - - } - - - diff --git a/dss-user-manager/src/main/java/com/webank/wedatasphpere/dss/user/service/impl/WorkspaceCommand.java b/dss-user-manager/src/main/java/com/webank/wedatasphpere/dss/user/service/impl/WorkspaceCommand.java deleted file mode 100644 index 37b9f95972..0000000000 --- a/dss-user-manager/src/main/java/com/webank/wedatasphpere/dss/user/service/impl/WorkspaceCommand.java +++ /dev/null @@ -1,70 +0,0 @@ -/* - * Copyright 2019 WeBank - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - - -package com.webank.wedatasphpere.dss.user.service.impl; - -import com.webank.wedatasphpere.dss.user.dto.request.AuthorizationBody; -import com.webank.wedatasphpere.dss.user.service.AbsCommand; -import com.webank.wedatasphpere.dss.user.service.Command; - -import java.util.HashMap; -import java.util.List; - -/** - * @program: user-authorization - * @description: 创建用户空间 - * - * @create: 2020-08-13 13:39 - **/ - -public class WorkspaceCommand extends AbsCommand { - - @Override - public String authorization(AuthorizationBody body) throws Exception { - List> paths = body.getPaths(); - String result = ""; - Boolean isSuccess = true; - for(HashMap map : paths){ - String path = map.get("value"); - String rst = createDir(path, body); - result += rst; - if(!rst.equals(Command.SUCCESS)){ - isSuccess = false; - } - } - if(isSuccess){ - return Command.SUCCESS; - } - logger.error(result); - return result; - } - - private String createDir(String path, AuthorizationBody body) throws Exception { - String bashCommand; - - if(path.contains("hdfs:")){ - path = path.replace("hdfs://", "") + "/" + body.getUsername(); - bashCommand = getResource("default/HdfsPath.sh"); - }else { - path = path.replace("file://", "") + "/" + body.getUsername(); - bashCommand = getResource("default/LinuxPath.sh"); - } - String[] args = {body.getUsername(), path}; - return this.runShell(bashCommand, args); - } -} diff --git a/dss-user-manager/src/main/java/com/webank/wedatasphpere/dss/user/service/impl/YarnCommand.java b/dss-user-manager/src/main/java/com/webank/wedatasphpere/dss/user/service/impl/YarnCommand.java deleted file mode 100644 index a4d854780a..0000000000 --- a/dss-user-manager/src/main/java/com/webank/wedatasphpere/dss/user/service/impl/YarnCommand.java +++ /dev/null @@ -1,39 +0,0 @@ -/* - * Copyright 2019 WeBank - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - - -package com.webank.wedatasphpere.dss.user.service.impl; - - -import com.webank.wedatasphpere.dss.user.dto.request.AuthorizationBody; -import com.webank.wedatasphpere.dss.user.service.AbsCommand; -import com.webank.wedatasphpere.dss.user.service.Command; - -/** - * @program: user-authorization - * @description: 开通yarn权限的实现 - * - * @create: 2020-08-10 14:24 - **/ -public class YarnCommand extends AbsCommand { - - @Override - public String authorization(AuthorizationBody body) { - - return Command.SUCCESS; - } -} diff --git a/dss-user-manager/src/main/resources/config/properties.conf b/dss-user-manager/src/main/resources/config/properties.conf deleted file mode 100644 index ece97f39a3..0000000000 --- a/dss-user-manager/src/main/resources/config/properties.conf +++ /dev/null @@ -1,14 +0,0 @@ -#--Metastore配置 -base_path = #--元数据位于hdfs位置上的base -metastore_sh = default/Metastore.sh #--执行脚本的位置 -db_tail = #--hdfs上数据库文件的尾缀 -realm = #--kerberos域 -admin = #--获得hive metastore服务admin角色授权的用户（如hdfs,非常重要，否则无法完成授权） - -#--Kerberos配置 -shellFile = default/Kerberos.sh #--执行脚本的位置 -keytabPath = /etc/security/keytabs #--keytab在本节点上的存放位置 -sshPort = 22 #--ssh操作使用的端口 -kdcNode = #--kerberos服务kdc节点 -kdcUser = #--在进行ssh kdc操作是，对应kdc节点上的用户，应当知道该用户的密码，并且该用户应存在于kdc节点sudoers列表内 -password = "password" #--上面提到的kdc接应用户的登录密码，用于ssh操作 diff --git a/dss-user-manager/src/main/resources/default/AddschedulerUser.sh b/dss-user-manager/src/main/resources/default/AddschedulerUser.sh deleted file mode 100644 index d9a5a02e44..0000000000 --- a/dss-user-manager/src/main/resources/default/AddschedulerUser.sh +++ /dev/null @@ -1,18 +0,0 @@ -#!/usr/bin/env bash - -user=$1 -password=$2 -installDir=$3 - - -if grep -i "^${user}=" $installDir/token.properties; - then - if [ "$(uname)" == "Darwin" ]; - then - sed -i '' "s/^$user=.*/$user=$password/" $installDir/token.properties - else - sed -i "s/^$user=.*/$user=$password/" $installDir/token.properties - fi -else - echo "$user=$password" >> $installDir/token.properties -fi diff --git a/dss-user-manager/src/main/resources/default/CreateLdapAccount.sh b/dss-user-manager/src/main/resources/default/CreateLdapAccount.sh deleted file mode 100644 index 0f4c8cbe14..0000000000 --- a/dss-user-manager/src/main/resources/default/CreateLdapAccount.sh +++ /dev/null @@ -1,7 +0,0 @@ -#!/bin/bash -source /etc/profile -ldap_user=$1 -ldap_password=$2 -ldap_script_path=$3 -source $ldap_script_path/tools/venv/bin/activate && sudo $ldap_script_path/tools/venv/bin/python $ldap_script_path/tools/bin/ldap_user.py add_with_pw $ldap_user -p $ldap_password -echo "******************LDAP USER CREATED***********************" diff --git a/dss-user-manager/src/main/resources/default/CreateLinuxUser.sh b/dss-user-manager/src/main/resources/default/CreateLinuxUser.sh deleted file mode 100644 index 32e42bccc0..0000000000 --- a/dss-user-manager/src/main/resources/default/CreateLinuxUser.sh +++ /dev/null @@ -1,34 +0,0 @@ -#!/bin/bash - -source /etc/profile -server_user_strs=$1 -echo $server_user_strs -add_user_name=$2 -add_user_password=$3 -server_user_array=(${server_user_strs//,/ }) -for server_user_str in ${server_user_array[@]} -do - server_user_info=(${server_user_str//#/ }) - server_host=${server_user_info[0]} - server_user_name=${server_user_info[1]} - server_user_password=${server_user_info[2]} - echo "${server_host},${server_user_name},${server_user_password}" - - sudo sshpass -p $server_user_password ssh -o ConnectTimeout=1 $server_user_name@$server_host "echo success" - [ $? -ne 0 ] && echo "logon server:${server_host} failed" && exit 254 -done - -echo "************Start creating user*****************" - -for server_user_str in ${server_user_array[@]} -do - - server_user_info=(${server_user_str//#/ }) - server_host=${server_user_info[0]} - server_user_name=${server_user_info[1]} - server_user_password=${server_user_info[2]} - #sshpass -p $server_user_password ssh $server_user_name@$server_host "sudo useradd $add_user_name && echo $add_user_password |sudo -i passwd --stdin $add_user_name" - sshpass -p $server_user_password ssh $server_user_name@$server_host "sudo useradd $add_user_name -s /sbin/nologin" - - [ $? -ne 0 ] && echo "create user failed:${host}" && exit 254 -done diff --git a/dss-user-manager/src/main/resources/default/HdfsPath.sh b/dss-user-manager/src/main/resources/default/HdfsPath.sh deleted file mode 100644 index 1f29565014..0000000000 --- a/dss-user-manager/src/main/resources/default/HdfsPath.sh +++ /dev/null @@ -1,13 +0,0 @@ -#!/bin/bash -source /etc/profile -user=$1 -dir=$2 -echo $1 $2 -id $user -if [ $? -ne 0 ]; then - sudo useradd $user -s /sbin/nologin - echo "create user successfully" -fi - -hdfs dfs -mkdir -p $dir -hdfs dfs -chown $user:$user $dir diff --git a/dss-user-manager/src/main/resources/default/Kerberos.sh b/dss-user-manager/src/main/resources/default/Kerberos.sh deleted file mode 100644 index 80c3eb502c..0000000000 --- a/dss-user-manager/src/main/resources/default/Kerberos.sh +++ /dev/null @@ -1,146 +0,0 @@ -#!/bin/bash -source /etc/profile -#需要将当前登录用户，如dss加入到sudoers - -#函数 -check_principal_exist(){ - all_principal=`timeout 30 sshpass -p $PASSWORD ssh -p $SSH_PORT $KDC_USER@$KDCSERVER "sudo /usr/sbin/kadmin.local -q \"list_principals\""` #echo "all_principal:"$all_principal - principal=$1 - if [[ $all_principal =~ $principal ]] - then - #echo "包含" - return 1 - else - #echo "不包含" - return 0 - fi -} - -add_principal(){ - principalPrefix=$1 - echo "add_principal func,principalPrefix:"$principalPrefix - check_principal_exist "$principalPrefix@$REALM" - ifexist=$? - if [ $ifexist -eq 1 ] - then - echo "已有principal" - else - echo "没有principal,将会生成" - timeout 30 sshpass -p $PASSWORD ssh -p $SSH_PORT $KDC_USER@$KDCSERVER "sudo /usr/sbin/kadmin.local -q \"addprinc -randkey $principalPrefix\"" - fi -} - -generate_user(){ - username=$1 - if id -u $username >/dev/null 2>&1; then - echo "user exists" - else - echo "user does not exist, so we will create!" - sudo useradd $username - fi -} - -gen_keytab(){ - user=$1 - host=$2 - principalPrefix="$user/$host" - principal="$user/$host@$REALM" - add_principal $principalPrefix - if [[ $? -ne 0 ]];then - echo "create keytab failed!!!" - exit 1 - fi - timeout 30 sshpass -p $PASSWORD ssh -p $SSH_PORT $KDC_USER@$KDCSERVER "sudo rm -rf /tmp/$host.$user.keytab" - timeout 30 sshpass -p $PASSWORD ssh -p $SSH_PORT $KDC_USER@$KDCSERVER "sudo /usr/sbin/kadmin.local -q \"xst -norandkey -k /tmp/$host.$user.keytab $user/$host\"" - timeout 30 sshpass -p $PASSWORD ssh -p $SSH_PORT $KDC_USER@$KDCSERVER "sudo chmod 755 /tmp/$host.$user.keytab" - timeout 30 sshpass -p $PASSWORD scp -P $SSH_PORT $KDC_USER@$KDCSERVER:/tmp/$host.$user.keytab ./ - if [[ -f "$host.$user.keytab" ]]; then - sudo mv ./$host.$user.keytab $CENTER_KEYTAB_PATH/$user.keytab - if [[ $? != 0 ]];then - echo "rename keytab failed!" - else - generate_user $user - sudo chown $user $CENTER_KEYTAB_PATH/$user.keytab - sudo su - $user -c "kinit -kt $CENTER_KEYTAB_PATH/$user.keytab $principal" - deployUser=`whoami` - sudo su - $deployUser -c "crontab -l > conf && echo '* */12 * * * sudo -u $user kinit -kt $CENTER_KEYTAB_PATH/$user.keytab $principal' >> conf && crontab conf && rm -f conf" - fi - else - echo "the $user.keytab does not exist, please check your previous steps!" - fi -} - -gen_keytab_user(){ - user=$1 - principalPrefix="$user" - principal="$user@$REALM" - add_principal $principalPrefix - timeout 30 sshpass -p $PASSWORD ssh -p $SSH_PORT $KDC_USER@$KDCSERVER "sudo rm -rf /tmp/$user.keytab" - timeout 30 sshpass -p $PASSWORD ssh -p $SSH_PORT $KDC_USER@$KDCSERVER "sudo /usr/sbin/kadmin.local -q \"xst -norandkey -k /tmp/$user.keytab $user\"" - timeout 30 sshpass -p $PASSWORD ssh -p $SSH_PORT $KDC_USER@$KDCSERVER "sudo chmod 755 /tmp/$user.keytab" - timeout 30 sshpass -p $PASSWORD scp -P $SSH_PORT $KDC_USER@$KDCSERVER:/tmp/$user.keytab ./ - if [[ -f "$user.keytab" ]]; then - sudo mv ./$user.keytab $CENTER_KEYTAB_PATH/$user.keytab - generate_user $user - sudo chown $user $CENTER_KEYTAB_PATH/$user.keytab - sudo su - $user -c "kinit -kt $CENTER_KEYTAB_PATH/$user.keytab $principal" - sudo su - op -c "crontab -l > conf && echo '* */12 * * * sudo -u $user kinit -kt $CENTER_KEYTAB_PATH/$user.keytab $principal' >> conf && crontab conf && rm -f conf" - else - echo "the $user.keytab does not exist, please check your previous steps!" - fi - -} - - - -#第一个参数为功能参数（必须有）,第二个为user（必须有），第三个为host（可以有） -if [ $# -lt 3 ] || [ $# -gt 9 ]; then - echo -e "\033[31m \033[05m请确认您的操作,输入格式如下 功能参数 [user|user hostname]\033[0m" - echo "Usage: $0 genenateKeytab {username|username hostname}" - echo `date '+%Y-%m-%d %H:%M:%S'`" parameters:"$* >>/tmp/deltaKerberos.log - exit 1 -else - if [ $# -eq 8 ]; then - user=$1 - CENTER_KEYTAB_PATH=$2 - SSH_PORT=$3 - KDCSERVER=$4 - KDC_USER=$5 - PASSWORD=$6 - REALM=$7 - KERBEROS_ENABLE=$8 - echo $user - echo $CENTER_KEYTAB_PATH - echo $SSH_PORT - echo $KDCSERVER_$KDC_USER - echo $REALM - echo $KERBEROS_ENABLE - if [ $KERBEROS_ENABLE = "0" ]; then - echo "kerberos is disabled" - else - echo "kerberos is enable" - echo `date '+%Y-%m-%d %H:%M:%S'`" in genenate_key_tab username:"$user >>/tmp/deltaKerberos.log - gen_keytab_user $user - fi - else - user=$1 - host=$2 - CENTER_KEYTAB_PATH=$3 - SSH_PORT=$4 - KDCSERVER=$5 - KDC_USER=$6 - PASSWORD=$7 - REALM=$8 - KERBEROS_ENABLE=$9 - echo $REALM - echo $KERBEROS_ENABLE - if [ $KERBEROS_ENABLE = "0" ]; then - echo "kerberos is disabled" - else - echo "kerberos1 is enable" - echo `date '+%Y-%m-%d %H:%M:%S'`" in genenate_key_tab username:"$user" hostname:"$host >>/tmp/deltaKerberos.log - gen_keytab $user $host - fi - fi -fi -exit 0 diff --git a/dss-user-manager/src/main/resources/default/LinuxPath.sh b/dss-user-manager/src/main/resources/default/LinuxPath.sh deleted file mode 100644 index 03db38d857..0000000000 --- a/dss-user-manager/src/main/resources/default/LinuxPath.sh +++ /dev/null @@ -1,12 +0,0 @@ -#!/bin/bash -source /etc/profile -user=$1 -dir=$2 -echo $1 $2; -id $user -if [ $? -ne 0 ]; then - useradd $user -s /sbin/nologin - echo "create user successfully" -fi -sudo mkdir -p $dir -sudo chown $user:$user $dir diff --git a/dss-user-manager/src/main/resources/default/Metastore.sh b/dss-user-manager/src/main/resources/default/Metastore.sh deleted file mode 100644 index 3d72f66589..0000000000 --- a/dss-user-manager/src/main/resources/default/Metastore.sh +++ /dev/null @@ -1,44 +0,0 @@ -#!/bin/bash -source /etc/profile -user_name=$1 -db_name=$2 -path=$3 -realm=$4 -admin=$5 -ktPath=$6 -host_name=`hostname` -kerberos_enable=$7 -echo $kerberos_enable -if [ $kerberos_enable = "0" ]; then - echo "kerberos is disabled" -else - echo "kerberos is enabled" - kinit -kt $ktPath/$admin.keytab $admin/${host_name}@${realm} -fi - - #二、metastore操作 -hive -e "create database if not exists $db_name" -if [[ $? -ne 0 ]]; then - echo "create database failed!" -else - #修改数据库所属，将principal用户添加到metastore侧hive-site.xml hive.users.in.admin.role中 - if [ $kerberos_enable = "0" ]; then - hive -e "grant all on database $db_name to user $user_name" - else - hive -e "set role admin; grant all on database $db_name to user $user_name" - fi -fi - - #三、hdfs操作 -if [[ $? -ne 0 ]]; then - #回滚 - hive -e "drop database $db_name" - echo "grant database failed,rollback finished!" -else - echo "grant database $db_name successfully!" - #修改hdfs路径所属 - hdfs dfs -chown $user_name:$user_name $path - #修改hdfs路径权限 - hdfs dfs -chmod -R 700 $path - echo "hdfs operation successfully!" -fi diff --git a/dss-user-manager/src/main/resources/read_anlexander.txt b/dss-user-manager/src/main/resources/read_anlexander.txt deleted file mode 100644 index fc60487e09..0000000000 --- a/dss-user-manager/src/main/resources/read_anlexander.txt +++ /dev/null @@ -1,20 +0,0 @@ -一键化开通账户 - 服务目录结构： - 进入口调用： - 入口调用：dss-user-manager\src\main\java\com\webank\wedatasphpere\dss\user\service\impl\UserAuthorizationClient.java - -#参考配置文件resource/config/properties.conf -#metastore功能使用说明 - 参数：base_path --元数据位于hdfs位置上的base - metastore_sh --执行脚本的位置 - db_tail --hdfs上数据库文件的尾缀 - realm --kerberos域 - admin --获得hive metastore服务admin角色授权的用户（如hdfs,非常重要，否则无法完成授权） - -#kerberos功能使用说明 - 参数：keytabPath --keytab在本节点上的存放位置 - shellFile --执行脚本的位置 - sshPort --ssh操作使用的端口 - kdcNode --kerberos服务kdc节点 - kdcUser --在进行ssh kdc操作是，对应kdc节点上的用户，应当知道该用户的密码，并且该用户应存在于kdc节点sudoers列表内 - password --上面提到的kdc接应用户的登录密码，用于ssh操作 \ No newline at end of file diff --git a/dss-user-manager/src/test/java/com/webank/wedatasphpere/dss/user/service/impl/KerberosCommandTest.java b/dss-user-manager/src/test/java/com/webank/wedatasphpere/dss/user/service/impl/KerberosCommandTest.java deleted file mode 100644 index 445b4889d4..0000000000 --- a/dss-user-manager/src/test/java/com/webank/wedatasphpere/dss/user/service/impl/KerberosCommandTest.java +++ /dev/null @@ -1,43 +0,0 @@ -/* - * Copyright 2019 WeBank - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - - -package com.webank.wedatasphpere.dss.user.service.impl; - - -import com.webank.wedatasphpere.dss.user.dto.request.AuthorizationBody; -import org.junit.jupiter.api.Test; - -import java.io.IOException; - -class KerberosCommandTest { - - @Test -// @MethodSource("body") - void authorization() { - AuthorizationBody body = new AuthorizationBody(); - body.setUsername("anlexander"); - body.setPassword("123321"); - KerberosCommand test = new KerberosCommand(); - try { - test.authorization(body); - } catch (Exception e) { - e.printStackTrace(); - } - System.out.println("当前测试方法结束"); - } -} diff --git a/dss-user-manager/src/test/java/com/webank/wedatasphpere/dss/user/service/impl/MetastoreCommandTest.java b/dss-user-manager/src/test/java/com/webank/wedatasphpere/dss/user/service/impl/MetastoreCommandTest.java deleted file mode 100644 index da1fcb91e2..0000000000 --- a/dss-user-manager/src/test/java/com/webank/wedatasphpere/dss/user/service/impl/MetastoreCommandTest.java +++ /dev/null @@ -1,43 +0,0 @@ -/* - * Copyright 2019 WeBank - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - - -package com.webank.wedatasphpere.dss.user.service.impl; - - -import com.webank.wedatasphpere.dss.user.dto.request.AuthorizationBody; -import org.junit.jupiter.api.Test; - -class MetastoreCommandTest { - - @Test -// @MethodSource("body") - void authorization() { - AuthorizationBody body = new AuthorizationBody(); - body.setUsername("anlexander"); - body.setPassword("123321"); - MetastoreCommand test = new MetastoreCommand(); - - try { - test.authorization(body); - } catch (Exception e) { - e.printStackTrace(); - } - - System.out.println("当前测试方法结束"); - } -} diff --git a/dss-user-manager/src/test/java/com/webank/wedatasphpere/dss/user/service/impl/WorkspaceCommandTest.java b/dss-user-manager/src/test/java/com/webank/wedatasphpere/dss/user/service/impl/WorkspaceCommandTest.java deleted file mode 100644 index 68098eaaf4..0000000000 --- a/dss-user-manager/src/test/java/com/webank/wedatasphpere/dss/user/service/impl/WorkspaceCommandTest.java +++ /dev/null @@ -1,46 +0,0 @@ -/* - * Copyright 2019 WeBank - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - - -package com.webank.wedatasphpere.dss.user.service.impl; - - -import com.webank.wedatasphpere.dss.user.conf.DSSUserManagerConfig; -import com.webank.wedatasphpere.dss.user.dto.request.AuthorizationBody; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.params.provider.MethodSource; -import org.junit.jupiter.params.provider.ValueSource; - -class WorkspaceCommandTest { - - @Test -// @MethodSource("body") - void authorization() { - AuthorizationBody body = new AuthorizationBody(); - body.setUsername("luxl"); - body.setPassword("123321"); - WorkspaceCommand test = new WorkspaceCommand(); - try { - test.authorization(body); - } catch (Exception e) { - e.printStackTrace(); - } - System.out.println("当前测试方法结束"); - -// DSSUserManagerConfig.LOCAL_USER_ROOT_PATH.getValue(); - } -} diff --git a/eventchecker-appjoint/pom.xml b/eventchecker-appjoint/pom.xml deleted file mode 100644 index bdf035e3db..0000000000 --- a/eventchecker-appjoint/pom.xml +++ /dev/null @@ -1,139 +0,0 @@ - - - - - - dss - com.webank.wedatasphere.dss - 0.9.1 - - 4.0.0 - - dss-eventchecker-appjoint - - - - org.apache.commons - commons-lang3 - 3.4 - - - - com.alibaba - druid - 1.0.28 - - - - com.webank.wedatasphere.dss - dss-appjoint-core - ${dss.version} - provided - true - - - - - - - - - - - - - - - - log4j - log4j - 1.2.17 - - - - - - - - - - org.apache.maven.plugins - maven-deploy-plugin - - - - net.alchim31.maven - scala-maven-plugin - - - org.apache.maven.plugins - maven-jar-plugin - - - org.apache.maven.plugins - maven-assembly-plugin - 2.3 - false - - - make-assembly - package - - single - - - - src/main/assembly/distribution.xml - - - - - - false - dss-eventchecker-appjoint - false - false - - src/main/assembly/distribution.xml - - - - - - - src/main/java - - **/*.xml - - - - src/main/resources - - **/*.properties - **/application.yml - **/bootstrap.yml - **/log4j2.xml - - - - - - - \ No newline at end of file diff --git a/eventchecker-appjoint/src/main/assembly/distribution.xml b/eventchecker-appjoint/src/main/assembly/distribution.xml deleted file mode 100644 index 01d8fc345d..0000000000 --- a/eventchecker-appjoint/src/main/assembly/distribution.xml +++ /dev/null @@ -1,136 +0,0 @@ - - - - dss-eventchecker-appjoint - - zip - - true - eventchecker - - - - - - lib - true - true - false - true - true - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - ${basedir}/src/main/resources - - appjoint.properties - - 0777 - / - unix - - - - ${basedir}/src/main/resources - - log4j.properties - log4j2.xml - - 0777 - conf - unix - - - - - - diff --git a/eventchecker-appjoint/src/main/java/com/webank/wedatasphere/dss/appjoint/schedulis/jobtype/adapter/EventCheckAdapter.java b/eventchecker-appjoint/src/main/java/com/webank/wedatasphere/dss/appjoint/schedulis/jobtype/adapter/EventCheckAdapter.java deleted file mode 100644 index de40256743..0000000000 --- a/eventchecker-appjoint/src/main/java/com/webank/wedatasphere/dss/appjoint/schedulis/jobtype/adapter/EventCheckAdapter.java +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright 2019 WeBank - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package com.webank.wedatasphere.dss.appjoint.schedulis.jobtype.adapter; - -import org.apache.log4j.Logger; - -import java.util.Properties; - -/** - * @author georgeqiao - * @Title: EventCheckReceiver - * @ProjectName Azkaban-EventChecker - * @date 2019/9/1822:10 - * @Description: TODO - */ -public interface EventCheckAdapter { - - boolean sendMsg(int jobId, Properties props, Logger log); - - boolean reciveMsg(int jobId, Properties props, Logger log); - -} diff --git a/eventchecker-appjoint/src/main/java/com/webank/wedatasphere/dss/appjoint/schedulis/jobtype/connector/EventDruidFactory.java b/eventchecker-appjoint/src/main/java/com/webank/wedatasphere/dss/appjoint/schedulis/jobtype/connector/EventDruidFactory.java deleted file mode 100644 index fb1e3060de..0000000000 --- a/eventchecker-appjoint/src/main/java/com/webank/wedatasphere/dss/appjoint/schedulis/jobtype/connector/EventDruidFactory.java +++ /dev/null @@ -1,113 +0,0 @@ -/* - * Copyright 2019 WeBank - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package com.webank.wedatasphere.dss.appjoint.schedulis.jobtype.connector; - -import com.alibaba.druid.pool.DruidDataSource; - -import org.apache.commons.lang3.StringUtils; -import org.apache.log4j.Logger; - -import java.util.Base64; -import java.util.Properties; - -/** - * @author georgeqiao - * @Title: EventDruidFactory - * @ProjectName Azkaban-EventChecker - * @date 2019/9/1822:10 - * @Description: TODO - */ -public class EventDruidFactory { - private static DruidDataSource msgInstance; - - public static DruidDataSource getMsgInstance(Properties props, Logger log) { - if (msgInstance == null ) { - synchronized (EventDruidFactory.class) { - if(msgInstance == null) { - try { - msgInstance = createDataSource(props, log, "Msg"); - } catch (Exception e) { - throw new RuntimeException("Error creating Druid DataSource", e); - } - } - } - } - return msgInstance; - } - - private static DruidDataSource createDataSource(Properties props, Logger log, String type) { - String name = null; - String url = null; - String username = null; - String password = null; - - if(type.equals("Msg")){ - name = props.getProperty("msg.eventchecker.jdo.option.name"); - url = props.getProperty("msg.eventchecker.jdo.option.url"); - username = props.getProperty("msg.eventchecker.jdo.option.username"); - try { -// password = new String(Base64.getDecoder().decode(props.getProperty("msg.eventchecker.jdo.option.password").getBytes()),"UTF-8"); - password = props.getProperty("msg.eventchecker.jdo.option.password"); - } catch (Exception e){ - log.error("password decore failed" + e); - } - } - - int initialSize = Integer.valueOf(props.getProperty("option.initial.size", "1")); - int maxActive = Integer.valueOf(props.getProperty("option.max.active", "100")); - int minIdle = Integer.valueOf(props.getProperty("option.min.idle", "1")); - long maxWait = Long.valueOf(props.getProperty("option.max.wait", "60000")); - String validationQuery = props.getProperty("option.validation.quert", "SELECT 'x'"); - long timeBetweenEvictionRunsMillis = Long.valueOf(props.getProperty("option.time.between.eviction.runs.millis", "6000")); - long minEvictableIdleTimeMillis = Long.valueOf(props.getProperty("option.evictable.idle,time.millis", "300000")); - boolean testOnBorrow = Boolean.valueOf(props.getProperty("option.test.on.borrow", "true")); - int maxOpenPreparedStatements = Integer.valueOf(props.getProperty("option.max.open.prepared.statements", "-1")); - - if (timeBetweenEvictionRunsMillis > minEvictableIdleTimeMillis) { - timeBetweenEvictionRunsMillis = minEvictableIdleTimeMillis; - } - - DruidDataSource ds = new DruidDataSource(); - - if (StringUtils.isNotBlank(name)) { - ds.setName(name); - } - - ds.setUrl(url); - ds.setDriverClassName("com.mysql.jdbc.Driver"); - ds.setUsername(username); - ds.setPassword(password); - ds.setInitialSize(initialSize); - ds.setMinIdle(minIdle); - ds.setMaxActive(maxActive); - ds.setMaxWait(maxWait); - ds.setTestOnBorrow(testOnBorrow); - ds.setValidationQuery(validationQuery); - ds.setTimeBetweenEvictionRunsMillis(timeBetweenEvictionRunsMillis); - ds.setMinEvictableIdleTimeMillis(minEvictableIdleTimeMillis); - if (maxOpenPreparedStatements > 0) { - ds.setPoolPreparedStatements(true); - ds.setMaxPoolPreparedStatementPerConnectionSize( - maxOpenPreparedStatements); - } else { - ds.setPoolPreparedStatements(false); - } - log.info("Druid data source initialed!"); - return ds; - } -} diff --git a/eventchecker-appjoint/src/main/java/com/webank/wedatasphere/dss/appjoint/schedulis/jobtype/entity/EventChecker.java b/eventchecker-appjoint/src/main/java/com/webank/wedatasphere/dss/appjoint/schedulis/jobtype/entity/EventChecker.java deleted file mode 100644 index 2159eae253..0000000000 --- a/eventchecker-appjoint/src/main/java/com/webank/wedatasphere/dss/appjoint/schedulis/jobtype/entity/EventChecker.java +++ /dev/null @@ -1,277 +0,0 @@ -/* - * Copyright 2019 WeBank - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package com.webank.wedatasphere.dss.appjoint.schedulis.jobtype.entity; - -import com.google.gson.Gson; -import com.webank.wedatasphere.dss.appjoint.execution.common.NodeExecutionState; -import com.webank.wedatasphere.dss.appjoint.schedulis.jobtype.execution.EventCheckerNodeExecutionAction; -import com.webank.wedatasphere.dss.appjoint.schedulis.jobtype.service.AbstractEventCheck; -import com.webank.wedatasphere.dss.appjoint.schedulis.jobtype.service.EventCheckerService; -import org.apache.commons.lang3.StringUtils; -import org.apache.log4j.Logger; -import java.util.HashMap; -import java.util.Map; -import java.util.Properties; -import java.util.regex.Matcher; -import java.util.regex.Pattern; - -/** - * @author georgeqiao - * @Title: EventChecker - * @ProjectName Azkaban-EventChecker - * @date 2019/9/1822:10 - * @Description: TODO - */ -public class EventChecker implements Runnable{ - public final static String WAIT_TIME = "max.receive.hours"; - public final static String WAIT_FOR_TIME = "wait.for.time"; - public final static String QUERY_FREQUENCY = "query.frequency"; - public final static String MSGTYPE="msg.type"; - public final static String SENDER="msg.sender"; - public final static String RECEIVER="msg.receiver"; - public final static String TOPIC="msg.topic"; - public final static String MSGNAME="msg.name"; - public final static String MSG="msg.body"; - public final static String EXEC_ID = "azkaban.flow.execid"; - public final static String SAVE_KEY="msg.savekey"; - public final static String USER_TIME="msg.init.querytime"; - public final static String TODAY="only.receive.today"; - public final static String AFTERSEND="msg.after.send"; - - private Properties p; - private String jobId; - private int execId; - private EventCheckerService wbDao=null; - EventCheckerNodeExecutionAction backAction = null; - public Long maxWaitTime; - public int queryFrequency; - - private static Pattern pattern = Pattern.compile("[a-zA-Z_0-9@\\-]+"); - - private static final Logger logger = Logger.getRootLogger(); - - public EventChecker(String jobName, Properties p, EventCheckerNodeExecutionAction action) { - this.p = p; -// this.jobId = p.getProperty(EXEC_ID); - this.jobId = "1"; - backAction = action; - String waitTime = p.getProperty(EventChecker.WAIT_TIME, "1"); - Double doubleWaitTime = Double.valueOf(waitTime) * 3600 * 1000; - maxWaitTime = Long.valueOf(doubleWaitTime.longValue()); - String query_frequency = p.getProperty(EventChecker.QUERY_FREQUENCY, "30000"); - queryFrequency = Integer.valueOf(query_frequency); - } - - public void run() { - try { - backAction.setState(NodeExecutionState.Running); -// getPid(); - if (p == null) { - throw new RuntimeException("Properties is null. Can't continue"); - } - if (checkParamMap(p, MSGTYPE)) { - throw new RuntimeException("parameter " + MSGTYPE + " can not be blank."); - } - if (checkParamMap(p, TOPIC)) { - throw new RuntimeException("parameter " + TOPIC + " can not be blank."); - } else { - String topic = p.getProperty(TOPIC); - if (!topic.matches("[^_]*_[^_]*_[^_]*")) { - throw new RuntimeException("Error format of topic parameter. Accept: XX_XX_XX."); - } - } - if (checkParamMap(p, MSGNAME)) { - throw new RuntimeException("parameter " + MSGNAME + " can not be blank."); - } - wbDao = EventCheckerService.getInstance(); - execId = Integer.parseInt(jobId); - boolean success = false; - if (p.getProperty(MSGTYPE).equals("SEND")) { - if (checkParamMap(p, SENDER)) { - throw new RuntimeException("parameter " + SENDER + " can not be blank."); - } else { - String sender = p.getProperty(SENDER); - if (!sender.matches("[^@]*@[^@]*@[^@]*")) { - throw new RuntimeException("Error format of sender parameter. Accept: XX@XX@XX."); - } - } -// if (checkParamMap(p, MSG)) { -// throw new RuntimeException("Must specify a " + MSG -// + " key and value."); -// } - if (p.containsKey(MSG) && StringUtils.isNotEmpty(p.getProperty(MSG)) && p.getProperty(MSG).length() > 250) { - throw new RuntimeException("parameter " + MSG + " length less than 250 !"); - } - success = wbDao.sendMsg(execId, p, logger); - if (success) { - backAction.setState(NodeExecutionState.Success); - - } else { - throw new RuntimeException("Failed Send message."); - } - }else if(p.getProperty(MSGTYPE).equals("RECEIVE")) { - backAction.eventType("RECEIVE"); - receiveMsg(); - } else - { - throw new RuntimeException("Please input correct parameter of msg.type, Select RECEIVE Or SEND."); - } - }catch (Exception ex){ - backAction.setState(NodeExecutionState.Failed); - throw ex; - } - - } - - public boolean receiveMsg(){ - boolean success = false; - if(p.getProperty(MSGTYPE).equals("RECEIVE")) { - if (checkParamMap(p, RECEIVER)) { - backAction.setState(NodeExecutionState.Failed); - throw new RuntimeException("parameter " + RECEIVER + " can not be blank."); - } else { - String receiver = p.getProperty(RECEIVER); - if (!receiver.matches("[^@]*@[^@]*@[^@]*")) { - backAction.setState(NodeExecutionState.Failed); - throw new RuntimeException("Error format of receiver parameter. Accept: XX@XX@XX."); - } - } -// checkTimeParam(p, WAIT_FOR_TIME); - String userTime = checkTimeParamMap(p, USER_TIME); - if (StringUtils.isNotEmpty(userTime)) { - p.put(USER_TIME, userTime); - } - success = wbDao.reciveMsg(execId, p, logger); - if (success) { - backAction.saveKeyAndValue(getJobSaveKeyAndValue()); - backAction.setState(NodeExecutionState.Success); - } else { - backAction.setState(NodeExecutionState.Running); - } - } - return success; - } - - public String getJobSaveKeyAndValue(){ - Map saveValueMap = new HashMap<>(); - String msgBody=p.getProperty(MSG, "{}"); - String saveKey=p.getProperty(SAVE_KEY,"msg.body"); - saveValueMap.put(saveKey, msgBody); - Gson gson = new Gson(); - String saveValueJson = gson.toJson(saveValueMap); - logger.info("Output msg body: "+saveValueJson); - return saveValueJson; - } - - public void cancel() throws InterruptedException { -// AbstractEventCheck.closeDruidDataSource(); -// throw new RuntimeException("Kill this eventchecker."); - } - - private boolean checkParamMap(Properties p, String key){ - boolean checkFlag = false; - if(!p.containsKey(key)){ - throw new RuntimeException("parameter " + key + " is Empty."); - } - if(p.containsKey(key)){ - if(StringUtils.isEmpty(p.getProperty(key))){ - checkFlag = true; - } - } - if(!MSG.equals(key) && StringUtils.contains(p.getProperty(key), " ")){ - throw new RuntimeException("parameter " + key + " can not contains space !"); - } - if(!checkNoStandardStr(p.getProperty(key))){ - throw new RuntimeException("parameter " + key + " Accept letter and number and _@- only."); - } - if(p.getProperty(key).length() > 45){ - throw new RuntimeException("parameter " + key + " length less than 45 !"); - } - return checkFlag; - } - - private boolean checkNoStandardStr(String param){ - Matcher matcher = pattern.matcher(param); - return matcher.matches(); - } - - private void checkTimeParam(Properties p, String key){ - if(p.containsKey(key)){ - String waitForTime= p.getProperty(key); - if(!waitForTime.matches("^(0?[0-9]|1[0-9]|2[0-3]):(0?[0-9]|[1-5][0-9])$")){ - throw new RuntimeException("Parameter " + key + " Time format error ! For example: HH:mm"); - } - } - } - - private String checkTimeParamMap(Properties p, String key){ - if(p.containsKey(key)){ - String userTime = p.getProperty(key); - Pattern ptime = Pattern.compile("^([1][7-9][0-9][0-9]|[2][0][0-9][0-9])(\\-)([0][1-9]|[1][0-2])(\\-)([0-2][1-9]|[3][0-1])(\\s)([0-1][0-9]|[2][0-3])(:)([0-5][0-9])(:)([0-5][0-9])$"); - Matcher m = ptime.matcher(userTime); - if(!m.matches()){ - throw new RuntimeException("Parameter " + key + " Time format error ! For example: yyyy-MM-dd HH:mm:ss"); - } - return userTime; - }else{ - return null; - } - } - -// private String getPid(){ -// // get name representing the running Java virtual machine. -// String name = ManagementFactory.getRuntimeMXBean().getName(); -// System.out.println(name); -// // get pid -// String pid = name.split("@")[0]; -// logger.info("EventCheck Pid is:" + pid); -// return pid; -// } - - public static void main(String[] args) { - Properties p = new Properties(); - p.put("azkaban.flow.execid","111"); - - p.put("msg.type","RECEIVE"); - p.put("msg.receiver","project@job@v_zhu"); - p.put("msg.topic","bdp_new_test"); - p.put("msg.name","TestCheck"); - p.put("msg.savekey","msg.body"); - p.put("query.frequency","60"); - p.put("wait.time","1"); - p.put("only.receive.today","true"); - p.put("msg.after.send","true"); -// p.put("wait.for.time","22:00"); - -// p.put("msg.type","SEND"); -// p.put("msg.sender","project@job@v_zhu"); -// p.put("msg.topic","bdp_new_test"); -// p.put("msg.name","TestCheck"); -// p.put("msg.body","msg.body"); - - p.put("msg.eventchecker.jdo.option.name","msg"); - p.put("msg.eventchecker.jdo.option.url","jdbc:mysql://127.0..0.1:3306/wtss_qyh_test?useUnicode=true&characterEncoding=UTF-8"); - p.put("msg.eventchecker.jdo.option.username","root"); - p.put("msg.eventchecker.jdo.option.password","MTIzNDU2"); - -// EventChecker ec = new EventChecker("AA",p); -// ec.run(); - } - - -} diff --git a/eventchecker-appjoint/src/main/java/com/webank/wedatasphere/dss/appjoint/schedulis/jobtype/exception/UndefinedPropertyException.java b/eventchecker-appjoint/src/main/java/com/webank/wedatasphere/dss/appjoint/schedulis/jobtype/exception/UndefinedPropertyException.java deleted file mode 100644 index d92be51c36..0000000000 --- a/eventchecker-appjoint/src/main/java/com/webank/wedatasphere/dss/appjoint/schedulis/jobtype/exception/UndefinedPropertyException.java +++ /dev/null @@ -1,31 +0,0 @@ -/* - * Copyright 2019 WeBank - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package com.webank.wedatasphere.dss.appjoint.schedulis.jobtype.exception; - -/** - * Indicates that a required property is missing from the Props - */ -public class UndefinedPropertyException extends RuntimeException { - - private static final long serialVersionUID = 1; - - public UndefinedPropertyException(final String message) { - super(message); - } - -} diff --git a/eventchecker-appjoint/src/main/java/com/webank/wedatasphere/dss/appjoint/schedulis/jobtype/service/AbstractEventCheck.java b/eventchecker-appjoint/src/main/java/com/webank/wedatasphere/dss/appjoint/schedulis/jobtype/service/AbstractEventCheck.java deleted file mode 100644 index 3cecbb0e1f..0000000000 --- a/eventchecker-appjoint/src/main/java/com/webank/wedatasphere/dss/appjoint/schedulis/jobtype/service/AbstractEventCheck.java +++ /dev/null @@ -1,207 +0,0 @@ -/* - * Copyright 2019 WeBank - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package com.webank.wedatasphere.dss.appjoint.schedulis.jobtype.service; - -import com.alibaba.druid.pool.DruidDataSource; -import com.webank.wedatasphere.dss.appjoint.schedulis.jobtype.adapter.EventCheckAdapter; -import com.webank.wedatasphere.dss.appjoint.schedulis.jobtype.connector.EventDruidFactory; -import com.webank.wedatasphere.dss.appjoint.schedulis.jobtype.entity.EventChecker; - -import org.apache.log4j.Logger; - -import java.io.BufferedReader; -import java.io.IOException; -import java.io.InputStream; -import java.io.InputStreamReader; -import java.lang.management.ManagementFactory; -import java.net.InetAddress; -import java.net.NetworkInterface; -import java.net.SocketException; -import java.sql.Connection; -import java.sql.PreparedStatement; -import java.sql.ResultSet; -import java.sql.SQLException; -import java.util.ArrayList; -import java.util.Enumeration; -import java.util.List; -import java.util.Properties; - -import javax.sql.DataSource; - -/** - * @author georgeqiao - * @Title: AbstractEventCheck - * @ProjectName Azkaban-EventChecker - * @date 2019/9/1822:10 - * @Description: TODO - */ -public abstract class AbstractEventCheck implements EventCheckAdapter { - static DataSource msgDS; - String topic; - String msgName; - String receiver; - String sender; - String receiveToday; - String userTime; - String waitTime; - String query_frequency; - String wait_for_time; - String msg; - String afterSend; - - DataSource getMsgDS(Properties props, Logger log){ - if (msgDS == null) { - msgDS = EventDruidFactory.getMsgInstance(props, log); - if (msgDS == null) { - log.error("Error getting Druid DataSource instance"); - } - } - return msgDS; - } - - void initECParams(Properties props){ - topic = props.getProperty(EventChecker.TOPIC); - msgName = props.getProperty(EventChecker.MSGNAME); - receiver = props.getProperty(EventChecker.RECEIVER); - sender = props.getProperty(EventChecker.SENDER); - msg = props.getProperty(EventChecker.MSG); - receiveToday = props.getProperty(EventChecker.TODAY); - userTime = props.getProperty(EventChecker.USER_TIME); - waitTime = props.getProperty(EventChecker.WAIT_TIME, "1"); - query_frequency = props.getProperty(EventChecker.QUERY_FREQUENCY, "30000"); -// wait_for_time = props.getProperty(EventChecker.WAIT_FOR_TIME); - afterSend = props.getProperty(EventChecker.AFTERSEND); - } - - Connection getEventCheckerConnection(Properties props, Logger log){ - Connection connection = null; - try { - connection = getMsgDS(props,log).getConnection(); - } catch (SQLException e) { - throw new RuntimeException("Error getting DB Connection instance {} " + e); - } - return connection; - } - - @Override - public boolean sendMsg(int jobId, Properties props, Logger log) { - return false; - } - - @Override - public boolean reciveMsg(int jobId, Properties props, Logger log) { - return false; - } - - void closeConnection(Connection conn, Logger log) { - if (conn != null) { - try { - conn.close(); - } catch (SQLException e) { - log.error("Error closing connection", e); - } - } - } - - void closeQueryRef(ResultSet rs, Logger log) { - if (rs != null) { - try { - rs.close(); - } catch (SQLException e) { - log.error("Error closing result set", e); - } - } - - } - - void closeQueryStmt(PreparedStatement stmt, Logger log) { - if (stmt != null) { - try { - stmt.close(); - } catch (SQLException e) { - log.error("Error closing result stmt", e); - } - } - - } - - - public static void closeDruidDataSource() { - DruidDataSource msgDSObject = (DruidDataSource) msgDS; - if (msgDSObject != null) { - msgDSObject.close(); - } - - } - - String getLinuxLocalIp(Logger log) { - String ip = "127.0.0.1"; - try { - for (Enumeration en = NetworkInterface.getNetworkInterfaces(); en.hasMoreElements(); ) { - NetworkInterface intf = en.nextElement(); - String name = intf.getName(); - if (!name.contains("docker") && !name.contains("lo")) { - for (Enumeration enumIpAddr = intf.getInetAddresses(); enumIpAddr.hasMoreElements(); ) { - InetAddress inetAddress = enumIpAddr.nextElement(); - if (!inetAddress.isLoopbackAddress()) { - String ipaddress = inetAddress.getHostAddress().toString(); - if (!ipaddress.contains("::") && !ipaddress.contains("0:0:") && !ipaddress.contains("fe80")) { - ip = ipaddress; - } - } - } - } - } - } catch (SocketException ex) { - log.warn("get ip failed", ex); - - } - log.info("Send IP:" + ip); - return ip; - } - -// static void AzkabanStatusMonitor() { -// String name = ManagementFactory.getRuntimeMXBean().getName(); -// System.out.println(name); -// String pid = name.split("@")[0]; -// Process process = null; -// List processList = new ArrayList(); -// InputStream in = null; -// String[] cmds = {"/bin/sh", "-c", "ps -ef | grep " + pid + " | grep -v 'grep' | awk '{print $3}'"}; -// try { -// process = Runtime.getRuntime().exec(cmds); -// process.waitFor(); -// in = process.getInputStream(); -// BufferedReader reader = new BufferedReader(new InputStreamReader(in)); -// String line; -// while ((line = reader.readLine()) != null) { -// processList.add(line); -// } -// reader.close(); -// } catch (IOException e) { -// e.printStackTrace(); -// } catch (InterruptedException e) { -// e.printStackTrace(); -// } -// -// for (String ppid : processList) { -// System.out.println(ppid); -// -// } -// } -} diff --git a/eventchecker-appjoint/src/main/java/com/webank/wedatasphere/dss/appjoint/schedulis/jobtype/service/AbstractEventCheckReceiver.java b/eventchecker-appjoint/src/main/java/com/webank/wedatasphere/dss/appjoint/schedulis/jobtype/service/AbstractEventCheckReceiver.java deleted file mode 100644 index fbd1c887ae..0000000000 --- a/eventchecker-appjoint/src/main/java/com/webank/wedatasphere/dss/appjoint/schedulis/jobtype/service/AbstractEventCheckReceiver.java +++ /dev/null @@ -1,225 +0,0 @@ -/* - * Copyright 2019 WeBank - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package com.webank.wedatasphere.dss.appjoint.schedulis.jobtype.service; - -import com.webank.wedatasphere.dss.appjoint.schedulis.jobtype.entity.EventChecker; - -import org.apache.commons.lang3.StringUtils; -import org.apache.commons.lang3.time.DateFormatUtils; -import org.apache.log4j.Logger; - -import java.sql.Connection; -import java.sql.PreparedStatement; -import java.sql.ResultSet; -import java.sql.SQLException; -import java.util.Date; -import java.util.Properties; - -/** - * @author georgeqiao - * @Title: AbstractEventCheckReceiver - * @ProjectName Azkaban-EventChecker - * @date 2019/9/1822:10 - * @Description: TODO - */ -public class AbstractEventCheckReceiver extends AbstractEventCheck{ - /** - * Fill the result into the source - * @param props - * @param log - * @param consumedMsgInfo - * @return - */ - String setConsumedMsg(Properties props, Logger log, String[] consumedMsgInfo){ - String vNewMsgID = ""; - try { - if(consumedMsgInfo!=null && consumedMsgInfo.length == 4){ - vNewMsgID = consumedMsgInfo[0]; - String vMsgName = consumedMsgInfo[1]; - String vSender = consumedMsgInfo[2]; - String vMsg = consumedMsgInfo[3]; - if (null == vMsg) { - props.put(EventChecker.MSG, "NULL"); - } else { - props.put(EventChecker.MSG, vMsg); - } - log.info("Received message : messageID: " + vNewMsgID + ", messageName: " + vMsgName + ", receiver: " + vSender - + ", messageBody: " + vMsg); - } - }catch (Exception e) { - log.error("Error set consumed message failed {} setConsumedMsg failed" + e); - return vNewMsgID; - } - return vNewMsgID; - } - - /** - * Update consumption status - * @param jobId - * @param props - * @param log - * @param consumedMsgInfo - * @return - */ - boolean updateMsgOffset(int jobId, Properties props, Logger log, String[] consumedMsgInfo,String lastMsgId){ - boolean result = false; - String vNewMsgID = "-1"; - PreparedStatement updatePstmt = null; - PreparedStatement pstmtForGetID = null; - Connection msgConn = null; - vNewMsgID = setConsumedMsg(props,log,consumedMsgInfo); - try { - if(StringUtils.isNotEmpty(vNewMsgID) && StringUtils.isNotBlank(vNewMsgID) && !"-1".equals(vNewMsgID)){ - msgConn = getEventCheckerConnection(props,log); - if(msgConn == null) return false; - msgConn.setAutoCommit(false); - String sqlForReadMsgID = "SELECT msg_id FROM event_status WHERE receiver=? AND topic=? AND msg_name=? for update"; - pstmtForGetID = msgConn.prepareCall(sqlForReadMsgID); - pstmtForGetID.setString(1, receiver); - pstmtForGetID.setString(2, topic); - pstmtForGetID.setString(3, msgName); - ResultSet rs = pstmtForGetID.executeQuery(); - String nowLastMsgId = rs.last()==true ? rs.getString("msg_id"):"0"; - log.info("receive message successfully , Now check to see if the latest offset has changed ,nowLastMsgId is {} " + nowLastMsgId); - if("0".equals(nowLastMsgId) || nowLastMsgId.equals(lastMsgId)){ - - int vProcessID = jobId; - String vReceiveTime = DateFormatUtils.format(new Date(), "yyyy-MM-dd HH:mm:ss");; - String sqlForUpdateMsg = "INSERT INTO event_status(receiver,topic,msg_name,receive_time,msg_id) VALUES(?,?,?,?,?) ON DUPLICATE KEY UPDATE receive_time=VALUES(receive_time),msg_id= CASE WHEN msg_id= " + lastMsgId + " THEN VALUES(msg_id) ELSE msg_id END"; - log.info("last message offset {} is:" + lastMsgId); - updatePstmt = msgConn.prepareCall(sqlForUpdateMsg); - updatePstmt.setString(1, receiver); - updatePstmt.setString(2, topic); - updatePstmt.setString(3, msgName); - updatePstmt.setString(4, vReceiveTime); - updatePstmt.setString(5, vNewMsgID); - int updaters = updatePstmt.executeUpdate(); - log.info("updateMsgOffset successful {} update result is:" + updaters); - if(updaters != 0){ - log.info("Received message successfully , update message status succeeded, consumed flow execution ID: " + vProcessID); - //return true after update success - result = true; - }else{ - log.info("Received message successfully , update message status failed, consumed flow execution ID: " + vProcessID); - result = false; - } - }else{ - log.info("the latest offset has changed , Keep waiting for the signal"); - result = false; - } - msgConn.commit(); - }else{ - result = false; - } - }catch (SQLException e){ - log.error("Error update Msg Offset" + e); - try { - msgConn.rollback(); - } catch (SQLException ex) { - log.error("transaction rollback failed " + e); - } - return false; - }finally { - closeQueryStmt(pstmtForGetID, log); - closeQueryStmt(updatePstmt, log); - closeConnection(msgConn, log); - } - return result; - } - - /** - * get consumption progress - * @param jobId - * @param props - * @param log - * @return - */ - String getOffset(int jobId, Properties props, Logger log){ - String sqlForReadMsgID = "SELECT msg_id FROM event_status WHERE receiver=? AND topic=? AND msg_name=?"; - PreparedStatement pstmtForGetID = null; - Connection msgConn = null; - ResultSet rs = null; - boolean flag = false; - String lastMsgId = "0"; - try { - msgConn = getEventCheckerConnection(props,log); - pstmtForGetID = msgConn.prepareCall(sqlForReadMsgID); - pstmtForGetID.setString(1, receiver); - pstmtForGetID.setString(2, topic); - pstmtForGetID.setString(3, msgName); - rs = pstmtForGetID.executeQuery(); - lastMsgId = rs.last()==true ? rs.getString("msg_id"):"0"; - } catch (SQLException e) { - throw new RuntimeException("get Offset failed " + e); - }finally { - closeQueryStmt(pstmtForGetID,log); - closeConnection(msgConn,log); - closeQueryRef(rs,log); - } - log.info("The last record id was " + lastMsgId); - return lastMsgId; - } - - /** - * Consistent entrance to consumer message - * @param jobId - * @param props - * @param log - * @param params params[startQueryTime,endQueryTime,vMsgID] - * @return - */ - String[] getMsg(Properties props, Logger log,String ... params){ - String sqlForReadTMsg = "SELECT * FROM event_queue WHERE topic=? AND msg_name=? AND send_time >=? AND send_time <=? AND msg_id >? ORDER BY msg_id ASC LIMIT 1"; - PreparedStatement pstmt = null; - Connection msgConn = null; - ResultSet rs = null; - String[] consumedMsgInfo = null; - try { - msgConn = getEventCheckerConnection(props,log); - pstmt = msgConn.prepareCall(sqlForReadTMsg); - pstmt.setString(1, topic); - pstmt.setString(2, msgName); - pstmt.setString(3, params[0]); - pstmt.setString(4, params[1]); - pstmt.setString(5, params[2]); - log.info("param {} StartTime: " + params[0] + ", EndTime: " + params[1] - + ", Topic: " + topic + ", MessageName: " + msgName + ", LastMessageID: " + params[2]); - rs = pstmt.executeQuery(); - - if(rs.last()){ - consumedMsgInfo = new String[4]; - String[] msgKey = new String[]{"msg_id","msg_name","sender","msg"}; - for (int i = 0;i <= 3;i++) { - consumedMsgInfo[i] = rs.getString(msgKey[i]); - } - } - } catch (SQLException e) { - throw new RuntimeException("EventChecker failed to receive message" + e); - } finally { - closeQueryStmt(pstmt, log); - closeConnection(msgConn, log); - closeQueryRef(rs, log); - } - return consumedMsgInfo; - } - - @Override - public boolean reciveMsg(int jobId, Properties props, Logger log) { - return super.reciveMsg(jobId, props, log); - } -} diff --git a/eventchecker-appjoint/src/main/java/com/webank/wedatasphere/dss/appjoint/schedulis/jobtype/service/DefaultEventcheckReceiver.java b/eventchecker-appjoint/src/main/java/com/webank/wedatasphere/dss/appjoint/schedulis/jobtype/service/DefaultEventcheckReceiver.java deleted file mode 100644 index 2759256b97..0000000000 --- a/eventchecker-appjoint/src/main/java/com/webank/wedatasphere/dss/appjoint/schedulis/jobtype/service/DefaultEventcheckReceiver.java +++ /dev/null @@ -1,151 +0,0 @@ -/* - * Copyright 2019 WeBank - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package com.webank.wedatasphere.dss.appjoint.schedulis.jobtype.service; - - - -import com.webank.wedatasphere.dss.appjoint.schedulis.jobtype.entity.EventChecker; -import org.apache.commons.lang3.time.DateFormatUtils; -import org.apache.log4j.Logger; - -import java.text.DateFormat; -import java.text.ParseException; -import java.text.SimpleDateFormat; -import java.util.Date; -import java.util.Properties; - -/** - * @author georgeqiao - * @Title: DefaultEventcheckReceiver - * @ProjectName Azkaban-EventChecker - * @date 2019/9/1822:10 - * @Description: TODO - */ -public class DefaultEventcheckReceiver extends AbstractEventCheckReceiver { - String todayStartTime; - String todayEndTime; - String allStartTime; - String allEndTime; - String nowStartTime; - - public DefaultEventcheckReceiver(Properties props) { - initECParams(props); - initReceiverTimes(); - } - - private void initReceiverTimes(){ - todayStartTime = DateFormatUtils.format(new Date(), "yyyy-MM-dd 00:00:00"); - todayEndTime = DateFormatUtils.format(new Date(), "yyyy-MM-dd 23:59:59"); - allStartTime = DateFormatUtils.format(new Date(), "10000-01-01 00:00:00"); - allEndTime = DateFormatUtils.format(new Date(), "9999-12-31 23:59:59"); - nowStartTime = DateFormatUtils.format(new Date(), "yyyy-MM-dd HH:mm:ss"); - } - - @Override - public boolean reciveMsg(int jobId, Properties props, Logger log) { - boolean result = false; - try{ - String lastMsgId = getOffset(jobId,props,log); - String[] executeType = createExecuteType(jobId,props,log,lastMsgId); - if(executeType!=null && executeType.length ==3){ - String[] consumedMsgInfo = getMsg(props, log,executeType); - if(consumedMsgInfo!=null && consumedMsgInfo.length == 4){ - result = updateMsgOffset(jobId,props,log,consumedMsgInfo,lastMsgId); - } - }else{ - log.error("executeType error {} " + executeType.toString()); - return result; - } - }catch (Exception e){ - log.error("EventChecker failed to receive the message {}" + e); - return result; - } - return result; - } - - private String[] createExecuteType(int jobId, Properties props, Logger log,String lastMsgId){ - boolean receiveTodayFlag = (null != receiveToday && "true".equals(receiveToday.trim().toLowerCase())); - boolean afterSendFlag = (null != afterSend && "true".equals(afterSend.trim().toLowerCase())); - String[] executeType = null; - try { - if ("0".equals(lastMsgId)){ - if(receiveTodayFlag){ - if(afterSendFlag){ - executeType = new String[]{nowStartTime,todayEndTime,"0"}; - }else{ - executeType = new String[]{todayStartTime,todayEndTime,"0"}; - } - }else{ - if(afterSendFlag){ - executeType = new String[]{nowStartTime,allEndTime,"0"}; - }else{ - executeType = new String[]{allStartTime,allEndTime,"0"}; - } - } - }else{ - if(receiveTodayFlag){ - if(afterSendFlag){ - executeType = new String[]{nowStartTime,todayEndTime,lastMsgId}; - }else{ - executeType = new String[]{todayStartTime,todayEndTime,lastMsgId}; - } - }else{ - if(afterSendFlag){ - executeType = new String[]{nowStartTime,allEndTime,lastMsgId}; - }else{ - executeType = new String[]{allStartTime,allEndTime,lastMsgId}; - } - } - } - }catch(Exception e){ - log.error("create executeType failed {}" + e); - } - return executeType; - } - - private void waitForTime(Logger log,Long waitTime){ - String waitForTime = wait_for_time; - String formatWaitForTime = DateFormatUtils.format(new Date(),"yyyy-MM-dd " + waitForTime + ":00"); - DateFormat fmt =new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); - Date targetWaitTime = null; - try { - targetWaitTime = fmt.parse(formatWaitForTime); - } catch (ParseException e) { - log.error("parse date failed {}" + e); - } - - log.info("It will success at a specified time: " + targetWaitTime); - long wt = targetWaitTime.getTime() - System.currentTimeMillis(); - if(wt > 0){ - //wt must less than wait.time - if(wt <= waitTime){ - log.info("EventChecker will wait "+ wt + " milliseconds before starting execution"); - try { - Thread.sleep(wt); - } catch (InterruptedException e) { - throw new RuntimeException("EventChecker throws an exception during the waiting time {}"+e); - } - }else{ - throw new RuntimeException("The waiting time from Job starttime to wait.for.time"+ wt +"(ms) greater than wait.time , unreasonable setting！"); - } - }else{ - log.info("EventChecker has reached the specified time"); - } - } - -} diff --git a/eventchecker-appjoint/src/main/java/com/webank/wedatasphere/dss/appjoint/schedulis/jobtype/service/EventCheckSender.java b/eventchecker-appjoint/src/main/java/com/webank/wedatasphere/dss/appjoint/schedulis/jobtype/service/EventCheckSender.java deleted file mode 100644 index 662234ec4d..0000000000 --- a/eventchecker-appjoint/src/main/java/com/webank/wedatasphere/dss/appjoint/schedulis/jobtype/service/EventCheckSender.java +++ /dev/null @@ -1,75 +0,0 @@ -/* - * Copyright 2019 WeBank - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package com.webank.wedatasphere.dss.appjoint.schedulis.jobtype.service; - -import org.apache.commons.lang3.time.DateFormatUtils; -import org.apache.log4j.Logger; - -import java.sql.Connection; -import java.sql.PreparedStatement; -import java.sql.SQLException; -import java.util.Date; -import java.util.Properties; - -/** - * @author georgeqiao - * @Title: EventCheckSender - * @ProjectName Azkaban-EventChecker - * @date 2019/9/1822:10 - * @Description: TODO - */ -public class EventCheckSender extends AbstractEventCheck { - - public EventCheckSender(Properties props) { - initECParams(props); - } - - @Override - public boolean sendMsg(int jobId, Properties props, Logger log) { - boolean result = false; - PreparedStatement pstmt = null; - Connection msgConn = null; - String sendTime = DateFormatUtils.format(new Date(), "yyyy-MM-dd HH:mm:ss"); - String sqlForSendMsg = "INSERT INTO event_queue (sender,send_time,topic,msg_name,msg,send_ip) VALUES(?,?,?,?,?,?)"; - try { - String vIP = getLinuxLocalIp(log); - msgConn = getEventCheckerConnection(props,log); - if(msgConn==null) return false; - pstmt = msgConn.prepareCall(sqlForSendMsg); - pstmt.setString(1, sender); - pstmt.setString(2, sendTime); - pstmt.setString(3, topic); - pstmt.setString(4, msgName); - pstmt.setString(5, msg); - pstmt.setString(6, vIP); - int rs = pstmt.executeUpdate(); - if (rs == 1) { - result = true; - log.info("Send msg success!"); - } else { - log.error("Send msg failed for update database!"); - } - } catch (SQLException e) { - throw new RuntimeException("Send EventChecker msg failed!" + e); - } finally { - closeQueryStmt(pstmt, log); - closeConnection(msgConn, log); - } - return result; - } -} diff --git a/eventchecker-appjoint/src/main/java/com/webank/wedatasphere/dss/appjoint/schedulis/jobtype/service/EventCheckerService.java b/eventchecker-appjoint/src/main/java/com/webank/wedatasphere/dss/appjoint/schedulis/jobtype/service/EventCheckerService.java deleted file mode 100644 index 7f75170f54..0000000000 --- a/eventchecker-appjoint/src/main/java/com/webank/wedatasphere/dss/appjoint/schedulis/jobtype/service/EventCheckerService.java +++ /dev/null @@ -1,72 +0,0 @@ -/* - * Copyright 2019 WeBank - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package com.webank.wedatasphere.dss.appjoint.schedulis.jobtype.service; - -import org.apache.log4j.Logger; - -import java.util.Properties; - -/** - * @author georgeqiao - * @Title: EventCheckerService - * @ProjectName Azkaban-EventChecker - * @date 2019/9/1822:10 - * @Description: TODO - */ -public class EventCheckerService { - private static EventCheckerService instance; - - public static EventCheckerService getInstance() { - if (instance == null) { - synchronized (EventCheckerService.class) { - if (instance == null) { - instance = new EventCheckerService(); - } - } - } - return instance; - } - - public boolean sendMsg(int jobId, Properties props, Logger log) { - if(props!=null){ - return new EventCheckSender(props).sendMsg(jobId,props,log); - }else{ - log.error("create EventCheckSender failed {}"); - return false; - } - } - - /** - * 接收消息 接收消息先查询消费记录，有则从上一次消费后开始消费，没有则从任务启动时间点后开始消费。 - * 接收消息是以主动查询的方式进行的，在没有超出设定目标的时间内，反复查询目标消息。 - * Receiving a message first queries the consumption record, - * and then starts to consume after the last consumption, and no consumption - * starts after the job starts. The received message is performed in an active - * query manner, and the target message is repeatedly queried within a time period - * when the set target is not exceeded. - */ - public boolean reciveMsg(int jobId, Properties props, Logger log) { - if(props!=null){ - return new DefaultEventcheckReceiver(props).reciveMsg(jobId,props,log); - }else{ - log.error("create EventCheckSender failed {}"); - return false; - } - } - -} diff --git a/eventchecker-appjoint/src/main/java/com/webank/wedatasphere/dss/appjoint/schedulis/jobtype/utils/Props.java b/eventchecker-appjoint/src/main/java/com/webank/wedatasphere/dss/appjoint/schedulis/jobtype/utils/Props.java deleted file mode 100644 index 1e19756a58..0000000000 --- a/eventchecker-appjoint/src/main/java/com/webank/wedatasphere/dss/appjoint/schedulis/jobtype/utils/Props.java +++ /dev/null @@ -1,869 +0,0 @@ -/* - * Copyright 2019 WeBank - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package com.webank.wedatasphere.dss.appjoint.schedulis.jobtype.utils; - -import com.webank.wedatasphere.dss.appjoint.schedulis.jobtype.exception.UndefinedPropertyException; - -import org.apache.log4j.Logger; - -import java.io.BufferedInputStream; -import java.io.BufferedOutputStream; -import java.io.BufferedReader; -import java.io.File; -import java.io.FileInputStream; -import java.io.FileOutputStream; -import java.io.IOException; -import java.io.InputStream; -import java.io.InputStreamReader; -import java.io.OutputStream; -import java.net.URI; -import java.net.URISyntaxException; -import java.util.Arrays; -import java.util.Collections; -import java.util.HashMap; -import java.util.HashSet; -import java.util.Iterator; -import java.util.List; -import java.util.Map; -import java.util.Properties; -import java.util.Set; -import java.util.TreeMap; - -/** - * Hashmap implementation of a hierarchical properties with helpful converter functions and - * Exception throwing. This class is not threadsafe. - */ -public class Props { - - private final Map _current; - private Props _parent; - private String source = null; - - /** - * Constructor for empty props with empty parent. - */ - public Props() { - this(null); - } - - /** - * Constructor for empty Props with parent override. - */ - public Props(final Props parent) { - this._current = new HashMap(); - this._parent = parent; - } - - /** - * Load props from a file. - */ - public Props(final Props parent, final String filepath) throws IOException { - this(parent, new File(filepath)); - } - - /** - * Load props from a file. - */ - public Props(final Props parent, final File file) throws IOException { - this(parent); - setSource(file.getPath()); - - final InputStream input = new BufferedInputStream(new FileInputStream(file)); - try { - loadFrom(input); - } catch (final IOException e) { - throw e; - } finally { - input.close(); - } - } - - /** - * Create props from property input streams - */ - public Props(final Props parent, final InputStream inputStream) throws IOException { - this(parent); - loadFrom(inputStream); - } - - /** - * Create properties from maps of properties - */ - public Props(final Props parent, final Map... props) { - this(parent); - for (int i = props.length - 1; i >= 0; i--) { - this.putAll(props[i]); - } - } - - /** - * Create properties from Properties objects - */ - public Props(final Props parent, final Properties... properties) { - this(parent); - for (int i = properties.length - 1; i >= 0; i--) { - this.put(properties[i]); - } - } - - /** - * Create a Props object with the contents set to that of props. - */ - public Props(final Props parent, final Props props) { - this(parent); - if (props != null) { - putAll(props); - } - } - - /** - * Create a Props with a null parent from a list of key value pairing. i.e. [key1, value1, key2, - * value2 ...] - */ - public static Props of(final String... args) { - return of((Props) null, args); - } - - /** - * Create a Props from a list of key value pairing. i.e. [key1, value1, key2, value2 ...] - */ - public static Props of(final Props parent, final String... args) { - if (args.length % 2 != 0) { - throw new IllegalArgumentException( - "Must have an equal number of keys and values."); - } - - final Map vals = new HashMap<>(args.length / 2); - - for (int i = 0; i < args.length; i += 2) { - vals.put(args[i], args[i + 1]); - } - return new Props(parent, vals); - } - - /** - * Clones the Props p object and all of its parents. - */ - public static Props clone(final Props p) { - return copyNext(p); - } - - /** - * - * @param source - * @return - */ - private static Props copyNext(final Props source) { - Props priorNodeCopy = null; - if (source.getParent() != null) { - priorNodeCopy = copyNext(source.getParent()); - } - final Props dest = new Props(priorNodeCopy); - for (final String key : source.localKeySet()) { - dest.put(key, source.get(key)); - } - - return dest; - } - - /** - * - * @param inputStream - * @throws IOException - */ - private void loadFrom(final InputStream inputStream) throws IOException { - final Properties properties = new Properties(); - //解决.job文件中包含中文，读取乱码的问题。 - BufferedReader bufferedReader = new BufferedReader(new InputStreamReader(inputStream , "UTF-8")); - properties.load(bufferedReader); - this.put(properties); - } - - public Props getEarliestAncestor() { - if (this._parent == null) { - return this; - } - - return this._parent.getEarliestAncestor(); - } - - public void setEarliestAncestor(final Props parent) { - final Props props = getEarliestAncestor(); - props.setParent(parent); - } - - /** - * Clear the current Props, but leaves the parent untouched. - */ - public void clearLocal() { - this._current.clear(); - } - - /** - * Check key in current Props then search in parent - */ - public boolean containsKey(final Object k) { - return this._current.containsKey(k) - || (this._parent != null && this._parent.containsKey(k)); - } - - /** - * Check value in current Props then search in parent - */ - public boolean containsValue(final Object value) { - return this._current.containsValue(value) - || (this._parent != null && this._parent.containsValue(value)); - } - - /** - * Return value if available in current Props otherwise return from parent - */ - public String get(final Object key) { - if (this._current.containsKey(key)) { - return this._current.get(key); - } else if (this._parent != null) { - return this._parent.get(key); - } else { - return null; - } - } - - /** - * Get the key set from the current Props - */ - public Set localKeySet() { - return this._current.keySet(); - } - - /** - * Get parent Props - */ - public Props getParent() { - return this._parent; - } - - public void setParent(final Props prop) { - this._parent = prop; - } - - /** - * Put the given string value for the string key. This method performs any variable substitution - * in the value replacing any occurance of ${name} with the value of get("name"). - * - * @param key The key to put the value to - * @param value The value to do substitution on and store - * @throws IllegalArgumentException If the variable given for substitution is not a valid key in - * this Props. - */ - public String put(final String key, final String value) { - return this._current.put(key, value); - } - - /** - * Put the given Properties into the Props. This method performs any variable substitution in the - * value replacing any occurrence of ${name} with the value of get("name"). get() is called first - * on the Props and next on the Properties object. - * - * @param properties The properties to put - * @throws IllegalArgumentException If the variable given for substitution is not a valid key in - * this Props. - */ - public void put(final Properties properties) { - for (final String propName : properties.stringPropertyNames()) { - this._current.put(propName, properties.getProperty(propName)); - } - } - - /** - * Put integer - */ - public String put(final String key, final Integer value) { - return this._current.put(key, value.toString()); - } - - /** - * Put Long. Stores as String. - */ - public String put(final String key, final Long value) { - return this._current.put(key, value.toString()); - } - - /** - * Put Double. Stores as String. - */ - public String put(final String key, final Double value) { - return this._current.put(key, value.toString()); - } - - /** - * Put everything in the map into the props. - */ - public void putAll(final Map m) { - if (m == null) { - return; - } - - for (final Map.Entry entry : m.entrySet()) { - this.put(entry.getKey(), entry.getValue()); - } - } - - /** - * Put all properties in the props into the current props. Will handle null p. - */ - public void putAll(final Props p) { - if (p == null) { - return; - } - - for (final String key : p.getKeySet()) { - this.put(key, p.get(key)); - } - } - - /** - * Puts only the local props from p into the current properties - */ - public void putLocal(final Props p) { - for (final String key : p.localKeySet()) { - this.put(key, p.get(key)); - } - } - - /** - * Remove only the local value of key s, and not the parents. - */ - public String removeLocal(final Object s) { - return this._current.remove(s); - } - - /** - * The number of unique keys defined by this Props and all parent Props - */ - public int size() { - return getKeySet().size(); - } - - /** - * The number of unique keys defined by this Props (keys defined only in parent Props are not - * counted) - */ - public int localSize() { - return this._current.size(); - } - - /** - * Attempts to return the Class that corresponds to the Props value. If the class doesn't exit, an - * IllegalArgumentException will be thrown. - */ - public Class getClass(final String key) { - try { - if (containsKey(key)) { - return Class.forName(get(key)); - } else { - throw new UndefinedPropertyException("Missing required property '" - + key + "'"); - } - } catch (final ClassNotFoundException e) { - throw new IllegalArgumentException(e); - } - } - - public Class getClass(final String key, final boolean initialize, final ClassLoader cl) { - try { - if (containsKey(key)) { - return Class.forName(get(key), initialize, cl); - } else { - throw new UndefinedPropertyException("Missing required property '" - + key + "'"); - } - } catch (final ClassNotFoundException e) { - throw new IllegalArgumentException(e); - } - } - - /** - * Gets the class from the Props. If it doesn't exist, it will return the defaultClass - */ - public Class getClass(final String key, final Class defaultClass) { - if (containsKey(key)) { - return getClass(key); - } else { - return defaultClass; - } - } - - /** - * Gets the string from the Props. If it doesn't exist, it will return the defaultValue - */ - public String getString(final String key, final String defaultValue) { - if (containsKey(key)) { - return get(key); - } else { - return defaultValue; - } - } - - /** - * Gets the string from the Props. If it doesn't exist, throw and UndefinedPropertiesException - */ - public String getString(final String key) { - if (containsKey(key)) { - return get(key); - } else { - throw new UndefinedPropertyException("Missing required property '" + key - + "'"); - } - } - - /** - * Returns a list of strings with the comma as the separator of the value - */ - public List getStringList(final String key) { - return getStringList(key, "\\s*,\\s*"); - } - - /** - * Returns a list of clusters with the comma as the separator of the value - * e.g., for input string: "thrift://hcat1:port,thrift://hcat2:port;thrift://hcat3:port,thrift://hcat4:port;" - * we will get ["thrift://hcat1:port,thrift://hcat2:port", "thrift://hcat3:port,thrift://hcat4:port"] as output - */ - public List getStringListFromCluster(final String key) { - List curlist = getStringList(key, "\\s*;\\s*"); - // remove empty elements in the array - for (Iterator iter = curlist.listIterator(); iter.hasNext(); ) { - String a = iter.next(); - if (a.length() == 0) { - iter.remove(); - } - } - return curlist; - } - - /** - * Returns a list of strings with the sep as the separator of the value - */ - public List getStringList(final String key, final String sep) { - final String val = get(key); - if (val == null || val.trim().length() == 0) { - return Collections.emptyList(); - } - - if (containsKey(key)) { - return Arrays.asList(val.split(sep)); - } else { - throw new UndefinedPropertyException("Missing required property '" + key - + "'"); - } - } - - /** - * Returns a list of strings with the comma as the separator of the value. If the value is null, - * it'll return the defaultValue. - */ - public List getStringList(final String key, final List defaultValue) { - if (containsKey(key)) { - return getStringList(key); - } else { - return defaultValue; - } - } - - /** - * Returns a list of strings with the sep as the separator of the value. If the value is null, - * it'll return the defaultValue. - */ - public List getStringList(final String key, final List defaultValue, - final String sep) { - if (containsKey(key)) { - return getStringList(key, sep); - } else { - return defaultValue; - } - } - - /** - * Returns true if the value equals "true". If the value is null, then the default value is - * returned. - */ - public boolean getBoolean(final String key, final boolean defaultValue) { - if (containsKey(key)) { - return "true".equalsIgnoreCase(get(key).trim()); - } else { - return defaultValue; - } - } - - /** - * Returns true if the value equals "true". If the value is null, then an - * UndefinedPropertyException is thrown. - */ - public boolean getBoolean(final String key) { - if (containsKey(key)) { - return "true".equalsIgnoreCase(get(key)); - } else { - throw new UndefinedPropertyException("Missing required property '" + key - + "'"); - } - } - - /** - * Returns the long representation of the value. If the value is null, then the default value is - * returned. If the value isn't a long, then a parse exception will be thrown. - */ - public long getLong(final String name, final long defaultValue) { - if (containsKey(name)) { - return Long.parseLong(get(name)); - } else { - return defaultValue; - } - } - - /** - * Returns the long representation of the value. If the value is null, then a - * UndefinedPropertyException will be thrown. If the value isn't a long, then a parse exception - * will be thrown. - */ - public long getLong(final String name) { - if (containsKey(name)) { - return Long.parseLong(get(name)); - } else { - throw new UndefinedPropertyException("Missing required property '" + name - + "'"); - } - } - - /** - * Returns the int representation of the value. If the value is null, then the default value is - * returned. If the value isn't a int, then a parse exception will be thrown. - */ - public int getInt(final String name, final int defaultValue) { - if (containsKey(name)) { - return Integer.parseInt(get(name).trim()); - } else { - return defaultValue; - } - } - - /** - * Returns the int representation of the value. If the value is null, then a - * UndefinedPropertyException will be thrown. If the value isn't a int, then a parse exception - * will be thrown. - */ - public int getInt(final String name) { - if (containsKey(name)) { - return Integer.parseInt(get(name).trim()); - } else { - throw new UndefinedPropertyException("Missing required property '" + name - + "'"); - } - } - - /** - * Returns the double representation of the value. If the value is null, then the default value is - * returned. If the value isn't a double, then a parse exception will be thrown. - */ - public double getDouble(final String name, final double defaultValue) { - if (containsKey(name)) { - return Double.parseDouble(get(name).trim()); - } else { - return defaultValue; - } - } - - /** - * Returns the double representation of the value. If the value is null, then a - * UndefinedPropertyException will be thrown. If the value isn't a double, then a parse exception - * will be thrown. - */ - public double getDouble(final String name) { - if (containsKey(name)) { - return Double.parseDouble(get(name).trim()); - } else { - throw new UndefinedPropertyException("Missing required property '" + name - + "'"); - } - } - - /** - * Returns the uri representation of the value. If the value is null, then the default value is - * returned. If the value isn't a uri, then a IllegalArgumentException will be thrown. - */ - public URI getUri(final String name) { - if (containsKey(name)) { - try { - return new URI(get(name)); - } catch (final URISyntaxException e) { - throw new IllegalArgumentException(e.getMessage()); - } - } else { - throw new UndefinedPropertyException("Missing required property '" + name - + "'"); - } - } - - /** - * Returns the double representation of the value. If the value is null, then the default value is - * returned. If the value isn't a uri, then a IllegalArgumentException will be thrown. - */ - public URI getUri(final String name, final URI defaultValue) { - if (containsKey(name)) { - return getUri(name); - } else { - return defaultValue; - } - } - - public URI getUri(final String name, final String defaultValue) { - try { - return getUri(name, new URI(defaultValue)); - } catch (final URISyntaxException e) { - throw new IllegalArgumentException(e.getMessage()); - } - } - - /** - * Store only those properties defined at this local level - * - * @param file The file to write to - * @throws IOException If the file can't be found or there is an io error - */ - public void storeLocal(final File file) throws IOException { - final BufferedOutputStream out = - new BufferedOutputStream(new FileOutputStream(file)); - try { - storeLocal(out); - } finally { - out.close(); - } - } - - /** - * Returns a copy of only the local values of this props - */ - public Props local() { - return new Props(null, this._current); - } - - /** - * Store only those properties defined at this local level - * - * @param out The output stream to write to - * @throws IOException If the file can't be found or there is an io error - */ - public void storeLocal(final OutputStream out) throws IOException { - final Properties p = new Properties(); - for (final String key : this._current.keySet()) { - p.setProperty(key, get(key)); - } - p.store(out, null); - } - - /** - * Returns a java.util.Properties file populated with the current Properties in here. - * Note: if you want to import parent properties (e.g., database credentials), please use - * toAllProperties - */ - public Properties toProperties() { - final Properties p = new Properties(); - for (final String key : this._current.keySet()) { - p.setProperty(key, get(key)); - } - - return p; - } - - /** - * Returns a java.util.Properties file populated with both current and parent properties. - */ - public Properties toAllProperties() { - Properties allProp = new Properties(); - // import local properties - allProp.putAll(toProperties()); - // import parent properties - if(_parent != null){ - allProp.putAll(_parent.toProperties()); - } - return allProp; - } - - /** - * Store all properties, those local and also those in parent props - * - * @param file The file to store to - * @throws IOException If there is an error writing - */ - public void storeFlattened(final File file) throws IOException { - final BufferedOutputStream out = - new BufferedOutputStream(new FileOutputStream(file)); - try { - storeFlattened(out); - } finally { - out.close(); - } - } - - /** - * Store all properties, those local and also those in parent props - * - * @param out The stream to write to - * @throws IOException If there is an error writing - */ - public void storeFlattened(final OutputStream out) throws IOException { - final Properties p = new Properties(); - for (Props curr = this; curr != null; curr = curr.getParent()) { - for (final String key : curr.localKeySet()) { - if (!p.containsKey(key)) { - p.setProperty(key, get(key)); - } - } - } - - p.store(out, null); - } - - /** - * Returns a map of all the flattened properties, the item in the returned map is sorted - * alphabetically by the key value. - * - * @Return - */ - public Map getFlattened() { - final TreeMap returnVal = new TreeMap<>(); - returnVal.putAll(getMapByPrefix("")); - return returnVal; - } - - /** - * Get a map of all properties by string prefix - * - * @param prefix The string prefix - */ - public Map getMapByPrefix(final String prefix) { - final Map values = this._parent == null ? new HashMap<>() : - this._parent.getMapByPrefix(prefix); - - // when there is a conflict, value from the child takes the priority. - for (final String key : this.localKeySet()) { - if (key.startsWith(prefix)) { - values.put(key.substring(prefix.length()), get(key)); - } - } - return values; - } - - /** - * Returns a set of all keys, including the parents - */ - public Set getKeySet() { - final HashSet keySet = new HashSet<>(); - - keySet.addAll(localKeySet()); - - if (this._parent != null) { - keySet.addAll(this._parent.getKeySet()); - } - - return keySet; - } - - /** - * Logs the property in the given logger - */ - public void logProperties(final Logger logger, final String comment) { - logger.info(comment); - - for (final String key : getKeySet()) { - logger.info(" key=" + key + " value=" + get(key)); - } - } - - /** - */ - @Override - public boolean equals(final Object o) { - if (o == this) { - return true; - } else if (o == null) { - return false; - } else if (o.getClass() != Props.class) { - return false; - } - - final Props p = (Props) o; - return this._current.equals(p._current) && Utils.equals(this._parent, p._parent); - } - - /** - * Returns true if the properties are equivalent, regardless of the hierarchy. - */ - public boolean equalsProps(final Props p) { - if (p == null) { - return false; - } - - final Set myKeySet = getKeySet(); - for (final String s : myKeySet) { - if (!get(s).equals(p.get(s))) { - return false; - } - } - - return myKeySet.size() == p.getKeySet().size(); - } - - /** - * - */ - @Override - public int hashCode() { - int code = this._current.hashCode(); - if (this._parent != null) { - code += this._parent.hashCode(); - } - return code; - } - - /** - * - */ - @Override - public String toString() { - final StringBuilder builder = new StringBuilder("{"); - for (final Map.Entry entry : this._current.entrySet()) { - builder.append(entry.getKey()); - builder.append(": "); - builder.append(entry.getValue()); - builder.append(", "); - } - if (this._parent != null) { - builder.append(" parent = "); - builder.append(this._parent.toString()); - } - builder.append("}"); - return builder.toString(); - } - - public String getSource() { - return this.source; - } - - public void setSource(final String source) { - this.source = source; - } -} diff --git a/eventchecker-appjoint/src/main/java/com/webank/wedatasphere/dss/appjoint/schedulis/jobtype/utils/Utils.java b/eventchecker-appjoint/src/main/java/com/webank/wedatasphere/dss/appjoint/schedulis/jobtype/utils/Utils.java deleted file mode 100644 index 682033e55e..0000000000 --- a/eventchecker-appjoint/src/main/java/com/webank/wedatasphere/dss/appjoint/schedulis/jobtype/utils/Utils.java +++ /dev/null @@ -1,35 +0,0 @@ -/* - * Copyright 2019 WeBank - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package com.webank.wedatasphere.dss.appjoint.schedulis.jobtype.utils; - -/** - * A util helper class full of static methods that are commonly used. - */ -public class Utils { - /** - * Equivalent to Object.equals except that it handles nulls. If a and b are both null, true is - * returned. - */ - public static boolean equals(final Object a, final Object b) { - if (a == null || b == null) { - return a == b; - } - - return a.equals(b); - } -} diff --git a/eventchecker-appjoint/src/main/resources/appjoint.properties b/eventchecker-appjoint/src/main/resources/appjoint.properties deleted file mode 100644 index 86c926c42a..0000000000 --- a/eventchecker-appjoint/src/main/resources/appjoint.properties +++ /dev/null @@ -1,24 +0,0 @@ -# -# Copyright 2019 WeBank -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# - -msg.eventchecker.jdo.option.name=msg -msg.eventchecker.jdo.option.url=jdbc:mysql://127.0..0.1:3306/ -msg.eventchecker.jdo.option.username=user -msg.eventchecker.jdo.option.password= - - - diff --git a/eventchecker-appjoint/src/main/resources/log4j.properties b/eventchecker-appjoint/src/main/resources/log4j.properties deleted file mode 100644 index 0807e60877..0000000000 --- a/eventchecker-appjoint/src/main/resources/log4j.properties +++ /dev/null @@ -1,37 +0,0 @@ -# -# Copyright 2019 WeBank -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# - -### set log levels ### - -log4j.rootCategory=INFO,console - -log4j.appender.console=org.apache.log4j.ConsoleAppender -log4j.appender.console.Threshold=INFO -log4j.appender.console.layout=org.apache.log4j.PatternLayout -#log4j.appender.console.layout.ConversionPattern= %d{ISO8601} %-5p (%t) [%F:%M(%L)] - %m%n -log4j.appender.console.layout.ConversionPattern= %d{ISO8601} %-5p (%t) %p %c{1} - %m%n - - -log4j.appender.com.webank.bdp.ide.core=org.apache.log4j.DailyRollingFileAppender -log4j.appender.com.webank.bdp.ide.core.Threshold=INFO -log4j.additivity.com.webank.bdp.ide.core=false -log4j.appender.com.webank.bdp.ide.core.layout=org.apache.log4j.PatternLayout -log4j.appender.com.webank.bdp.ide.core.Append=true -log4j.appender.com.webank.bdp.ide.core.File=logs/linkis.log -log4j.appender.com.webank.bdp.ide.core.layout.ConversionPattern= %d{ISO8601} %-5p (%t) [%F:%M(%L)] - %m%n - -log4j.logger.org.springframework=INFO diff --git a/eventchecker-appjoint/src/main/resources/log4j2.xml b/eventchecker-appjoint/src/main/resources/log4j2.xml deleted file mode 100644 index 3923cd9f39..0000000000 --- a/eventchecker-appjoint/src/main/resources/log4j2.xml +++ /dev/null @@ -1,39 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - diff --git a/eventchecker-appjoint/src/main/scala/com/webank/wedatasphere/dss/appjoint/schedulis/jobtype/execution/EventCheckerAppJoint.scala b/eventchecker-appjoint/src/main/scala/com/webank/wedatasphere/dss/appjoint/schedulis/jobtype/execution/EventCheckerAppJoint.scala deleted file mode 100644 index a9c6b0e4e0..0000000000 --- a/eventchecker-appjoint/src/main/scala/com/webank/wedatasphere/dss/appjoint/schedulis/jobtype/execution/EventCheckerAppJoint.scala +++ /dev/null @@ -1,51 +0,0 @@ -/* - * Copyright 2019 WeBank - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package com.webank.wedatasphere.dss.appjoint.schedulis.jobtype.execution - -import java.util - -import com.webank.wedatasphere.dss.appjoint.AppJoint -import com.webank.wedatasphere.dss.appjoint.execution.NodeExecution -import com.webank.wedatasphere.dss.appjoint.service.AppJointUrlImpl - -/** - * Created by enjoyyin on 2019/11/5. - */ -class EventCheckerAppJoint extends AppJointUrlImpl with AppJoint { - - private var params: util.Map[String, AnyRef] = _ - private var nodeExecution: NodeExecution = _ - - override def getAppJointName: String = "EventChecker" - - override def init(baseUrl: String, params: util.Map[String, AnyRef]): Unit = { - setBaseUrl(baseUrl) - this.params = params - } - - override def getNodeExecution: NodeExecution = { - if(nodeExecution == null) synchronized { - if(nodeExecution == null) { - nodeExecution = new EventCheckerNodeExecution() - nodeExecution.setBaseUrl(getBaseUrl) - nodeExecution.init(params) - } - } - nodeExecution - } -} diff --git a/eventchecker-appjoint/src/main/scala/com/webank/wedatasphere/dss/appjoint/schedulis/jobtype/execution/EventCheckerNodeExecution.scala b/eventchecker-appjoint/src/main/scala/com/webank/wedatasphere/dss/appjoint/schedulis/jobtype/execution/EventCheckerNodeExecution.scala deleted file mode 100644 index 36876ee36a..0000000000 --- a/eventchecker-appjoint/src/main/scala/com/webank/wedatasphere/dss/appjoint/schedulis/jobtype/execution/EventCheckerNodeExecution.scala +++ /dev/null @@ -1,165 +0,0 @@ -/* - * Copyright 2019 WeBank - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package com.webank.wedatasphere.dss.appjoint.schedulis.jobtype.execution; - -import java.util -import java.util.{Properties, UUID} - -import com.webank.wedatasphere.dss.appjoint.execution.common._ -import com.webank.wedatasphere.dss.appjoint.execution.core._ -import com.webank.wedatasphere.dss.appjoint.schedulis.jobtype.entity.EventChecker -import com.webank.wedatasphere.dss.appjoint.service.AppJointUrl -import com.webank.wedatasphere.dss.appjoint.service.session.Session -import com.webank.wedatasphere.linkis.common.utils.Utils -import com.webank.wedatasphere.linkis.storage.LineRecord -import org.apache.commons.io.IOUtils - - - -/** - * Created by allenlliu on 2019/11/11. - */ -class EventCheckerNodeExecution extends LongTermNodeExecution with AppJointUrl with Procedure with Killable{ - import scala.collection.JavaConversions.mapAsScalaMap - var appjointParams : scala.collection.mutable.Map[String,AnyRef]= null - - /** - * 表示任务能否提交到该AppJoint去执行 - * - * @param node AppJointNode - * @return true is ok while false is not - */ - override def canExecute(node: AppJointNode, context: NodeContext, session: Session): Boolean = node.getNodeType.toLowerCase.contains("eventchecker") - - - - override def progress(action: NodeExecutionAction): Float = { - //temp set - 0.5f - } - - override def log(action: NodeExecutionAction): String = { - action match { - case action: EventCheckerNodeExecutionAction => { - if (!action.state.isCompleted) { - "EventChecker is sending or waiting for message" - }else{ - "EventChecker successfully received or sent message" - } - } - } - } - - override def kill(action: NodeExecutionAction): Boolean = action match { - case longTermAction: EventCheckerNodeExecutionAction => - getScheduler.removeAsyncResponse(longTermAction) - true - } - - protected def putErrorMsg(errorMsg: String, t: Throwable, action: EventCheckerNodeExecutionAction): EventCheckerNodeExecutionAction = t match { - - case t: Exception => - val response = action.response - response.setErrorMsg(errorMsg) - response.setException(t) - response.setIsSucceed(false) - action - } - - override def init(params: util.Map[String, AnyRef]): Unit = { - this.appjointParams = params - } - - override def submit(node: AppJointNode, context: NodeContext, session:Session): NodeExecutionAction = { - val nodeAction = new EventCheckerNodeExecutionAction() - nodeAction.setId(UUID.randomUUID().toString()) - val jobName = node.getName - val scalaParams: scala.collection.mutable.Map[String,Object] =context.getRuntimeMap - val properties = new Properties() - this.appjointParams.foreach{ - case (key: String, value: Object) => properties.put(key, value.toString) - } - scalaParams.foreach { case (key: String, value: Object) => - properties.put(key, value.toString) - } - val ec = new EventChecker(jobName,properties,nodeAction) - ec.run() - nodeAction.setEc(ec) - nodeAction - } - - override def state(action: NodeExecutionAction): NodeExecutionState = { - action match { - case action: EventCheckerNodeExecutionAction => { - if (action.state.isCompleted) return action.state - if(action.eventType.equals("RECEIVE")){ - action.ec.receiveMsg() - } - action.state - } - } - } - - override def result(action: NodeExecutionAction, nodeContext: NodeContext): CompletedNodeExecutionResponse = { - val response = new CompletedNodeExecutionResponse - action match { - case action:EventCheckerNodeExecutionAction =>{ - if(action.state.equals(NodeExecutionState.Success)) { - if(action.saveKeyAndValue != null){ - val resultSetWriter = nodeContext.createTextResultSetWriter() - Utils.tryFinally { - resultSetWriter.addMetaData(null) - resultSetWriter.addRecord(new LineRecord(action.saveKeyAndValue)) - }(IOUtils.closeQuietly(resultSetWriter)) - } - response.setIsSucceed(true) - }else{ - response.setIsSucceed(false) - } - response - } - case _ => - response.setIsSucceed(false); - response - } - - } - - - private var baseUrl:String ="" - - override def getBaseUrl: String = baseUrl - - override def setBaseUrl(basicUrl: String): Unit = { - this.baseUrl = basicUrl - } - - override def createAsyncNodeExecutionResponse(node: AppJointNode, context: NodeContext, action: NodeExecutionAction): AsyncNodeExecutionResponse = { - action match { - case action: EventCheckerNodeExecutionAction => { - val response = new AsyncNodeExecutionResponse - response.setAction(action) - response.setAppJointNode(node) - response.setNodeContext(context) - response.setMaxLoopTime(action.ec.maxWaitTime) - response.setAskStatePeriod(action.ec.queryFrequency) - response - } - } - } -} diff --git a/eventchecker-appjoint/src/main/scala/com/webank/wedatasphere/dss/appjoint/schedulis/jobtype/execution/EventCheckerNodeExecutionAction.scala b/eventchecker-appjoint/src/main/scala/com/webank/wedatasphere/dss/appjoint/schedulis/jobtype/execution/EventCheckerNodeExecutionAction.scala deleted file mode 100644 index 0a92f9a9e3..0000000000 --- a/eventchecker-appjoint/src/main/scala/com/webank/wedatasphere/dss/appjoint/schedulis/jobtype/execution/EventCheckerNodeExecutionAction.scala +++ /dev/null @@ -1,66 +0,0 @@ -/* - * Copyright 2019 WeBank - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package com.webank.wedatasphere.dss.appjoint.schedulis.jobtype.execution - -import com.webank.wedatasphere.dss.appjoint.execution.common.{AbstractNodeExecutionAction, CompletedNodeExecutionResponse, LongTermNodeExecutionAction, NodeExecutionState} -import com.webank.wedatasphere.dss.appjoint.schedulis.jobtype.entity.EventChecker - -/** - * Created by allenlliu on 2019/11/12. - */ -class EventCheckerNodeExecutionAction extends AbstractNodeExecutionAction with LongTermNodeExecutionAction { - private[this] var _state: NodeExecutionState = _ - private var schedulerId: Int = _ - - def state: NodeExecutionState = _state - - def setState(value: NodeExecutionState): Unit = { - _state = value - } - - - val response = new CompletedNodeExecutionResponse - - private[this] var _saveKeyAndValue: String = null - - def saveKeyAndValue: String = _saveKeyAndValue - - def saveKeyAndValue(value: String): Unit = { - _saveKeyAndValue = value - } - - private[this] var _eventType: String = "SEND" - - def eventType: String = _eventType - - def eventType(value: String): Unit = { - _eventType = value - } - - private[this] var _ec: EventChecker = null - - def ec: EventChecker = _ec - - def setEc(value: EventChecker): Unit = { - _ec = value - } - - override def setSchedulerId(schedulerId: Int): Unit = this.schedulerId = schedulerId - - override def getSchedulerId: Int = schedulerId -} diff --git a/plugins/linkis/linkis-appjoint-entrance/bin/start-linkis-appjoint-entrance.sh b/plugins/linkis/linkis-appjoint-entrance/bin/start-linkis-appjoint-entrance.sh deleted file mode 100644 index 4436def35d..0000000000 --- a/plugins/linkis/linkis-appjoint-entrance/bin/start-linkis-appjoint-entrance.sh +++ /dev/null @@ -1,37 +0,0 @@ -#!/bin/bash -cd `dirname $0` -cd .. -HOME=`pwd` - -export SERVER_PID=$HOME/bin/linkis.pid -export SERVER_LOG_PATH=$HOME/logs -export SERVER_CLASS=com.webank.wedatasphere.linkis.DataWorkCloudApplication - -if test -z "$SERVER_HEAP_SIZE" -then - export SERVER_HEAP_SIZE="512M" -fi - -if test -z "$SERVER_JAVA_OPTS" -then - export SERVER_JAVA_OPTS=" -Xmx$SERVER_HEAP_SIZE -XX:+UseG1GC -Xloggc:$HOME/logs/linkis-gc.log" -fi - -if [[ -f "${SERVER_PID}" ]]; then - pid=$(cat ${SERVER_PID}) - if kill -0 ${pid} >/dev/null 2>&1; then - echo "Server is already running." - exit 1 - fi -fi - -nohup java $SERVER_JAVA_OPTS -cp $HOME/conf:$HOME/lib/* $SERVER_CLASS 2>&1 > $SERVER_LOG_PATH/linkis.out & -pid=$! -if [[ -z "${pid}" ]]; then - echo "server $SERVER_NAME start failed!" - exit 1 -else - echo "server $SERVER_NAME start succeeded!" - echo $pid > $SERVER_PID - sleep 1 -fi diff --git a/plugins/linkis/linkis-appjoint-entrance/bin/stop-linkis-appjoint-entrance.sh b/plugins/linkis/linkis-appjoint-entrance/bin/stop-linkis-appjoint-entrance.sh deleted file mode 100644 index a1528811c7..0000000000 --- a/plugins/linkis/linkis-appjoint-entrance/bin/stop-linkis-appjoint-entrance.sh +++ /dev/null @@ -1,47 +0,0 @@ -#!/bin/bash - -cd `dirname $0` -cd .. -HOE=`pwd` - -export DSS_ENTRANCE_PID=$HOE/bin/linkis.pid - -function wait_for_DSS_ENGINE_MANAGER_to_die() { - local pid - local count - pid=$1 - timeout=$2 - count=0 - timeoutTime=$(date "+%s") - let "timeoutTime+=$timeout" - currentTime=$(date "+%s") - forceKill=1 - - while [[ $currentTime -lt $timeoutTime ]]; do - $(kill ${pid} > /dev/null 2> /dev/null) - if kill -0 ${pid} > /dev/null 2>&1; then - sleep 3 - else - forceKill=0 - break - fi - currentTime=$(date "+%s") - done - - if [[ forceKill -ne 0 ]]; then - $(kill -9 ${pid} > /dev/null 2> /dev/null) - fi -} - -if [[ ! -f "${DSS_ENTRANCE_PID}" ]]; then - echo "AppJoint Entrance is not running" -else - pid=$(cat ${DSS_ENTRANCE_PID}) - if [[ -z "${pid}" ]]; then - echo "AppJoint Entrance is not running" - else - wait_for_DSS_ENGINE_MANAGER_to_die $pid 40 - $(rm -f ${DSS_ENTRANCE_PID}) - echo "AppJoint Entrance is stopped." - fi -fi diff --git a/plugins/linkis/linkis-appjoint-entrance/conf/application.yml b/plugins/linkis/linkis-appjoint-entrance/conf/application.yml deleted file mode 100644 index 2f21e55c0d..0000000000 --- a/plugins/linkis/linkis-appjoint-entrance/conf/application.yml +++ /dev/null @@ -1,22 +0,0 @@ -server: - port: 9006 -spring: - application: - name: appjointEntrance - -eureka: - client: - serviceUrl: - defaultZone: http://127.0.0.1:20303/eureka/ - instance: - metadata-map: - test: enjoyyin -management: - endpoints: - web: - exposure: - include: refresh,info -logging: - config: classpath:log4j2.xml -# register-with-eureka: false -# fetch-registry: false diff --git a/plugins/linkis/linkis-appjoint-entrance/conf/linkis.properties b/plugins/linkis/linkis-appjoint-entrance/conf/linkis.properties deleted file mode 100644 index 1317805e2a..0000000000 --- a/plugins/linkis/linkis-appjoint-entrance/conf/linkis.properties +++ /dev/null @@ -1,38 +0,0 @@ -# -# Copyright 2019 WeBank -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# - - -wds.linkis.server.restful.scan.packages=com.webank.wedatasphere.linkis.entrance.restful - -wds.linkis.engine.application.name=hiveEngine -wds.linkis.enginemanager.application.name=hiveEngineManager - -wds.linkis.query.application.name=cloud-publicservice - -wds.linkis.console.config.application.name=cloud-publicservice -wds.linkis.engine.creation.wait.time.max=20m -wds.linkis.server.version=v1 -#hadoop config dir -#hadoop.config.dir=/appcom/config/hadoop-config -wds.linkis.entrance.config.logPath=file:///tmp/linkis/ - -wds.linkis.resultSet.store.path=hdfs:///tmp/linkis - -wds.linkis.server.socket.mode=true - - -wds.linkis.entrance.job.persist.wait.max=10s diff --git a/plugins/linkis/linkis-appjoint-entrance/conf/log4j2.xml b/plugins/linkis/linkis-appjoint-entrance/conf/log4j2.xml deleted file mode 100644 index f48149dd2d..0000000000 --- a/plugins/linkis/linkis-appjoint-entrance/conf/log4j2.xml +++ /dev/null @@ -1,39 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - diff --git a/plugins/linkis/linkis-appjoint-entrance/pom.xml b/plugins/linkis/linkis-appjoint-entrance/pom.xml deleted file mode 100644 index bb7db19018..0000000000 --- a/plugins/linkis/linkis-appjoint-entrance/pom.xml +++ /dev/null @@ -1,141 +0,0 @@ - - - - - - dss - com.webank.wedatasphere.dss - 0.9.1 - - 4.0.0 - - linkis-appjoint-entrance - ${dss.version} - - - - com.webank.wedatasphere.dss - dss-appjoint-core - ${dss.version} - - - - com.webank.wedatasphere.dss - dss-appjoint-loader - ${dss.version} - - - - - com.webank.wedatasphere.linkis - linkis-ujes-entrance - ${linkis.version} - - - org.springframework.cloud - spring-cloud-starter-openfeign - - - - - com.webank.wedatasphere.linkis - linkis-cloudRPC - ${linkis.version} - - - com.webank.wedatasphere.linkis - linkis-storage - ${linkis.version} - - - com.webank.wedatasphere.linkis - linkis-httpclient - ${linkis.version} - - - - - - - - - org.apache.maven.plugins - maven-deploy-plugin - - - - net.alchim31.maven - scala-maven-plugin - - - org.apache.maven.plugins - maven-jar-plugin - - - org.apache.maven.plugins - maven-assembly-plugin - 2.3 - false - - - make-assembly - package - - single - - - - src/main/assembly/distribution.xml - - - - - - false - linkis-appjoint-entrance - false - false - - src/main/assembly/distribution.xml - - - - - - - src/main/java - - **/*.xml - - - - src/main/resources - - **/*.properties - **/application.yml - **/bootstrap.yml - **/log4j2.xml - - - - - - - \ No newline at end of file diff --git a/plugins/linkis/linkis-appjoint-entrance/src/main/assembly/distribution.xml b/plugins/linkis/linkis-appjoint-entrance/src/main/assembly/distribution.xml deleted file mode 100644 index b3ac1a0b59..0000000000 --- a/plugins/linkis/linkis-appjoint-entrance/src/main/assembly/distribution.xml +++ /dev/null @@ -1,335 +0,0 @@ - - - - linkis-appjoint-entrance - - zip - - true - linkis-appjoint-entrance - - - - - - lib - true - true - false - true - true - - - antlr:antlr:jar - aopalliance:aopalliance:jar - asm:asm:jar - cglib:cglib:jar - - - - - - - com.fasterxml.jackson.core:jackson-annotations:jar - com.fasterxml.jackson.core:jackson-core:jar - com.fasterxml.jackson.core:jackson-databind:jar - - com.fasterxml.jackson.datatype:jackson-datatype-jdk8:jar - com.fasterxml.jackson.datatype:jackson-datatype-jsr310:jar - com.fasterxml.jackson.jaxrs:jackson-jaxrs-base:jar - com.fasterxml.jackson.jaxrs:jackson-jaxrs-json-provider:jar - com.fasterxml.jackson.module:jackson-module-jaxb-annotations:jar - com.fasterxml.jackson.module:jackson-module-parameter-names:jar - com.fasterxml.jackson.module:jackson-module-paranamer:jar - com.fasterxml.jackson.module:jackson-module-scala_2.11:jar - - - com.google.code.findbugs:annotations:jar - - com.google.code.gson:gson:jar - com.google.guava:guava:jar - com.google.inject:guice:jar - - com.netflix.archaius:archaius-core:jar - com.netflix.eureka:eureka-client:jar - com.netflix.eureka:eureka-core:jar - com.netflix.hystrix:hystrix-core:jar - com.netflix.netflix-commons:netflix-commons-util:jar - com.netflix.netflix-commons:netflix-eventbus:jar - com.netflix.netflix-commons:netflix-infix:jar - com.netflix.netflix-commons:netflix-statistics:jar - com.netflix.ribbon:ribbon:jar - com.netflix.ribbon:ribbon-core:jar - com.netflix.ribbon:ribbon-eureka:jar - com.netflix.ribbon:ribbon-httpclient:jar - com.netflix.ribbon:ribbon-loadbalancer:jar - com.netflix.ribbon:ribbon-transport:jar - com.netflix.servo:servo-core:jar - com.sun.jersey.contribs:jersey-apache-client4:jar - com.sun.jersey:jersey-client:jar - com.sun.jersey:jersey-core:jar - com.sun.jersey:jersey-json:jar - com.sun.jersey:jersey-server:jar - com.sun.jersey:jersey-servlet:jar - com.sun.xml.bind:jaxb-impl:jar - com.thoughtworks.paranamer:paranamer:jar - com.thoughtworks.xstream:xstream:jar - com.webank.wedatasphere.linkis:linkis-common:jar - com.webank.wedatasphere.linkis:linkis-module:jar - - - - - - - - - - commons-httpclient:commons-httpclient:jar - commons-io:commons-io:jar - commons-jxpath:commons-jxpath:jar - commons-lang:commons-lang:jar - commons-logging:commons-logging:jar - commons-net:commons-net:jar - commons-pool:commons-pool:jar - io.micrometer:micrometer-core:jar - io.netty:netty:jar - io.netty:netty-all:jar - io.netty:netty-buffer:jar - io.netty:netty-codec:jar - io.netty:netty-codec-http:jar - io.netty:netty-common:jar - io.netty:netty-handler:jar - io.netty:netty-transport:jar - io.netty:netty-transport-native-epoll:jar - io.reactivex:rxjava:jar - io.reactivex:rxnetty:jar - io.reactivex:rxnetty-contexts:jar - io.reactivex:rxnetty-servo:jar - javax.activation:activation:jar - javax.annotation:javax.annotation-api:jar - javax.inject:javax.inject:jar - javax.servlet:javax.servlet-api:jar - javax.servlet.jsp:jsp-api:jar - javax.validation:validation-api:jar - javax.websocket:javax.websocket-api:jar - javax.ws.rs:javax.ws.rs-api:jar - javax.xml.bind:jaxb-api:jar - javax.xml.stream:stax-api:jar - joda-time:joda-time:jar - log4j:log4j:jar - mysql:mysql-connector-java:jar - org.antlr:antlr-runtime:jar - org.antlr:stringtemplate:jar - - org.apache.commons:commons-math:jar - org.apache.commons:commons-math3:jar - org.apache.curator:curator-client:jar - org.apache.curator:curator-framework:jar - org.apache.curator:curator-recipes:jar - org.apache.directory.api:api-asn1-api:jar - org.apache.directory.api:api-util:jar - org.apache.directory.server:apacheds-i18n:jar - org.apache.directory.server:apacheds-kerberos-codec:jar - - - - - - - - org.apache.logging.log4j:log4j-api:jar - org.apache.logging.log4j:log4j-core:jar - org.apache.logging.log4j:log4j-jul:jar - org.apache.logging.log4j:log4j-slf4j-impl:jar - org.apache.zookeeper:zookeeper:jar - org.aspectj:aspectjweaver:jar - org.bouncycastle:bcpkix-jdk15on:jar - org.bouncycastle:bcprov-jdk15on:jar - org.codehaus.jackson:jackson-jaxrs:jar - org.codehaus.jackson:jackson-xc:jar - org.codehaus.jettison:jettison:jar - org.codehaus.woodstox:stax2-api:jar - org.codehaus.woodstox:woodstox-core-asl:jar - org.eclipse.jetty:jetty-annotations:jar - org.eclipse.jetty:jetty-client:jar - org.eclipse.jetty:jetty-continuation:jar - org.eclipse.jetty:jetty-http:jar - org.eclipse.jetty:jetty-io:jar - org.eclipse.jetty:jetty-plus:jar - org.eclipse.jetty:jetty-security:jar - org.eclipse.jetty:jetty-server:jar - org.eclipse.jetty:jetty-servlet:jar - org.eclipse.jetty:jetty-servlets:jar - org.eclipse.jetty:jetty-util:jar - org.eclipse.jetty:jetty-webapp:jar - org.eclipse.jetty:jetty-xml:jar - org.eclipse.jetty.websocket:javax-websocket-client-impl:jar - org.eclipse.jetty.websocket:javax-websocket-server-impl:jar - org.eclipse.jetty.websocket:websocket-api:jar - org.eclipse.jetty.websocket:websocket-client:jar - org.eclipse.jetty.websocket:websocket-common:jar - org.eclipse.jetty.websocket:websocket-server:jar - org.eclipse.jetty.websocket:websocket-servlet:jar - org.fusesource.leveldbjni:leveldbjni-all:jar - org.glassfish.hk2:class-model:jar - org.glassfish.hk2:config-types:jar - org.glassfish.hk2.external:aopalliance-repackaged:jar - org.glassfish.hk2.external:asm-all-repackaged:jar - org.glassfish.hk2.external:bean-validator:jar - org.glassfish.hk2.external:javax.inject:jar - org.glassfish.hk2:hk2:jar - org.glassfish.hk2:hk2-api:jar - org.glassfish.hk2:hk2-config:jar - org.glassfish.hk2:hk2-core:jar - org.glassfish.hk2:hk2-locator:jar - org.glassfish.hk2:hk2-runlevel:jar - org.glassfish.hk2:hk2-utils:jar - org.glassfish.hk2:osgi-resource-locator:jar - org.glassfish.hk2:spring-bridge:jar - org.glassfish.jersey.bundles:jaxrs-ri:jar - org.glassfish.jersey.bundles.repackaged:jersey-guava:jar - org.glassfish.jersey.containers:jersey-container-servlet:jar - org.glassfish.jersey.containers:jersey-container-servlet-core:jar - org.glassfish.jersey.core:jersey-client:jar - org.glassfish.jersey.core:jersey-common:jar - org.glassfish.jersey.core:jersey-server:jar - org.glassfish.jersey.ext:jersey-entity-filtering:jar - org.glassfish.jersey.ext:jersey-spring3:jar - org.glassfish.jersey.media:jersey-media-jaxb:jar - org.glassfish.jersey.media:jersey-media-json-jackson:jar - org.glassfish.jersey.media:jersey-media-multipart:jar - org.hdrhistogram:HdrHistogram:jar - org.javassist:javassist:jar - org.json4s:json4s-ast_2.11:jar - org.json4s:json4s-core_2.11:jar - org.json4s:json4s-jackson_2.11:jar - org.jvnet.mimepull:mimepull:jar - org.jvnet:tiger-types:jar - org.latencyutils:LatencyUtils:jar - org.mortbay.jasper:apache-el:jar - org.mortbay.jetty:jetty:jar - org.mortbay.jetty:jetty-util:jar - org.ow2.asm:asm-analysis:jar - org.ow2.asm:asm-commons:jar - org.ow2.asm:asm-tree:jar - org.reflections:reflections:jar - org.scala-lang.modules:scala-parser-combinators_2.11:jar - org.scala-lang.modules:scala-xml_2.11:jar - org.scala-lang:scala-compiler:jar - org.scala-lang:scala-library:jar - org.scala-lang:scala-reflect:jar - org.scala-lang:scalap:jar - org.slf4j:jul-to-slf4j:jar - org.slf4j:slf4j-api:jar - org.springframework.boot:spring-boot:jar - org.springframework.boot:spring-boot-actuator:jar - org.springframework.boot:spring-boot-actuator-autoconfigure:jar - org.springframework.boot:spring-boot-autoconfigure:jar - org.springframework.boot:spring-boot-starter:jar - org.springframework.boot:spring-boot-starter-actuator:jar - org.springframework.boot:spring-boot-starter-aop:jar - org.springframework.boot:spring-boot-starter-jetty:jar - org.springframework.boot:spring-boot-starter-json:jar - org.springframework.boot:spring-boot-starter-log4j2:jar - org.springframework.boot:spring-boot-starter-web:jar - org.springframework.cloud:spring-cloud-commons:jar - org.springframework.cloud:spring-cloud-config-client:jar - org.springframework.cloud:spring-cloud-context:jar - - org.springframework.cloud:spring-cloud-netflix-core:jar - org.springframework.cloud:spring-cloud-netflix-eureka-client:jar - - org.springframework.cloud:spring-cloud-starter:jar - org.springframework.cloud:spring-cloud-starter-config:jar - org.springframework.cloud:spring-cloud-starter-eureka:jar - org.springframework.cloud:spring-cloud-starter-netflix-archaius:jar - org.springframework.cloud:spring-cloud-starter-netflix-eureka-client:jar - org.springframework.cloud:spring-cloud-starter-netflix-ribbon:jar - org.springframework.security:spring-security-crypto:jar - org.springframework.security:spring-security-rsa:jar - org.springframework:spring-aop:jar - org.springframework:spring-beans:jar - org.springframework:spring-context:jar - org.springframework:spring-core:jar - org.springframework:spring-expression:jar - org.springframework:spring-jcl:jar - org.springframework:spring-web:jar - org.springframework:spring-webmvc:jar - org.tukaani:xz:jar - org.yaml:snakeyaml:jar - - xerces:xercesImpl:jar - xmlenc:xmlenc:jar - xmlpull:xmlpull:jar - xpp3:xpp3_min:jar - - - - - - - - ${basedir}/conf - - * - - 0777 - conf - unix - - - ${basedir}/bin - - * - - 0777 - bin - unix - - - . - - */** - - logs - - - - - diff --git a/plugins/linkis/linkis-appjoint-entrance/src/main/java/com/webank/wedatasphere/dss/linkis/appjoint/entrance/AppJointEntranceRestfulApi.java b/plugins/linkis/linkis-appjoint-entrance/src/main/java/com/webank/wedatasphere/dss/linkis/appjoint/entrance/AppJointEntranceRestfulApi.java deleted file mode 100644 index 2b12ad7cb7..0000000000 --- a/plugins/linkis/linkis-appjoint-entrance/src/main/java/com/webank/wedatasphere/dss/linkis/appjoint/entrance/AppJointEntranceRestfulApi.java +++ /dev/null @@ -1,96 +0,0 @@ -/* - * Copyright 2019 WeBank - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package com.webank.wedatasphere.dss.linkis.appjoint.entrance; - -import com.webank.wedatasphere.dss.appjoint.AppJoint; -import com.webank.wedatasphere.dss.appjoint.execution.NodeExecution; -import com.webank.wedatasphere.dss.appjoint.execution.core.CallbackLongTermNodeExecution; -import com.webank.wedatasphere.dss.linkis.appjoint.entrance.appjoint.AppJointManager; -import com.webank.wedatasphere.linkis.entrance.EntranceContext; -import com.webank.wedatasphere.linkis.server.Message; -import org.apache.commons.lang.StringUtils; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.stereotype.Component; - -import javax.servlet.http.HttpServletRequest; -import javax.servlet.http.HttpServletResponse; -import javax.ws.rs.Consumes; -import javax.ws.rs.POST; -import javax.ws.rs.Path; -import javax.ws.rs.Produces; -import javax.ws.rs.core.Context; -import javax.ws.rs.core.MediaType; -import javax.ws.rs.core.Response; -import java.util.Map; - -/** - * created by cooperyang on 2019/9/30 - * Description: - */ -@Path("/entrance") -@Component -@Produces(MediaType.APPLICATION_JSON) -@Consumes(MediaType.APPLICATION_JSON) -public class AppJointEntranceRestfulApi { - - public static final String APPJOINT_NAME_STR = "appJointName"; - - public static final String APPJOINT_LOG_STR = "log"; - - public static final String APPJOINT_STATUS_STR = "status"; - - public static final String APPJOINT_PROGRESS_STR = "progress"; - - public static final String ENGINE_TAG_STR = "engineTag"; - - public static final String CALL_BACK_URL = "/api/entrance/callback"; - - /** - * callback接口是为了接收外部系统进行回馈各种信息的接口，外部系统调用此接口必须提供的参数有entranceEngine的唯一标识， - * 因为只有entranceEngine的唯一标识，才能进行定位到相应的job - * jsonNode 是外部系统传递的参数,这些参数可以包含有日志，进度 和 状态等等 - */ - @POST - @Path("/callback") - public Response callback(@Context HttpServletRequest request, - @Context HttpServletResponse response, - Map params) { - Object appJointName = params.get(APPJOINT_NAME_STR); - if(appJointName == null || StringUtils.isBlank(appJointName.toString())) { - return error(APPJOINT_NAME_STR + "为空！"); - } - AppJoint appJoint = AppJointManager.getAppJoint(appJointName.toString()); - if(appJoint == null) { - return error("找不到AppJoint: " + appJointName); - } - NodeExecution nodeExecution = appJoint.getNodeExecution(); - if(!(nodeExecution instanceof CallbackLongTermNodeExecution)) { - return error("找不到CallbackLongTermNodeExecution来处理该请求，实际NodeExecution为 " + nodeExecution.getClass().getSimpleName()); - } - ((CallbackLongTermNodeExecution) nodeExecution).acceptCallback(params); - Message message = Message.ok("处理成功!"); - message.setMethod(CALL_BACK_URL); - return Message.messageToResponse(message); - } - - private Response error(String msg) { - Message message = Message.error(msg); - message.setMethod(CALL_BACK_URL); - return Message.messageToResponse(message); - } -} diff --git a/plugins/linkis/linkis-appjoint-entrance/src/main/scala/com/webank/wedatasphere/dss/linkis/appjoint/entrance/appjoint/AppJointManager.scala b/plugins/linkis/linkis-appjoint-entrance/src/main/scala/com/webank/wedatasphere/dss/linkis/appjoint/entrance/appjoint/AppJointManager.scala deleted file mode 100644 index 5a2dea709f..0000000000 --- a/plugins/linkis/linkis-appjoint-entrance/src/main/scala/com/webank/wedatasphere/dss/linkis/appjoint/entrance/appjoint/AppJointManager.scala +++ /dev/null @@ -1,75 +0,0 @@ -/* - * Copyright 2019 WeBank - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package com.webank.wedatasphere.dss.linkis.appjoint.entrance.appjoint - -import java.util -import java.util.concurrent.TimeUnit - -import com.google.common.cache.CacheBuilder -import com.webank.wedatasphere.dss.appjoint.AppJoint -import com.webank.wedatasphere.dss.appjoint.execution.scheduler.ListenerEventBusNodeExecutionScheduler -import com.webank.wedatasphere.dss.appjoint.loader.AppJointLoader -import com.webank.wedatasphere.dss.common.protocol.RequestDSSApplication -import com.webank.wedatasphere.dss.linkis.appjoint.entrance.conf.AppJointEntranceConfiguration -import com.webank.wedatasphere.linkis.common.utils.{Logging, Utils} -import com.webank.wedatasphere.linkis.rpc.Sender - -/** - * Created by enjoyyin on 2019/11/13. - */ -object AppJointManager extends Logging{ - - private val scheduler = new ListenerEventBusNodeExecutionScheduler - - scheduler.start() - - - private val sender = Sender.getSender(AppJointEntranceConfiguration.DSS_SERVER_SPRING_APPLICATION.getValue) - - private val urlStr:String = "url" - - def getScheduler:ListenerEventBusNodeExecutionScheduler = scheduler - - - /** - * 通过appJointName获取一个可以使用的AppJoint - * @param appJointName appjoint的名字,例如 sendemail,qualitis - * @return - */ - - - def getAppJoint(appJointName: String): AppJoint = { - val request = RequestDSSApplication(appJointName) - val result = Utils.tryCatch{ - sender.ask(request) - }{ - t:Throwable => logger.warn(s"can not get application $appJointName from dss-Server", t) - null - } - var params:java.util.Map[String, Object] = new util.HashMap[String, Object]() - val baseUrl:String = if (result != null){ - result match { - case map:java.util.Map[String, Object] => params = map - if (null != map.get(urlStr)) map.get(urlStr).toString else "" - case _ => "" - } - }else "" - AppJointLoader.getAppJointLoader.getAppJoint("", appJointName, params) - } - -} diff --git a/plugins/linkis/linkis-appjoint-entrance/src/main/scala/com/webank/wedatasphere/dss/linkis/appjoint/entrance/conf/AppJointConst.scala b/plugins/linkis/linkis-appjoint-entrance/src/main/scala/com/webank/wedatasphere/dss/linkis/appjoint/entrance/conf/AppJointConst.scala deleted file mode 100644 index 6aa98b5808..0000000000 --- a/plugins/linkis/linkis-appjoint-entrance/src/main/scala/com/webank/wedatasphere/dss/linkis/appjoint/entrance/conf/AppJointConst.scala +++ /dev/null @@ -1,39 +0,0 @@ -/* - * Copyright 2019 WeBank - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package com.webank.wedatasphere.dss.linkis.appjoint.entrance.conf - -import java.lang -import java.lang.reflect.Type - -import com.google.gson._ -import com.webank.wedatasphere.linkis.server.BDPJettyServerHelper - -/** - * created by cooperyang on 2019/11/11 - * Description: - */ -object AppJointConst { - - implicit val gson = new GsonBuilder().setDateFormat("yyyy-MM-dd'T'HH:mm:ssZ").serializeNulls - .registerTypeAdapter(classOf[java.lang.Double], new JsonSerializer[java.lang.Double] { - override def serialize(t: lang.Double, `type`: Type, jsonSerializationContext: JsonSerializationContext): JsonElement = - if(t == t.longValue()) new JsonPrimitive(t.longValue()) else new JsonPrimitive(t) - }).create - - val WORKFLOW_SHARED_NODES_JOBIDS:String = "workflow.shared.nodes.jobids" -} diff --git a/plugins/linkis/linkis-appjoint-entrance/src/main/scala/com/webank/wedatasphere/dss/linkis/appjoint/entrance/conf/AppJointEntranceConfiguration.scala b/plugins/linkis/linkis-appjoint-entrance/src/main/scala/com/webank/wedatasphere/dss/linkis/appjoint/entrance/conf/AppJointEntranceConfiguration.scala deleted file mode 100644 index 9bb45ed396..0000000000 --- a/plugins/linkis/linkis-appjoint-entrance/src/main/scala/com/webank/wedatasphere/dss/linkis/appjoint/entrance/conf/AppJointEntranceConfiguration.scala +++ /dev/null @@ -1,40 +0,0 @@ -/* - * Copyright 2019 WeBank - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package com.webank.wedatasphere.dss.linkis.appjoint.entrance.conf - -import com.webank.wedatasphere.linkis.common.conf.CommonVars - -/** - * created by cooperyang on 2019/9/26 - * Description: - */ -object AppJointEntranceConfiguration { - val APP_JOINT_CLASS = CommonVars("wds.linkis.external.appjoint", - "com.webank.wedatasphere.dss.linkis.appjoint.example.ExampleAppJoint") - - val APPLICATION_SPRING_APPLICATION = CommonVars("wds.linkis.application.spring.name", "cloud-application") - - val CALLBACK_TIME_OUT:CommonVars[Long] = CommonVars[Long]("wds.linkis.callback.timeout", 5 * 60 * 1000L) - - - val PUBLIC_SERVICE_SPRING_APPLICATION = CommonVars("wds.linkis.publicservice.spring.name", "cloud-publicserivce") - - val GATEWAY_SPRING_APPLICATION = CommonVars("wds.linkis.gateway.spring.name", "dataworkcloud-gateway") - - val DSS_SERVER_SPRING_APPLICATION = CommonVars("wds.linkis.dssserver.spring.name", "dss-server") -} diff --git a/plugins/linkis/linkis-appjoint-entrance/src/main/scala/com/webank/wedatasphere/dss/linkis/appjoint/entrance/conf/AppJointEntranceSpringConfiguration.scala b/plugins/linkis/linkis-appjoint-entrance/src/main/scala/com/webank/wedatasphere/dss/linkis/appjoint/entrance/conf/AppJointEntranceSpringConfiguration.scala deleted file mode 100644 index 7c1d35edd5..0000000000 --- a/plugins/linkis/linkis-appjoint-entrance/src/main/scala/com/webank/wedatasphere/dss/linkis/appjoint/entrance/conf/AppJointEntranceSpringConfiguration.scala +++ /dev/null @@ -1,53 +0,0 @@ -/* - * Copyright 2019 WeBank - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package com.webank.wedatasphere.dss.linkis.appjoint.entrance.conf - -import com.webank.wedatasphere.dss.linkis.appjoint.entrance.execute.AppJointEngineExecutorManagerImpl -import com.webank.wedatasphere.dss.linkis.appjoint.entrance.parser.AppJointEntranceParser -import com.webank.wedatasphere.linkis.entrance.EntranceParser -import com.webank.wedatasphere.linkis.entrance.annotation._ -import com.webank.wedatasphere.linkis.entrance.execute._ -import com.webank.wedatasphere.linkis.scheduler.queue.GroupFactory -import org.slf4j.LoggerFactory -import org.springframework.beans.factory.annotation.Autowired -import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean -import org.springframework.context.annotation.Configuration - -/** - * created by chaogefeng on 2019/9/26 17:44 - * Description: - */ -@Configuration -class AppJointEntranceSpringConfiguration { - private val logger = LoggerFactory.getLogger(classOf[AppJointEntranceSpringConfiguration]) - @EntranceExecutorManagerBeanAnnotation - def generateExternalEntranceExecutorManager(@GroupFactoryBeanAnnotation.GroupFactoryAutowiredAnnotation groupFactory: GroupFactory, - @EngineBuilderBeanAnnotation.EngineBuilderAutowiredAnnotation engineBuilder: EngineBuilder, - @EngineRequesterBeanAnnotation.EngineRequesterAutowiredAnnotation engineRequester: EngineRequester, - @EngineSelectorBeanAnnotation.EngineSelectorAutowiredAnnotation engineSelector: EngineSelector, - @EngineManagerBeanAnnotation.EngineManagerAutowiredAnnotation engineManager: EngineManager, - @Autowired entranceExecutorRulers: Array[EntranceExecutorRuler]): EntranceExecutorManager = - new AppJointEngineExecutorManagerImpl(groupFactory, engineBuilder, engineRequester, engineSelector, engineManager, entranceExecutorRulers) - - - @EntranceParserBeanAnnotation - def generateEntranceParser():EntranceParser = { - new AppJointEntranceParser() - } - -} diff --git a/plugins/linkis/linkis-appjoint-entrance/src/main/scala/com/webank/wedatasphere/dss/linkis/appjoint/entrance/exception/AppJointEntranceIllegalParamException.scala b/plugins/linkis/linkis-appjoint-entrance/src/main/scala/com/webank/wedatasphere/dss/linkis/appjoint/entrance/exception/AppJointEntranceIllegalParamException.scala deleted file mode 100644 index cdb6ca9bed..0000000000 --- a/plugins/linkis/linkis-appjoint-entrance/src/main/scala/com/webank/wedatasphere/dss/linkis/appjoint/entrance/exception/AppJointEntranceIllegalParamException.scala +++ /dev/null @@ -1,28 +0,0 @@ -/* - * Copyright 2019 WeBank - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package com.webank.wedatasphere.dss.linkis.appjoint.entrance.exception - -import com.webank.wedatasphere.linkis.common.exception.ErrorException - -/** - * created by cooperyang on 2019/10/8 - * Description: - */ -case class AppJointEntranceIllegalParamException(errorMsg:String) extends ErrorException(70012, errorMsg) - -case class AppJointErrorException(errorMsg:String) extends ErrorException(70058, errorMsg) \ No newline at end of file diff --git a/plugins/linkis/linkis-appjoint-entrance/src/main/scala/com/webank/wedatasphere/dss/linkis/appjoint/entrance/exception/OperateNotSupportedException.scala b/plugins/linkis/linkis-appjoint-entrance/src/main/scala/com/webank/wedatasphere/dss/linkis/appjoint/entrance/exception/OperateNotSupportedException.scala deleted file mode 100644 index 4177aaf9bb..0000000000 --- a/plugins/linkis/linkis-appjoint-entrance/src/main/scala/com/webank/wedatasphere/dss/linkis/appjoint/entrance/exception/OperateNotSupportedException.scala +++ /dev/null @@ -1,26 +0,0 @@ -/* - * Copyright 2019 WeBank - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package com.webank.wedatasphere.dss.linkis.appjoint.entrance.exception - -import com.webank.wedatasphere.linkis.common.exception.ErrorException - -/** - * created by cooperyang on 2019/9/30 - * Description: - */ -case class OperateNotSupportedException(errMsg:String) extends ErrorException(81235, errMsg) diff --git a/plugins/linkis/linkis-appjoint-entrance/src/main/scala/com/webank/wedatasphere/dss/linkis/appjoint/entrance/exception/RPCFailedException.scala b/plugins/linkis/linkis-appjoint-entrance/src/main/scala/com/webank/wedatasphere/dss/linkis/appjoint/entrance/exception/RPCFailedException.scala deleted file mode 100644 index 0c4796ca0a..0000000000 --- a/plugins/linkis/linkis-appjoint-entrance/src/main/scala/com/webank/wedatasphere/dss/linkis/appjoint/entrance/exception/RPCFailedException.scala +++ /dev/null @@ -1,27 +0,0 @@ -/* - * Copyright 2019 WeBank - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package com.webank.wedatasphere.dss.linkis.appjoint.entrance.exception - -import com.webank.wedatasphere.linkis.common.exception.ErrorException - -/** - * created by cooperyang on 2019/10/12 - * Description: - */ -case class RPCFailedException(errMsg:String) extends ErrorException(80056, errMsg) - diff --git a/plugins/linkis/linkis-appjoint-entrance/src/main/scala/com/webank/wedatasphere/dss/linkis/appjoint/entrance/execute/AppJointEngineExecutorManagerImpl.scala b/plugins/linkis/linkis-appjoint-entrance/src/main/scala/com/webank/wedatasphere/dss/linkis/appjoint/entrance/execute/AppJointEngineExecutorManagerImpl.scala deleted file mode 100644 index 709c19990c..0000000000 --- a/plugins/linkis/linkis-appjoint-entrance/src/main/scala/com/webank/wedatasphere/dss/linkis/appjoint/entrance/execute/AppJointEngineExecutorManagerImpl.scala +++ /dev/null @@ -1,70 +0,0 @@ -/* - * Copyright 2019 WeBank - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package com.webank.wedatasphere.dss.linkis.appjoint.entrance.execute - -import com.webank.wedatasphere.dss.linkis.appjoint.entrance.job.AppJointEntranceJob -import com.webank.wedatasphere.linkis.common.utils.Logging -import com.webank.wedatasphere.linkis.entrance.execute._ -import com.webank.wedatasphere.linkis.entrance.execute.impl.EntranceExecutorManagerImpl -import com.webank.wedatasphere.linkis.scheduler.executer.Executor -import com.webank.wedatasphere.linkis.scheduler.listener.ExecutorListener -import com.webank.wedatasphere.linkis.scheduler.queue.{GroupFactory, Job, SchedulerEvent} - -import scala.concurrent.duration.Duration - -class AppJointEngineExecutorManagerImpl(groupFactory: GroupFactory, - engineBuilder: EngineBuilder, - engineRequester: EngineRequester, - engineSelector: EngineSelector, - engineManager: EngineManager, - entranceExecutorRulers: Array[EntranceExecutorRuler]) - extends EntranceExecutorManagerImpl(groupFactory, engineBuilder, engineRequester, - engineSelector, engineManager, entranceExecutorRulers) with Logging { - override def setExecutorListener(executorListener: ExecutorListener): Unit = super.setExecutorListener(executorListener) - - override def initialEntranceEngine(engine: EntranceEngine): Unit = super.initialEntranceEngine(engine) - - protected override def createExecutor(schedulerEvent: SchedulerEvent): EntranceEngine = schedulerEvent match { - case job: AppJointEntranceJob => - val externalEngineParams = job.getParams - val engine = new AppJointEntranceEngine(externalEngineParams) - engine - case _ => null - } - - override protected def findExecutors(job: Job): Array[EntranceEngine] = super.findExecutors(job) - - override def askExecutor(schedulerEvent: SchedulerEvent): Option[Executor] = schedulerEvent match { - case job: AppJointEntranceJob => - Some(createExecutor(schedulerEvent)) - } - - override def askExecutor(schedulerEvent: SchedulerEvent, wait: Duration): Option[Executor] = schedulerEvent match { - case job: AppJointEntranceJob => - Some(createExecutor(schedulerEvent)) - } - - - override def getById(id: Long): Option[Executor] = super.getById(id) - - override def getByGroup(groupName: String): Array[Executor] = super.getByGroup(groupName) - - protected override def delete(executor: Executor): Unit = super.delete(executor) - - override def shutdown(): Unit = super.shutdown() -} diff --git a/plugins/linkis/linkis-appjoint-entrance/src/main/scala/com/webank/wedatasphere/dss/linkis/appjoint/entrance/execute/AppJointEntranceEngine.scala b/plugins/linkis/linkis-appjoint-entrance/src/main/scala/com/webank/wedatasphere/dss/linkis/appjoint/entrance/execute/AppJointEntranceEngine.scala deleted file mode 100644 index e16c9b93e5..0000000000 --- a/plugins/linkis/linkis-appjoint-entrance/src/main/scala/com/webank/wedatasphere/dss/linkis/appjoint/entrance/execute/AppJointEntranceEngine.scala +++ /dev/null @@ -1,228 +0,0 @@ -/* - * Copyright 2019 WeBank - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package com.webank.wedatasphere.dss.linkis.appjoint.entrance.execute - -import java.util -import java.util.UUID - -import com.webank.wedatasphere.dss.appjoint.execution.NodeExecution -import com.webank.wedatasphere.dss.appjoint.execution.async.NodeExecutionResponseListener -import com.webank.wedatasphere.dss.appjoint.execution.common._ -import com.webank.wedatasphere.dss.appjoint.execution.core._ -import com.webank.wedatasphere.dss.linkis.appjoint.entrance.appjoint.AppJointManager -import com.webank.wedatasphere.dss.linkis.appjoint.entrance.conf.AppJointConst -import com.webank.wedatasphere.dss.linkis.appjoint.entrance.exception.OperateNotSupportedException -import com.webank.wedatasphere.dss.linkis.appjoint.entrance.job.AppJointExecuteRequest -import com.webank.wedatasphere.linkis.common.exception.ErrorException -import com.webank.wedatasphere.linkis.common.utils.{Logging, Utils} -import com.webank.wedatasphere.linkis.entrance.execute.{EngineExecuteAsynReturn, EntranceEngine, EntranceJob} -import com.webank.wedatasphere.linkis.entrance.interceptor.impl.CustomVariableUtils -import com.webank.wedatasphere.linkis.protocol.engine.{JobProgressInfo, RequestTask} -import com.webank.wedatasphere.linkis.protocol.query.RequestPersistTask -import com.webank.wedatasphere.linkis.scheduler.executer._ -import com.webank.wedatasphere.linkis.scheduler.queue.Job -import org.apache.commons.lang.StringUtils - -import scala.collection.mutable.ArrayBuffer -import scala.collection.JavaConversions._ -/** - * created by chaogefeng on 2019/9/26 20:23 - * Description: - */ -class AppJointEntranceEngine(properties: util.Map[String, Any]) - extends EntranceEngine(id = 0) with SingleTaskOperateSupport with SingleTaskInfoSupport { - - private val defaultProgress = 0.0f - private val defaultLog = "" - - - private var nodeExecution:NodeExecution = _ - - private var appJointAction:NodeExecutionAction = _ - - - private var response:AppJointEntranceAsyncExecuteResponse = _ - - private var job:Job = _ - - - def setJob(job: Job):Unit = this.job = job - def getJob:Job = this.job - - def setResponse(response:AppJointEntranceAsyncExecuteResponse):Unit = this.response = response - def getResponse:AppJointEntranceAsyncExecuteResponse = this.response - - - - /** - * 设置一个tag是用一个唯一记号标识一个AppJointEntranceEngine，由于每一个job都会对应一个AppJointEntranceEngine - * 所以在callback的时候带上这个 - */ - private val tag = UUID.randomUUID().toString - - override protected def callExecute(request: RequestTask): EngineExecuteAsynReturn = null - - def getTag:String = this.tag - - override def kill(): Boolean = { - nodeExecution match { - case killable:Killable => if (appJointAction != null) killable.kill(appJointAction) else false - case _ => logger.warn(s"${nodeExecution.getClass.getSimpleName} cannot be killed") - throw OperateNotSupportedException(s"${nodeExecution.getClass.getSimpleName} cannot be killed") - } - } - - override def pause(): Boolean = { - throw OperateNotSupportedException(s"${nodeExecution.getClass.getSimpleName} cannot be paused") - } - - override def resume(): Boolean = { - throw OperateNotSupportedException(s"${nodeExecution.getClass.getSimpleName} cannot be resumed") - } - - - override def progress(): Float = { - nodeExecution match { - case procedure:Procedure => if (appJointAction != null) procedure.progress(appJointAction) else defaultProgress - case _ => logger.debug(s"${nodeExecution.getClass.getSimpleName} cannot get progress") - defaultProgress - } - } - - override def getProgressInfo: Array[JobProgressInfo] = Array.empty - - override def log(): String = { - nodeExecution match { - case procedure:Procedure => if (appJointAction != null) procedure.log(appJointAction) else defaultLog - case _ => logger.debug(s"${nodeExecution.getClass.getSimpleName} cannot get log") - defaultLog - } - } - - def generateContext(node: AppJointNode, runTimeMap:util.Map[String, Object]):NodeContext = { - node match { - case commonNode:CommonAppJointNode => val nodeContext = new AppJointNodeContextImpl() - Utils.tryCatch{ - nodeContext.setAppJointNode(node) - nodeContext.setJob(getJob) - nodeContext.setRuntimeMap(runTimeMap) - nodeContext.setStorePath(job.asInstanceOf[EntranceJob].getTask.asInstanceOf[RequestPersistTask].getResultLocation) - nodeContext.setUser(runTimeMap.get("user").asInstanceOf[String]) - nodeContext.setStorePath(runTimeMap.get("storePath").asInstanceOf[String]) - if (runTimeMap.containsKey(AppJointConst.WORKFLOW_SHARED_NODES_JOBIDS)){ - nodeContext.setIsReadNode(true) - runTimeMap.get(AppJointConst.WORKFLOW_SHARED_NODES_JOBIDS) match { - case ids:util.Map[String, String] => val jobIds = new ArrayBuffer[Long]() - ids foreach { - case (k ,v) => Utils.tryAndWarnMsg(jobIds += v.toLong)(s"$v is not a long type") - } - nodeContext.setJobIdsOfShareNode(jobIds.toArray) - case _ => - } - }else{ - nodeContext.setIsReadNode(false) - } - }{ - case e:Exception => logger.warn("创建nodeContext时出现异常,警告信息", e) - } - nodeContext - case _ => logger.warn(s"$node is not a CommonAppJointNode") - null - } - } - - - override def close(): Unit = {} - - override def execute(executeRequest: ExecuteRequest): ExecuteResponse = executeRequest match { - case appJointExecuteRequest:AppJointExecuteRequest => - val node = appJointExecuteRequest.node - val runTimeMap = appJointExecuteRequest.runTimeParams - val nodeContext = generateContext(node, runTimeMap) - val nodeType = nodeContext.getAppJointNode.getNodeType - val realAppJointType = if (nodeType.contains(".")) nodeType.substring(0, nodeType.indexOf(".")) else nodeType - val appJoint = AppJointManager.getAppJoint(realAppJointType) - if((realAppJointType.toLowerCase()).contains("datacheck")){ - replaceCustomVariables(nodeContext.getRuntimeMap) - } - val user = if (null != runTimeMap.get("user")) runTimeMap.get("user").toString else null - val session = if (StringUtils.isNotEmpty(user)){ - if (appJoint.getSecurityService != null) appJoint.getSecurityService.login(user) else null - } else null - this.nodeExecution = appJoint.getNodeExecution - this.nodeExecution match{ - case longTermNodeExecution:LongTermNodeExecution => longTermNodeExecution.setScheduler(AppJointManager.getScheduler) - case _ => - } - if (!nodeExecution.canExecute(node, nodeContext, session)) throw new ErrorException(70058, s"appJoint ${appJoint.getAppJointName} can not execute $appJointExecuteRequest.") - this.nodeExecution.execute(node, nodeContext, session) match { - case completedResponse:CompletedNodeExecutionResponse => - if (completedResponse.isSucceed) SuccessExecuteResponse() - else { - logger.error(s"Failed Reason is ${completedResponse.getErrorMsg}", completedResponse.getException) - ErrorExecuteResponse(completedResponse.getErrorMsg, completedResponse.getException) - } - case asyncAppJointResponse:AsyncNodeExecutionResponse => - appJointAction = asyncAppJointResponse.getAction - response = new AppJointEntranceAsyncExecuteResponse - asyncAppJointResponse.addListener(new NodeExecutionResponseListener{ - override def onNodeExecutionCompleted(resp: CompletedNodeExecutionResponse): Unit = response.notifyStatus(resp) - }) - response - case _ => - logger.warn(s"Can not solve this type of response $response") - ErrorExecuteResponse(s"Can not solve this type of response $response", - new ErrorException(80056, s"Can not solve this type of response $response")) - } - case _ => - ErrorExecuteResponse(s"cannot do this executeRequest $executeRequest", - new ErrorException(80056, s"cannot do this executeRequest $executeRequest")) - } - private def replaceCustomVariables(runTimeMap:java.util.Map[String, Object]):Unit = { - val key = "check.object" - val value:String = if (null != runTimeMap.get(key)) runTimeMap.get(key).toString else "" - val task = new RequestPersistTask - task.setExecutionCode(value) - task.setParams(new util.HashMap[String, Object]()) - val (result, code) = CustomVariableUtils.replaceCustomVar(task, "sql") - logger.info(s"after code replace code is $code") - if (result) runTimeMap(key) = code - } -} - - -case class AppJointEntranceExecuteException(errMsg:String) extends ErrorException(70046, errMsg) - -class AppJointEntranceAsyncExecuteResponse extends AsynReturnExecuteResponse with Logging{ - - var rs:ExecuteResponse => Unit = _ - - def notifyStatus(response: CompletedNodeExecutionResponse):Unit = { - val executeResponse = if(response.isSucceed) SuccessExecuteResponse() - else ErrorExecuteResponse(response.getErrorMsg, response.getException) - if (rs == null) this synchronized(while(rs == null) this.wait(1000)) - rs(executeResponse) - } - - override def notify(rs: ExecuteResponse => Unit): Unit = { - this.rs = rs - this synchronized notifyAll() - } -} - - diff --git a/plugins/linkis/linkis-appjoint-entrance/src/main/scala/com/webank/wedatasphere/dss/linkis/appjoint/entrance/execute/AppJointNodeContextImpl.scala b/plugins/linkis/linkis-appjoint-entrance/src/main/scala/com/webank/wedatasphere/dss/linkis/appjoint/entrance/execute/AppJointNodeContextImpl.scala deleted file mode 100644 index 81657bbbec..0000000000 --- a/plugins/linkis/linkis-appjoint-entrance/src/main/scala/com/webank/wedatasphere/dss/linkis/appjoint/entrance/execute/AppJointNodeContextImpl.scala +++ /dev/null @@ -1,239 +0,0 @@ -/* - * Copyright 2019 WeBank - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package com.webank.wedatasphere.dss.linkis.appjoint.entrance.execute - -import java.util -import java.util.concurrent.atomic.AtomicInteger - -import com.google.gson.Gson -import com.webank.wedatasphere.dss.appjoint.execution.core.NodeContextImpl -import com.webank.wedatasphere.dss.linkis.appjoint.entrance.conf.AppJointEntranceConfiguration -import com.webank.wedatasphere.dss.linkis.appjoint.entrance.exception.{AppJointErrorException, RPCFailedException} -import com.webank.wedatasphere.linkis.DataWorkCloudApplication -import com.webank.wedatasphere.linkis.common.exception.ErrorException -import com.webank.wedatasphere.linkis.common.io.resultset.{ResultSet, ResultSetReader, ResultSetWriter} -import com.webank.wedatasphere.linkis.common.io.{FsPath, MetaData, Record} -import com.webank.wedatasphere.linkis.common.utils.{ByteTimeUtils, Logging, Utils} -import com.webank.wedatasphere.linkis.entrance.EntranceContext -import com.webank.wedatasphere.linkis.protocol.query.{RequestPersistTask, RequestQueryTask, ResponsePersist} -import com.webank.wedatasphere.linkis.rpc.Sender -import com.webank.wedatasphere.linkis.scheduler.queue.Job -import com.webank.wedatasphere.linkis.storage.resultset.table.{TableMetaData, TableRecord} -import com.webank.wedatasphere.linkis.storage.resultset.{ResultSetFactory, ResultSetReader, ResultSetWriter, StorageResultSetWriter} -import com.webank.wedatasphere.linkis.storage.{FSFactory, LineMetaData, LineRecord} -import org.apache.commons.lang.StringUtils - -import scala.collection.JavaConversions._ -import scala.collection.mutable.ArrayBuffer -/** - * created by cooperyang on 2019/10/12 - * Description: - */ -class AppJointNodeContextImpl extends NodeContextImpl with Logging{ - - - //querySender 是用来进行访query模块 - val publicServiceSender:Sender = Sender.getSender(AppJointEntranceConfiguration.PUBLIC_SERVICE_SPRING_APPLICATION.getValue) - - val entranceContext:EntranceContext = DataWorkCloudApplication.getApplicationContext.getBean(classOf[EntranceContext]) - - var job:Job = _ - - var storePath:String = _ - - val SUCCESS_FLAG = 0 - - val TASK_MAP_KEY = "task" - - def setJob(job:Job):Unit = this.job = job - - private val resultSetFactory = ResultSetFactory.getInstance - - private val aliasNum = new AtomicInteger(0) - - private val defaultResultSetAlias = "" - - /** - * 通过jobId拿回一个task - * @param jobId - * @return - */ - - override def getJobById(jobId: Long): RequestPersistTask = { - - val requestQueryTask = new RequestQueryTask() - requestQueryTask.setTaskID(jobId) - requestQueryTask.setSource(null) - val task = Utils.tryCatch{ - val taskResponse = publicServiceSender.ask(requestQueryTask) - taskResponse match { - case responsePersist:ResponsePersist => val status = responsePersist.getStatus - if (status != SUCCESS_FLAG){ - logger.error(s"query from jobHistory status failed, status is $status") - throw AppJointErrorException("query from jobHistory status failed") - }else{ - val data = responsePersist.getData - data.get(TASK_MAP_KEY) match { - case tasks:util.List[util.Map[String, Object]] => tasks.get(0) match { - case map:util.Map[String, Object] => val gson = new Gson() - val json = gson.toJson(map) - val requestPersistTask = gson.fromJson(json, classOf[RequestPersistTask]) - requestPersistTask - case _ => throw AppJointErrorException(s"query from jobhistory not a correct RequestPersistTask type taskId is $jobId") - } - case _ => throw AppJointErrorException(s"query from jobhistory not a correct List type taskId is $jobId") - } - } - case _ => logger.error("get query response incorrectly") - throw AppJointErrorException("get query response incorrectly") - } - }{ - case errorException:ErrorException => throw errorException - case e:Exception => val e1 = AppJointErrorException(s"query taskId $jobId error") - e1.initCause(e) - throw e - } - task - } - - override def appendLog(log: String): Unit = { - entranceContext.getOrCreateLogManager().onLogUpdate(job, log) - } - - override def updateProgress(progress: Float): Unit = { - entranceContext.getOrCreatePersistenceManager().onProgressUpdate(job, progress, Array.empty) - } - - override def setStorePath(storePath: String): Unit = this.storePath = storePath - - override def getStorePath: String = this.storePath - - private def getMaxCache:Long = ByteTimeUtils.byteStringAsBytes("0K") - - override def createTableResultSetWriter(): ResultSetWriter[TableMetaData, TableRecord] = { - createTableResultSetWriter(defaultResultSetAlias) - } - - - - override def createTableResultSetWriter(resultSetAlias: String): ResultSetWriter[TableMetaData, TableRecord] = { - val resultSetType = ResultSetFactory.resultSetType.getOrElse(ResultSetFactory.TABLE_TYPE, "TABLE") - val resultSet = resultSetFactory.getResultSetByType(ResultSetFactory.TABLE_TYPE) - val filePath = this.storePath - val fileName = if(StringUtils.isEmpty(resultSetAlias)) "_" + aliasNum.getAndIncrement() else resultSetAlias + "_" + aliasNum.getAndIncrement() - val resultSetPath = resultSet.getResultSetPath(new FsPath(filePath), fileName) - val resultSetWriter = ResultSetWriter.getResultSetWriter(resultSet, getMaxCache, resultSetPath) - resultSetWriter.asInstanceOf[StorageResultSetWriter[TableMetaData, TableRecord]].setProxyUser(this.getUser) - resultSetWriter match { - case r:ResultSetWriter[TableMetaData, TableRecord] => r - case _ => null - } - } - - override def createTextResultSetWriter(): ResultSetWriter[LineMetaData, LineRecord] = { - createTextResultSetWriter(defaultResultSetAlias) - } - - override def createTextResultSetWriter(resultSetAlias: String): ResultSetWriter[LineMetaData, LineRecord] = { - val resultSetType = ResultSetFactory.resultSetType.getOrElse(ResultSetFactory.TEXT_TYPE, "TEXT") - val resultSet = resultSetFactory.getResultSetByType(ResultSetFactory.TEXT_TYPE) - val filePath = this.storePath - val fileName = if(StringUtils.isEmpty(resultSetAlias)) "_" + aliasNum.getAndIncrement() else resultSetAlias + "_" + aliasNum.getAndIncrement() - val resultSetPath = resultSet.getResultSetPath(new FsPath(filePath), fileName) - val resultSetWriter = ResultSetWriter.getResultSetWriter(resultSet, getMaxCache, resultSetPath) - resultSetWriter.asInstanceOf[StorageResultSetWriter[LineMetaData, LineRecord]].setProxyUser(this.getUser) - resultSetWriter match { - case r:ResultSetWriter[LineMetaData, LineRecord] => r - case _ => null - } - } - - override def createHTMLResultSetWriter(): ResultSetWriter[LineMetaData, LineRecord] = { - createHTMLResultSetWriter(defaultResultSetAlias) - } - - override def createHTMLResultSetWriter(resultSetAlias: String): ResultSetWriter[LineMetaData, LineRecord] = { - val resultSetType = ResultSetFactory.resultSetType.getOrElse(ResultSetFactory.HTML_TYPE, "HTML") - val resultSet = resultSetFactory.getResultSetByType(ResultSetFactory.HTML_TYPE) - val filePath = this.storePath - val fileName = if(StringUtils.isEmpty(resultSetAlias)) "_" + aliasNum.getAndIncrement() else resultSetAlias + "_" + aliasNum.getAndIncrement() - val resultSetPath = resultSet.getResultSetPath(new FsPath(filePath), fileName) - val resultSetWriter = ResultSetWriter.getResultSetWriter(resultSet, getMaxCache, resultSetPath) - resultSetWriter.asInstanceOf[StorageResultSetWriter[LineMetaData, LineRecord]].setProxyUser(this.getUser) - resultSetWriter match { - case r:ResultSetWriter[LineMetaData, LineRecord] => r - case _ => null - } - } - - override def createPictureResultSetWriter(): ResultSetWriter[LineMetaData, LineRecord] = { - createPictureResultSetWriter(defaultResultSetAlias) - } - - override def createPictureResultSetWriter(resultSetAlias: String): ResultSetWriter[LineMetaData, LineRecord] = { - val resultSetType = ResultSetFactory.resultSetType.getOrElse(ResultSetFactory.PICTURE_TYPE, "PICTURE") - val resultSet = resultSetFactory.getResultSetByType(ResultSetFactory.PICTURE_TYPE) - val filePath = this.storePath - val fileName = if(StringUtils.isEmpty(resultSetAlias)) "_" + aliasNum.getAndIncrement() else resultSetAlias + "_" + aliasNum.getAndIncrement() - val resultSetPath = resultSet.getResultSetPath(new FsPath(filePath), fileName) - val resultSetWriter = ResultSetWriter.getResultSetWriter(resultSet, getMaxCache, resultSetPath) - resultSetWriter.asInstanceOf[StorageResultSetWriter[LineMetaData, LineRecord]].setProxyUser(this.getUser) - resultSetWriter match { - case r:ResultSetWriter[LineMetaData, LineRecord] => r - case _ => null - } - } - - override def createResultSetWriter(resultSet: ResultSet[_ <: MetaData, _ <: Record], resultSetAlias: String): ResultSetWriter[_ <: MetaData, _ <: Record] = ??? - - //todo - override def getResultSetReader(fsPath: FsPath): ResultSetReader[_ <: MetaData, _ <: Record] = { - ResultSetReader.getResultSetReader(fsPath.getPath) - } - - /** - * - * @return - */ - override def getGatewayUrl: String = { - val instances = Utils.tryCatch{ - Sender.getInstances(AppJointEntranceConfiguration.GATEWAY_SPRING_APPLICATION.getValue) - }{ - case e:Exception => val e1 = RPCFailedException("获取gateway的url失败") - e1.initCause(e) - throw e1 - } - if (instances.length == 0) throw RPCFailedException("获取gateway的url失败") - instances(0).getInstance - } - - override def getResultSetPathsByJobId(jobId: Long): Array[FsPath] = { - val paths = new ArrayBuffer[FsPath]() - val task = getJobById(jobId) - val resultSetLocation = task.getResultLocation - val user = task.getUmUser - val fileSystem = FSFactory.getFsByProxyUser(new FsPath(resultSetLocation), user) - fileSystem.init(new util.HashMap[String, String]()) - Utils.tryFinally{ - fileSystem.list(new FsPath(resultSetLocation)) foreach (paths += _) - }{ - Utils.tryQuietly(fileSystem.close()) - } - paths.toArray - } -} diff --git a/plugins/linkis/linkis-appjoint-entrance/src/main/scala/com/webank/wedatasphere/dss/linkis/appjoint/entrance/job/AppJointEntranceJob.scala b/plugins/linkis/linkis-appjoint-entrance/src/main/scala/com/webank/wedatasphere/dss/linkis/appjoint/entrance/job/AppJointEntranceJob.scala deleted file mode 100644 index f993cda8a4..0000000000 --- a/plugins/linkis/linkis-appjoint-entrance/src/main/scala/com/webank/wedatasphere/dss/linkis/appjoint/entrance/job/AppJointEntranceJob.scala +++ /dev/null @@ -1,100 +0,0 @@ -/* - * Copyright 2019 WeBank - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package com.webank.wedatasphere.dss.linkis.appjoint.entrance.job - -import java.util - -import com.webank.wedatasphere.dss.appjoint.execution.core.{AppJointNode, CommonAppJointNode} -import com.webank.wedatasphere.dss.linkis.appjoint.entrance.execute.AppJointEntranceEngine -import com.webank.wedatasphere.linkis.common.exception.ErrorException -import com.webank.wedatasphere.linkis.common.log.LogUtils -import com.webank.wedatasphere.linkis.common.utils.Utils -import com.webank.wedatasphere.linkis.entrance.execute.StorePathExecuteRequest -import com.webank.wedatasphere.linkis.entrance.job.EntranceExecutionJob -import com.webank.wedatasphere.linkis.protocol.query.RequestPersistTask -import com.webank.wedatasphere.linkis.scheduler.executer._ -import com.webank.wedatasphere.linkis.scheduler.queue.SchedulerEventState._ -import org.apache.commons.lang.StringUtils -/** - * created by cooperyang on 2019/9/26 - * Description: - */ -class AppJointEntranceJob extends EntranceExecutionJob{ - var node:AppJointNode = _ - var runTimeMap:util.Map[String, Object] = _ - - def setNode(node:AppJointNode):Unit = this.node = node - def getNode:AppJointNode = this.node - - def setRunTimeMap(runTimeMap:util.Map[String, Object]):Unit = this.runTimeMap = runTimeMap - def getRunTimeMap:util.Map[String, Object] = this.runTimeMap - - - override def jobToExecuteRequest(): ExecuteRequest = new ExecuteRequest with StorePathExecuteRequest with AppJointExecuteRequest { - - - override val code: String = "" - - val jobContent: util.Map[String, Object] = AppJointEntranceJob.this.getNode match { - case commonAppJointNode:CommonAppJointNode => commonAppJointNode.getJobContent - case _ => null - } - override val storePath: String = AppJointEntranceJob.this.getTask match{ - case requestPersistTask:RequestPersistTask => requestPersistTask.getResultLocation - case _ => null - } - override val node: AppJointNode = AppJointEntranceJob.this.node - override val runTimeParams: util.Map[String, Object] = AppJointEntranceJob.this.runTimeMap - } - - - override def run(): Unit = { - if(!isScheduled) return - info(s"$getId starts to run") - getLogListener.foreach(_.onLogUpdate(this, LogUtils.generateInfo(s"$getId starts to execute."))) - startTime = System.currentTimeMillis - Utils.tryAndErrorMsg(transition(Running))(s"transition $getId from Scheduler to Running failed.") - getExecutor match { - case appjointEntranceEngine:AppJointEntranceEngine => appjointEntranceEngine.setJob(this) - } - val rs = Utils.tryCatch(getExecutor.execute(jobToExecuteRequest())){ - case e:ErrorException => logger.error(s"execute job $getId failed", e) - ErrorExecuteResponse(s"execute job $getId failed", e) - case t:Throwable => logger.error(s"execute job $getId failed", t) - ErrorExecuteResponse(s"execute job $getId failed", t) - } - rs match { - case r: CompletedExecuteResponse => - setResultSize(0) - transitionCompleted(r) - case r: IncompleteExecuteResponse => - setResultSize(0) - transitionCompleted(ErrorExecuteResponse(if(StringUtils.isNotEmpty(r.message)) r.message else "incomplete code.", null)) - case r: AsynReturnExecuteResponse => - setResultSize(0) - r.notify(r1 => { - val realRS = r1 match { - case r: IncompleteExecuteResponse => - ErrorExecuteResponse(if(StringUtils.isNotEmpty(r.message)) r.message else "incomplete code.", null) - case r: CompletedExecuteResponse => r - } - transitionCompleted(realRS) - }) - } - } -} diff --git a/plugins/linkis/linkis-appjoint-entrance/src/main/scala/com/webank/wedatasphere/dss/linkis/appjoint/entrance/job/AppJointExecuteRequest.scala b/plugins/linkis/linkis-appjoint-entrance/src/main/scala/com/webank/wedatasphere/dss/linkis/appjoint/entrance/job/AppJointExecuteRequest.scala deleted file mode 100644 index 5c7d94812d..0000000000 --- a/plugins/linkis/linkis-appjoint-entrance/src/main/scala/com/webank/wedatasphere/dss/linkis/appjoint/entrance/job/AppJointExecuteRequest.scala +++ /dev/null @@ -1,33 +0,0 @@ -/* - * Copyright 2019 WeBank - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package com.webank.wedatasphere.dss.linkis.appjoint.entrance.job - -import com.webank.wedatasphere.dss.appjoint.execution.core.AppJointNode - -/** - * created by cooperyang on 2019/9/26 - * Description: - */ -trait AppJointExecuteRequest { - val node:AppJointNode - val runTimeParams:java.util.Map[String, Object] - - override def toString: String = { - node.getProjectName + " " + node.getFlowName + " " + node.getName - } -} diff --git a/plugins/linkis/linkis-appjoint-entrance/src/main/scala/com/webank/wedatasphere/dss/linkis/appjoint/entrance/parser/AppJointEntranceParser.scala b/plugins/linkis/linkis-appjoint-entrance/src/main/scala/com/webank/wedatasphere/dss/linkis/appjoint/entrance/parser/AppJointEntranceParser.scala deleted file mode 100644 index d5cd3e13c4..0000000000 --- a/plugins/linkis/linkis-appjoint-entrance/src/main/scala/com/webank/wedatasphere/dss/linkis/appjoint/entrance/parser/AppJointEntranceParser.scala +++ /dev/null @@ -1,95 +0,0 @@ -/* - * Copyright 2019 WeBank - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package com.webank.wedatasphere.dss.linkis.appjoint.entrance.parser - -import java.util - -import com.webank.wedatasphere.dss.appjoint.execution.core.CommonAppJointNode -import com.webank.wedatasphere.dss.linkis.appjoint.entrance.conf.AppJointConst -import com.webank.wedatasphere.dss.linkis.appjoint.entrance.job.AppJointEntranceJob -import com.webank.wedatasphere.linkis.common.utils.{Logging, Utils} -import com.webank.wedatasphere.linkis.entrance.parser.CommonEntranceParser -import com.webank.wedatasphere.linkis.protocol.query.RequestPersistTask -import com.webank.wedatasphere.linkis.protocol.task.Task -import com.webank.wedatasphere.linkis.protocol.utils.TaskUtils -import com.webank.wedatasphere.linkis.scheduler.queue.Job - -/** - * created by cooperyang on 2019/9/26 - * Description: - */ - -class AppJointEntranceParser extends CommonEntranceParser with Logging{ - - private val PROJECT_NAME_STR = "projectName" - - private val FLOW_NAME_STR = "flowName" - - private val NODE_NAME_STR = "nodeName" - - private val TYPE_NAME_STR = "type" - - - override def parseToJob(task: Task): Job = { - val job = new AppJointEntranceJob - task match{ - case requestPersistTask:RequestPersistTask => - job.setTask(task) - job.setUser(requestPersistTask.getUmUser) - job.setCreator(requestPersistTask.getRequestApplicationName) - job.setParams(requestPersistTask.getParams.asInstanceOf[util.Map[String, Any]]) - val projectName = Utils.tryCatch{ - requestPersistTask.getSource.get(PROJECT_NAME_STR) - }{ - case e:Exception => "" - } - val flowName = Utils.tryCatch{ - requestPersistTask.getSource.get(FLOW_NAME_STR) - }{ - case e:Exception => "" - } - val nodeName = Utils.tryCatch{ - requestPersistTask.getSource.get(NODE_NAME_STR) - }{ - case e:Exception => "" - } - val _type = Utils.tryCatch{ - requestPersistTask.getRunType - }{ - case e:Exception => "" - } - val executionCode = requestPersistTask.getExecutionCode - val jobContent = AppJointConst.gson.fromJson(executionCode, classOf[util.Map[String,Object ]]) - val node = new CommonAppJointNode() - node.setProjectName(projectName) - node.setFlowName(flowName) - node.setNodeType(_type) - node.setJobContent(jobContent) - job.setNode(node) - val um_user = requestPersistTask.getUmUser - val runTimeMap = TaskUtils.getRuntimeMap(requestPersistTask.getParams.asInstanceOf[util.Map[String, Any]]) - runTimeMap.put("user", um_user) - runTimeMap.put("storePath", requestPersistTask.getResultLocation) - job.setRunTimeMap(runTimeMap.asInstanceOf[java.util.Map[String, Object]]) - job.setEntranceListenerBus(getEntranceContext.getOrCreateEventListenerBus) - job.setListenerEventBus(null) - job.setProgress(0f) - } - job - } -} diff --git a/pom.xml b/pom.xml index 53346c0ffc..6c1abde2ed 100644 --- a/pom.xml +++ b/pom.xml @@ -1,9 +1,8 @@ - - - dss-qualitis-appjoint - - zip - - true - qualitis - - - - - - lib - true - true - false - true - true - - - - - - ${basedir}/src/main/resources - - appjoint.properties - - 0777 - / - unix - - - - ${basedir}/src/main/resources - - log4j.properties - log4j2.xml - - 0777 - conf - unix - - - - - - diff --git a/qualitis-appjoint/appjoint/src/main/java/com.webank/wedatasphere/Md5Utils.java b/qualitis-appjoint/appjoint/src/main/java/com.webank/wedatasphere/Md5Utils.java deleted file mode 100644 index 4d5abe6ebc..0000000000 --- a/qualitis-appjoint/appjoint/src/main/java/com.webank/wedatasphere/Md5Utils.java +++ /dev/null @@ -1,27 +0,0 @@ -package com.webank.wedatasphere; - -import org.apache.commons.lang.StringUtils; - -import java.math.BigInteger; -import java.security.MessageDigest; - -/** - * @author howeye - */ -public class Md5Utils { - - public static String getMd5L32(String str) throws Exception { - try { - // Generate a instance of md5 - MessageDigest md = MessageDigest.getInstance("MD5"); - // calculate - md.update(str.getBytes()); - String result = new BigInteger(1, md.digest()).toString(16); - - return StringUtils.leftPad(result, 32, '0'); - } catch (Exception e) { - throw new Exception("Error generating md5 of string: " + str, e); - } - } - -} diff --git a/qualitis-appjoint/appjoint/src/main/java/com.webank/wedatasphere/appjoint/QualitisAppJoint.java b/qualitis-appjoint/appjoint/src/main/java/com.webank/wedatasphere/appjoint/QualitisAppJoint.java deleted file mode 100644 index 42f1b30b53..0000000000 --- a/qualitis-appjoint/appjoint/src/main/java/com.webank/wedatasphere/appjoint/QualitisAppJoint.java +++ /dev/null @@ -1,56 +0,0 @@ -package com.webank.wedatasphere.appjoint; - -import com.webank.wedatasphere.project.QualitisProjectServiceImpl; -import com.webank.wedatasphere.dss.appjoint.AppJoint; -import com.webank.wedatasphere.dss.appjoint.exception.AppJointErrorException; -import com.webank.wedatasphere.dss.appjoint.execution.NodeExecution; -import com.webank.wedatasphere.dss.appjoint.service.ProjectService; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.util.Map; - -/** - * @author howeye - */ -public class QualitisAppJoint implements AppJoint { - - QualitisNodeExecution nodeExecution = null; - QualitisProjectServiceImpl projectService = null; - - private static final Logger LOGGER = LoggerFactory.getLogger(QualitisAppJoint.class); - - @Override - public String getAppJointName() { - return "Qualitis"; - } - - @Override - public void init(String s, Map map) throws AppJointErrorException { - nodeExecution = new QualitisNodeExecution(); - nodeExecution.setBaseUrl(s); - nodeExecution.init(map); - - projectService = new QualitisProjectServiceImpl(); - projectService.setBaseUrl(s); - projectService.init(map); - } - - @Override - public NodeExecution getNodeExecution() { - if (nodeExecution != null) { - return nodeExecution; - } - LOGGER.error("NodeExecution is not inited, return null"); - return null; - } - - @Override - public ProjectService getProjectService() { - if (projectService != null) { - return projectService; - } - LOGGER.error("ProjectService is not inited, return null"); - return null; - } -} diff --git a/qualitis-appjoint/appjoint/src/main/java/com.webank/wedatasphere/appjoint/QualitisNodeExecution.java b/qualitis-appjoint/appjoint/src/main/java/com.webank/wedatasphere/appjoint/QualitisNodeExecution.java deleted file mode 100644 index f2b0bbc389..0000000000 --- a/qualitis-appjoint/appjoint/src/main/java/com.webank/wedatasphere/appjoint/QualitisNodeExecution.java +++ /dev/null @@ -1,326 +0,0 @@ -package com.webank.wedatasphere.appjoint; - -import com.google.gson.Gson; -import com.webank.wedatasphere.Md5Utils; -import com.webank.wedatasphere.dss.appjoint.exception.AppJointErrorException; -import com.webank.wedatasphere.dss.appjoint.execution.common.CompletedNodeExecutionResponse; -import com.webank.wedatasphere.dss.appjoint.execution.common.NodeExecutionAction; -import com.webank.wedatasphere.dss.appjoint.execution.common.NodeExecutionState; -import com.webank.wedatasphere.dss.appjoint.execution.core.AppJointNode; -import com.webank.wedatasphere.dss.appjoint.execution.core.LongTermNodeExecution; -import com.webank.wedatasphere.dss.appjoint.execution.core.NodeContext; -import com.webank.wedatasphere.dss.appjoint.service.session.Session; -import com.webank.wedatasphere.dss.common.entity.node.Node; -import org.apache.commons.lang.RandomStringUtils; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.springframework.http.HttpEntity; -import org.springframework.http.HttpHeaders; -import org.springframework.http.HttpMethod; -import org.springframework.http.MediaType; -import org.springframework.web.client.RestTemplate; - -import javax.ws.rs.core.UriBuilder; -import java.net.URI; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -/** - * @author howeye - */ -public class QualitisNodeExecution extends LongTermNodeExecution { - - private static final String SUBMIT_TASK_PATH = "/qualitis/outer/api/v1/execution"; - private static final String GET_TASK_STATUS_PATH = "/qualitis/outer/api/v1/application/{applicationId}/status/"; - private static final String GET_TASK_RESULT_PATH = "/qualitis/outer/api/v1/application/{applicationId}/result/"; - - private static final Logger LOGGER = LoggerFactory.getLogger(QualitisAppJoint.class); - - private String host = null; - private Integer port = null; - private String appId = null; - private String appToken = null; - - private String baseUrl; - - public boolean canExecute(Node node, NodeContext nodeContext) { - return true; - } - - @Override - public void init(Map map) throws AppJointErrorException { - appId = (String) map.get("qualitis_appId"); - appToken = (String) map.get("qualitis_appToken"); - host = getHost(); - port = getPort(); - } - - @Override - public boolean canExecute(AppJointNode appJointNode, NodeContext context, Session session) { - return appJointNode.getNodeType().toLowerCase().contains("qualitis"); - } - - @Override - public NodeExecutionAction submit(AppJointNode appJointNode, NodeContext nodeContext, Session session) { - try { - Map map = nodeContext.getRuntimeMap(); - String filter = (String) map.get("filter"); - String executionUser = (String) map.get("executeUser"); - String createUser = (String) map.get("user"); - Map map1 = ((AppJointNode)appJointNode).getJobContent(); - String id = map1.get("ruleGroupId").toString(); - float f = Float.valueOf(id); - Long groupId = (long)f; - QualitisSubmitRequest submitRequest = new QualitisSubmitRequest(); - submitRequest.setCreateUser(createUser); - submitRequest.setExecutionUser(executionUser); - submitRequest.setPartition(filter); - submitRequest.setGroupId(groupId); - - RestTemplate restTemplate = new RestTemplate(); - HttpEntity entity = generateEntity(submitRequest); - - URI url = buildUrI(host, port, SUBMIT_TASK_PATH, appId, appToken, RandomStringUtils.randomNumeric(5), String.valueOf(System.currentTimeMillis())); - String startLog = String.format("Start to submit job to qualitis. url: %s, method: %s, body: %s", url, javax.ws.rs.HttpMethod.POST, entity); - LOGGER.info(startLog); - nodeContext.appendLog(startLog); - Map response = restTemplate.postForObject(url, entity, Map.class); - String finishLog = String.format("Succeed to submit job to qualitis. response: %s", response); - LOGGER.info(finishLog); - nodeContext.appendLog(finishLog); - - if (response == null) { - String errorMsg = "Error! Can not submit job, response is null"; - LOGGER.error(errorMsg); - nodeContext.appendLog(errorMsg); - return null; - } - - if (!checkResponse(response)) { - String message = (String) response.get("message"); - String errorMsg = String.format("Error! Can not submit job, exception: %s", message); - LOGGER.error(errorMsg); - nodeContext.appendLog(errorMsg); - return null; - } - - String applicationId = (String) ((Map) response.get("data")).get("application_id"); - return new QualitisNodeExecutionAction(applicationId); - } catch (Exception e) { - String errorMsg = "Error! Can not submit job"; - LOGGER.error(errorMsg, e); - nodeContext.appendLog(errorMsg); - return null; - } - } - - @Override - public NodeExecutionState state(NodeExecutionAction nodeExecutionAction) { - if (nodeExecutionAction == null) { - return NodeExecutionState.Failed; - } - - String applicationId = ((QualitisNodeExecutionAction)nodeExecutionAction).getApplicationId(); - try { - // 发送请求查看任务的状态 - RestTemplate restTemplate = new RestTemplate(); - HttpHeaders headers = new HttpHeaders(); - HttpEntity entity = new HttpEntity(headers); - - String path = GET_TASK_STATUS_PATH.replace("{applicationId}", applicationId); - URI url = buildUrI(host, port, path, appId, appToken, RandomStringUtils.randomNumeric(5), String.valueOf(System.currentTimeMillis())); - LOGGER.info("Start to get job status. url: {}, method: {}, body: {}", url, HttpMethod.GET, entity); - Map response = restTemplate.exchange(url, HttpMethod.GET, entity, Map.class).getBody(); - LOGGER.info("Succeed to get job status. response: {}", response); - - if (response == null) { - LOGGER.error("Error! Can not get job status, job_id: {}, response is null", applicationId); - return NodeExecutionState.Failed; - } - - if (!checkResponse(response)) { - String message = (String) response.get("message"); - LOGGER.error("Error! Can not get job status, exception: {}", message); - return NodeExecutionState.Failed; - } - - List> tasks = (List>) ((Map) response.get("data")).get("task"); - Map statusCountMap = new HashMap(8); - initCountMap(statusCountMap); - Integer taskSize = tasks.size(); - for (Map task : tasks) { - Integer taskStatus = (Integer) task.get("task_status"); - addStatus(taskStatus, statusCountMap); - } - - Integer runningCount = statusCountMap.get(NodeExecutionState.Running); - Integer successCount = statusCountMap.get(NodeExecutionState.Success); - Integer failedCount = statusCountMap.get(NodeExecutionState.Failed); - - if (runningCount != 0) { - return NodeExecutionState.Running; - } else if (successCount.equals(taskSize)) { - return NodeExecutionState.Success; - } else if (failedCount != 0) { - return NodeExecutionState.Failed; - } else { - return NodeExecutionState.Accepted; - } - } catch (Exception e) { - LOGGER.error("Error! Can not get job status, job_id: {}", applicationId, e); - return NodeExecutionState.Failed; - } - } - - @Override - public CompletedNodeExecutionResponse result(NodeExecutionAction nodeExecutionAction, NodeContext nodeContext) { - if (nodeExecutionAction == null) { - return null; - } - - String applicationId = ((QualitisNodeExecutionAction)nodeExecutionAction).getApplicationId(); - try { - // Send request and get response - RestTemplate restTemplate = new RestTemplate(); - HttpHeaders headers = new HttpHeaders(); - HttpEntity entity = new HttpEntity(headers); - - String path = GET_TASK_RESULT_PATH.replace("{applicationId}", applicationId); - URI url = buildUrI(host, port, path, appId, appToken, RandomStringUtils.randomNumeric(5), String.valueOf(System.currentTimeMillis())); - - String startLog = String.format("Start to get job result. url: %s, method: %s, body: %s", url, HttpMethod.GET, entity); - LOGGER.info(startLog); - nodeContext.appendLog(startLog); - Map response = restTemplate.exchange(url, HttpMethod.GET, entity, Map.class).getBody(); - String finishLog = String.format("Succeed to get job result. response: %s", response); - LOGGER.info(finishLog); - nodeContext.appendLog(finishLog); - - if (response == null) { - String errorMsg = String.format("Error! Can not get job result, job_id: %s, response is null", applicationId); - LOGGER.error(errorMsg); - nodeContext.appendLog(errorMsg); - return null; - } - - if (!checkResponse(response)) { - String message = (String) response.get("message"); - String errorMsg = String.format("Error! Can not get job result, exception: {}", message); - LOGGER.error(errorMsg); - nodeContext.appendLog(errorMsg); - return null; - } - - Integer passNum = (Integer) ((Map) response.get("data")).get("pass_num"); - Integer failedNum = (Integer) ((Map) response.get("data")).get("failed_num"); - Integer notPassNum = (Integer) ((Map) response.get("data")).get("not_pass_num"); - String resultMessage = (String) ((Map) response.get("data")).get("result_message"); - - String taskMsg = String.format("Task result: Pass/Failed/Not Pass ------- %s/%s/%s", passNum, failedNum, notPassNum); - LOGGER.info(taskMsg); - LOGGER.info(resultMessage); - nodeContext.appendLog(taskMsg); - nodeContext.appendLog(resultMessage); - - CompletedNodeExecutionResponse result = new CompletedNodeExecutionResponse(); - if (failedNum != 0 || notPassNum != 0) { - result.setIsSucceed(false); - } else { - result.setIsSucceed(true); - } - return result; - } catch (Exception e) { - String errorMsg = String.format("Error! Can not get job result, job_id: %s", applicationId); - LOGGER.error(errorMsg, e); - nodeContext.appendLog(errorMsg); - return null; - } - } - - private void addStatus(Integer status, Map statusCountMap) { - if (status.equals(QualitisTaskStatusEnum.SUBMITTED.getCode())) { - statusCountMap.put(NodeExecutionState.Accepted, statusCountMap.get(NodeExecutionState.Accepted) + 1); - } else if (status.equals(QualitisTaskStatusEnum.INITED.getCode())) { - statusCountMap.put(NodeExecutionState.Accepted, statusCountMap.get(NodeExecutionState.Accepted) + 1); - } else if (status.equals(QualitisTaskStatusEnum.RUNNING.getCode())) { - statusCountMap.put(NodeExecutionState.Running, statusCountMap.get(NodeExecutionState.Running) + 1); - } else if (status.equals(QualitisTaskStatusEnum.SUCCEED.getCode())) { - statusCountMap.put(NodeExecutionState.Success, statusCountMap.get(NodeExecutionState.Success) + 1); - } else if (status.equals(QualitisTaskStatusEnum.PASS_CHECKOUT.getCode())) { - statusCountMap.put(NodeExecutionState.Success, statusCountMap.get(NodeExecutionState.Success) + 1); - } else if (status.equals(QualitisTaskStatusEnum.FAIL_CHECKOUT.getCode())) { - statusCountMap.put(NodeExecutionState.Failed, statusCountMap.get(NodeExecutionState.Failed) + 1); - } else if (status.equals(QualitisTaskStatusEnum.FAILED.getCode())) { - statusCountMap.put(NodeExecutionState.Failed, statusCountMap.get(NodeExecutionState.Failed) + 1); - } else if (status.equals(QualitisTaskStatusEnum.TASK_NOT_EXIST.getCode())) { - statusCountMap.put(NodeExecutionState.Failed, statusCountMap.get(NodeExecutionState.Failed) + 1); - } else if (status.equals(QualitisTaskStatusEnum.CANCELLED.getCode())) { - statusCountMap.put(NodeExecutionState.Killed, statusCountMap.get(NodeExecutionState.Killed) + 1); - } else if (status.equals(QualitisTaskStatusEnum.TIMEOUT.getCode())) { - statusCountMap.put(NodeExecutionState.Failed, statusCountMap.get(NodeExecutionState.Failed) + 1); - } else if (status.equals(QualitisTaskStatusEnum.SCHEDULED.getCode())) { - statusCountMap.put(NodeExecutionState.Accepted, statusCountMap.get(NodeExecutionState.Accepted) + 1); - } - } - - private void initCountMap(Map statusCountMap) { - statusCountMap.put(NodeExecutionState.Accepted, 0); - statusCountMap.put(NodeExecutionState.Running, 0); - statusCountMap.put(NodeExecutionState.Success, 0); - statusCountMap.put(NodeExecutionState.Failed, 0); - - } - - private Boolean checkResponse(Map response) { - String responseStatus = (String) response.get("code"); - return "200".equals(responseStatus); - } - - private HttpEntity generateEntity(Object submitRequest) { - HttpHeaders headers = new HttpHeaders(); - headers.setContentType(MediaType.APPLICATION_JSON); - - Gson gson = new Gson(); - return new HttpEntity(gson.toJson(submitRequest), headers); - } - - private URI buildUrI(String host, int port, String path, String appId, String appToken, - String nonce, String timestamp) throws Exception { - String signature = getSignature(appId, appToken, nonce, timestamp); - String urlStr = "http://" + host + ":" + port; - URI uri = UriBuilder.fromUri(urlStr) - .path(path) - .queryParam("app_id", appId) - .queryParam("nonce", nonce) - .queryParam("timestamp", timestamp) - .queryParam("signature", signature) - .build(); - return uri; - } - - private String getSignature(String appId, String appToken, String nonce, String timestamp) - throws Exception { - return Md5Utils.getMd5L32(Md5Utils.getMd5L32(appId + nonce + timestamp) + appToken); - } - - private Integer getPort() { - String baseUrl = getBaseUrl(); - return UriBuilder.fromUri(baseUrl).build().getPort(); - } - - private String getHost() { - String baseUrl = getBaseUrl(); - return UriBuilder.fromUri(baseUrl).build().getHost(); - } - - @Override - public String getBaseUrl() { - return this.baseUrl; - } - - @Override - public void setBaseUrl(String s) { - this.baseUrl = s; - } -} \ No newline at end of file diff --git a/qualitis-appjoint/appjoint/src/main/java/com.webank/wedatasphere/appjoint/QualitisNodeExecutionAction.java b/qualitis-appjoint/appjoint/src/main/java/com.webank/wedatasphere/appjoint/QualitisNodeExecutionAction.java deleted file mode 100644 index a42b4733d8..0000000000 --- a/qualitis-appjoint/appjoint/src/main/java/com.webank/wedatasphere/appjoint/QualitisNodeExecutionAction.java +++ /dev/null @@ -1,19 +0,0 @@ -package com.webank.wedatasphere.appjoint; - -import com.webank.wedatasphere.dss.appjoint.execution.common.NodeExecutionAction; - -/** - * @author howeye - */ -public class QualitisNodeExecutionAction implements NodeExecutionAction { - - private String applicationId; - - public QualitisNodeExecutionAction(String applicationId) { - this.applicationId = applicationId; - } - - public String getApplicationId() { - return applicationId; - } -} diff --git a/qualitis-appjoint/appjoint/src/main/java/com.webank/wedatasphere/appjoint/QualitisSubmitRequest.java b/qualitis-appjoint/appjoint/src/main/java/com.webank/wedatasphere/appjoint/QualitisSubmitRequest.java deleted file mode 100644 index 180da44374..0000000000 --- a/qualitis-appjoint/appjoint/src/main/java/com.webank/wedatasphere/appjoint/QualitisSubmitRequest.java +++ /dev/null @@ -1,55 +0,0 @@ -package com.webank.wedatasphere.appjoint; - -import java.util.List; -import com.google.gson.annotations.SerializedName; - -/** - * @author howeye - */ -public class QualitisSubmitRequest { - - @SerializedName("group_id") - private Long groupId; - - private String partition; - @SerializedName("execution_user") - private String executionUser; - - @SerializedName("create_user") - private String createUser; - - public QualitisSubmitRequest() { - } - - public Long getGroupId() { - return groupId; - } - - public void setGroupId(Long groupId) { - this.groupId = groupId; - } - - public String getPartition() { - return partition; - } - - public void setPartition(String partition) { - this.partition = partition; - } - - public String getExecutionUser() { - return executionUser; - } - - public void setExecutionUser(String executionUser) { - this.executionUser = executionUser; - } - - public String getCreateUser() { - return createUser; - } - - public void setCreateUser(String createUser) { - this.createUser = createUser; - } -} diff --git a/qualitis-appjoint/appjoint/src/main/java/com.webank/wedatasphere/appjoint/QualitisTaskStatusEnum.java b/qualitis-appjoint/appjoint/src/main/java/com.webank/wedatasphere/appjoint/QualitisTaskStatusEnum.java deleted file mode 100644 index 904d87ecbc..0000000000 --- a/qualitis-appjoint/appjoint/src/main/java/com.webank/wedatasphere/appjoint/QualitisTaskStatusEnum.java +++ /dev/null @@ -1,43 +0,0 @@ -package com.webank.wedatasphere.appjoint; - -/** - * @author howeye - */ -public enum QualitisTaskStatusEnum { - /** - * Task status enum - */ - SUBMITTED(1, "已提交", "SUBMITTED"), - INITED(2, "初始化", "INITED"), - RUNNING(3, "运行中", "RUNNING"), - SUCCEED(4, "成功", "SUCCEED"), - PASS_CHECKOUT(5, "通过校验", "PASS_CHECKOUT"), - FAIL_CHECKOUT(6, "未通过校验", "FAIL_CHECKOUT"), - FAILED(7, "失败", "FAILED"), - TASK_NOT_EXIST(8, "Task不存在", "TASK_NOT_EXIST"), - CANCELLED(9, "取消", "CANCELLED"), - TIMEOUT(10, "超时", "TIMEOUT"), - SCHEDULED(11, "调度中", "SCHEDULED"),; - - private Integer code; - private String message; - private String state; - - QualitisTaskStatusEnum(Integer code, String message, String state) { - this.code = code; - this.message = message; - this.state = state; - } - - public Integer getCode() { - return code; - } - - public String getMessage() { - return message; - } - - public String getState() { - return state; - } -} diff --git a/qualitis-appjoint/appjoint/src/main/java/com.webank/wedatasphere/project/QualitisAddProjectRequest.java b/qualitis-appjoint/appjoint/src/main/java/com.webank/wedatasphere/project/QualitisAddProjectRequest.java deleted file mode 100644 index 1019f20569..0000000000 --- a/qualitis-appjoint/appjoint/src/main/java/com.webank/wedatasphere/project/QualitisAddProjectRequest.java +++ /dev/null @@ -1,41 +0,0 @@ -package com.webank.wedatasphere.project; - -import com.google.gson.annotations.SerializedName; - -/** - * @author howeye - */ -public class QualitisAddProjectRequest { - - @SerializedName("project_name") - private String projectName; - private String description; - private String username; - - public QualitisAddProjectRequest() { - } - - public String getProjectName() { - return projectName; - } - - public void setProjectName(String projectName) { - this.projectName = projectName; - } - - public String getDescription() { - return description; - } - - public void setDescription(String description) { - this.description = description; - } - - public String getUsername() { - return username; - } - - public void setUsername(String username) { - this.username = username; - } -} diff --git a/qualitis-appjoint/appjoint/src/main/java/com.webank/wedatasphere/project/QualitisDeleteProjectRequest.java b/qualitis-appjoint/appjoint/src/main/java/com.webank/wedatasphere/project/QualitisDeleteProjectRequest.java deleted file mode 100644 index d4909265a2..0000000000 --- a/qualitis-appjoint/appjoint/src/main/java/com.webank/wedatasphere/project/QualitisDeleteProjectRequest.java +++ /dev/null @@ -1,32 +0,0 @@ -package com.webank.wedatasphere.project; - -import com.google.gson.annotations.SerializedName; - -/** - * @author howeye - */ -public class QualitisDeleteProjectRequest { - - @SerializedName("project_id") - private Long projectId; - private String username; - - public QualitisDeleteProjectRequest() { - } - - public Long getProjectId() { - return projectId; - } - - public void setProjectId(Long projectId) { - this.projectId = projectId; - } - - public String getUsername() { - return username; - } - - public void setUsername(String username) { - this.username = username; - } -} diff --git a/qualitis-appjoint/appjoint/src/main/java/com.webank/wedatasphere/project/QualitisProjectServiceImpl.java b/qualitis-appjoint/appjoint/src/main/java/com.webank/wedatasphere/project/QualitisProjectServiceImpl.java deleted file mode 100644 index 0c3cde2b91..0000000000 --- a/qualitis-appjoint/appjoint/src/main/java/com.webank/wedatasphere/project/QualitisProjectServiceImpl.java +++ /dev/null @@ -1,261 +0,0 @@ -package com.webank.wedatasphere.project; - - -import com.google.gson.Gson; -import com.webank.wedatasphere.Md5Utils; -import com.webank.wedatasphere.dss.appjoint.exception.AppJointErrorException; -import com.webank.wedatasphere.dss.appjoint.service.AppJointUrlImpl; -import com.webank.wedatasphere.dss.appjoint.service.ProjectService; -import com.webank.wedatasphere.dss.appjoint.service.session.Session; -import com.webank.wedatasphere.dss.common.entity.project.DSSProject; -import com.webank.wedatasphere.dss.common.entity.project.Project; -import org.apache.commons.lang.RandomStringUtils; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.springframework.http.HttpEntity; -import org.springframework.http.HttpHeaders; -import org.springframework.http.HttpMethod; -import org.springframework.http.MediaType; -import org.springframework.web.client.RestTemplate; - -import javax.ws.rs.core.UriBuilder; -import java.net.URI; -import java.util.List; -import java.util.Map; - -/** - * @author howeye - */ -public class QualitisProjectServiceImpl extends AppJointUrlImpl implements ProjectService { - - private static final String CREATE_PROJECT_PATH = "/qualitis/outer/api/v1/project/workflow"; - private static final String DELETE_PROJECT_PATH = "/qualitis/outer/api/v1/project/workflow/delete"; - private static final String UPDATE_PROJECT_PATH = "/qualitis/outer/api/v1/project/workflow"; - private static final String AUTO_ADD_USER_PATH = "/qualitis/outer/api/v1/user/{username}"; - - private String appId = "linkis_id"; - private String appToken = "a33693de51"; - - private String host = "localhost"; - private Integer port = 8090; - - private static final String FAILURE_CODE = "400"; - - private static final Logger LOGGER = LoggerFactory.getLogger(QualitisProjectServiceImpl.class); - - public void init(Map map) throws AppJointErrorException { - appId = (String) map.get("qualitis_appId"); - appToken = (String) map.get("qualitis_appToken"); - host = getHost(); - port = getPort(); - } - - @Override - public List fetchProjects(Session session) throws AppJointErrorException { - return null; - } - - @Override - public Project createProject(Project project, Session session) throws AppJointErrorException { - try { - QualitisAddProjectRequest qualitisAddProjectRequest = new QualitisAddProjectRequest(); - - DSSProject dssProject = (DSSProject) project; - qualitisAddProjectRequest.setProjectName(project.getName()); - qualitisAddProjectRequest.setDescription(project.getDescription()); - qualitisAddProjectRequest.setUsername(dssProject.getUserName()); - - RestTemplate restTemplate = new RestTemplate(); - HttpEntity entity = generateEntity(qualitisAddProjectRequest); - - Map response = createProjectReal(restTemplate, entity); - - String responseStatus = (String) response.get("code"); - if (FAILURE_CODE.equals(responseStatus)) { - // Send request to auto create qualitis user - autoAddUser(restTemplate, dssProject.getUserName()); - - // restart to create project - response = createProjectReal(restTemplate, entity); - } - - if (!checkResponse(response)) { - String message = (String) response.get("message"); - String errorMessage = String.format("Error! Can not add project, exception: %s", message); - LOGGER.error(errorMessage); - throw new AppJointErrorException(500, errorMessage); - } - - Long projectId = Long.valueOf(String.valueOf(((Map) ((Map) response.get("data")).get("project_detail")).get("project_id"))); - project.setId(projectId); - - LOGGER.info("Succeed to add project to qualitis, project_id: {}", project.getId()); - return project; - } catch (Exception e) { - LOGGER.error("Failed to add project to qualitis, caused by: {}", e.getMessage(), e); - throw new AppJointErrorException(500, e.getMessage()); - } - } - - private void autoAddUser(RestTemplate restTemplate, String username) throws Exception { - String path = AUTO_ADD_USER_PATH.replace("{username}", username); - URI url = buildUrI(getHost(), getPort(), path, appId, appToken, RandomStringUtils.randomNumeric(5), String.valueOf(System.currentTimeMillis())); - - HttpHeaders headers = new HttpHeaders(); - HttpEntity entity = new HttpEntity(headers); - - LOGGER.info("Start to add user in qualitis. url: {}, method: {}", url, HttpMethod.POST); - Map response = restTemplate.exchange(url, org.springframework.http.HttpMethod.POST, entity, Map.class).getBody(); - LOGGER.info("Succeed to add user in qualitis. response: {}", response); - - if (response == null) { - String errorMessage = "Error! Can not add user, response is null"; - LOGGER.error(errorMessage); - throw new AppJointErrorException(500, errorMessage); - } - - if (!checkResponse(response)) { - String message = (String) response.get("message"); - String errorMessage = String.format("Error! Can not add user, exception: %s", message); - LOGGER.error(errorMessage); - throw new AppJointErrorException(500, errorMessage); - } - } - - private Map createProjectReal(RestTemplate restTemplate, HttpEntity entity) throws Exception { - URI url = buildUrI(getHost(), getPort(), CREATE_PROJECT_PATH, appId, appToken, RandomStringUtils.randomNumeric(5), String.valueOf(System.currentTimeMillis())); - LOGGER.info("Start to add project to qualitis. url: {}, method: {}, body: {}", url, HttpMethod.PUT, entity); - Map response = restTemplate.exchange(url, org.springframework.http.HttpMethod.PUT, entity, Map.class).getBody(); - LOGGER.info("Succeed to add project to qualitis. response: {}", response); - - if (response == null) { - String errorMessage = "Error! Can not add project, response is null"; - LOGGER.error(errorMessage); - throw new AppJointErrorException(500, errorMessage); - } - - return response; - } - - @Override - public void deleteProject(Project project, Session session) throws AppJointErrorException { - try { - QualitisDeleteProjectRequest qualitisDeleteProjectRequest = new QualitisDeleteProjectRequest(); - - DSSProject dssProject = (DSSProject) project; - qualitisDeleteProjectRequest.setProjectId(project.getId()); - qualitisDeleteProjectRequest.setUsername(dssProject.getUserName()); - - RestTemplate restTemplate = new RestTemplate(); - HttpEntity entity = generateEntity(qualitisDeleteProjectRequest); - - URI url = buildUrI(getHost(), getPort(), DELETE_PROJECT_PATH, appId, appToken, RandomStringUtils.randomNumeric(5), String.valueOf(System.currentTimeMillis())); - LOGGER.info("Start to delete project in qualitis. url: {}, method: {}, body: {}", url, javax.ws.rs.HttpMethod.POST, entity); - Map response = restTemplate.postForObject(url, entity, Map.class); - LOGGER.info("Succeed to delete project in qualitis. response: {}", response); - - if (response == null) { - String errorMessage = "Error! Can not delete project in qualitis, response is null"; - LOGGER.error(errorMessage); - throw new AppJointErrorException(500, errorMessage); - } - - if (!checkResponse(response)) { - String message = (String) response.get("message"); - String errorMessage = String.format("Error! Can not delete project in qualitis, exception: %s", message); - LOGGER.error(errorMessage); - throw new AppJointErrorException(500, errorMessage); - } - - LOGGER.info("Succeed to delete project in qualitis, project_id: {}", project.getId()); - } catch (Exception e) { - LOGGER.error("Failed to delete project in qualitis, caused by: {}", e.getMessage(), e); - throw new AppJointErrorException(500, e.getMessage()); - } - } - - public Project getProject(Project project) throws AppJointErrorException { - return null; - } - - @Override - public void updateProject(Project project, Session session) throws AppJointErrorException { - try { - QualitisUpdateProjectRequest qualitisUpdateProjectRequest = new QualitisUpdateProjectRequest(); - - DSSProject dssProject = (DSSProject) project; - qualitisUpdateProjectRequest.setProjectId(project.getId()); - qualitisUpdateProjectRequest.setProjectName(project.getName()); - qualitisUpdateProjectRequest.setDescription(project.getDescription()); - qualitisUpdateProjectRequest.setUsername(dssProject.getUserName()); - - RestTemplate restTemplate = new RestTemplate(); - HttpEntity entity = generateEntity(qualitisUpdateProjectRequest); - - URI url = buildUrI(getHost(), getPort(), UPDATE_PROJECT_PATH, appId, appToken, RandomStringUtils.randomNumeric(5), String.valueOf(System.currentTimeMillis())); - LOGGER.info("Start to update project in qualitis. url: {}, method: {}, body: {}", url, javax.ws.rs.HttpMethod.POST, entity); - Map response = restTemplate.postForObject(url, entity, Map.class); - LOGGER.info("Succeed to update project in qualitis. response: {}", response); - - if (response == null) { - String errorMessage = "Error! Can not update project in qualitis, response is null"; - LOGGER.error(errorMessage); - throw new AppJointErrorException(500, errorMessage); - } - - if (!checkResponse(response)) { - String message = (String) response.get("message"); - String errorMessage = String.format("Error! Can not update project in qualitis, exception: %s", message); - LOGGER.error(errorMessage); - throw new AppJointErrorException(500, errorMessage); - } - - LOGGER.info("Succeed to update project in qualitis, project_id: {}", project.getId()); - } catch (Exception e) { - LOGGER.error("Failed to update project in qualitis, caused by: {}", e.getMessage(), e); - throw new AppJointErrorException(500, e.getMessage()); - } - } - - private Boolean checkResponse(Map response) { - String responseStatus = (String) response.get("code"); - return "200".equals(responseStatus); - } - - private HttpEntity generateEntity(Object submitRequest) { - HttpHeaders headers = new HttpHeaders(); - headers.setContentType(MediaType.APPLICATION_JSON); - - Gson gson = new Gson(); - return new HttpEntity(gson.toJson(submitRequest), headers); - } - - private URI buildUrI(String host, int port, String path, String appId, String appToken, - String nonce, String timestamp) throws Exception { - String signature = getSignature(appId, appToken, nonce, timestamp); - String urlStr = "http://" + host + ":" + port; - URI uri = UriBuilder.fromUri(urlStr) - .path(path) - .queryParam("app_id", appId) - .queryParam("nonce", nonce) - .queryParam("timestamp", timestamp) - .queryParam("signature", signature) - .build(); - return uri; - } - - private String getSignature(String appId, String appToken, String nonce, String timestamp) - throws Exception { - return Md5Utils.getMd5L32(Md5Utils.getMd5L32(appId + nonce + timestamp) + appToken); - } - - private Integer getPort() { - String baseUrl = getBaseUrl(); - return UriBuilder.fromUri(baseUrl).build().getPort(); - } - - private String getHost() { - String baseUrl = getBaseUrl(); - return UriBuilder.fromUri(baseUrl).build().getHost(); - } -} \ No newline at end of file diff --git a/qualitis-appjoint/appjoint/src/main/java/com.webank/wedatasphere/project/QualitisUpdateProjectRequest.java b/qualitis-appjoint/appjoint/src/main/java/com.webank/wedatasphere/project/QualitisUpdateProjectRequest.java deleted file mode 100644 index cb95958bba..0000000000 --- a/qualitis-appjoint/appjoint/src/main/java/com.webank/wedatasphere/project/QualitisUpdateProjectRequest.java +++ /dev/null @@ -1,51 +0,0 @@ -package com.webank.wedatasphere.project; - -import com.google.gson.annotations.SerializedName; - -/** - * @author howeye - */ -public class QualitisUpdateProjectRequest { - - @SerializedName("project_id") - private Long projectId; - @SerializedName("project_name") - private String projectName; - private String description; - private String username; - - public QualitisUpdateProjectRequest() { - } - - public Long getProjectId() { - return projectId; - } - - public void setProjectId(Long projectId) { - this.projectId = projectId; - } - - public String getProjectName() { - return projectName; - } - - public void setProjectName(String projectName) { - this.projectName = projectName; - } - - public String getDescription() { - return description; - } - - public void setDescription(String description) { - this.description = description; - } - - public String getUsername() { - return username; - } - - public void setUsername(String username) { - this.username = username; - } -} diff --git a/qualitis-appjoint/appjoint/src/main/resources/appjoint.properties b/qualitis-appjoint/appjoint/src/main/resources/appjoint.properties deleted file mode 100644 index 65fad6835a..0000000000 --- a/qualitis-appjoint/appjoint/src/main/resources/appjoint.properties +++ /dev/null @@ -1,19 +0,0 @@ -# -# Copyright 2019 WeBank -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# -baseUrl=http://127.0.0.1:8090 -qualitis_appId=linkis_id -qualitis_appToken=a33693de51 diff --git a/qualitis-appjoint/appjoint/src/main/resources/log4j.properties b/qualitis-appjoint/appjoint/src/main/resources/log4j.properties deleted file mode 100644 index 0807e60877..0000000000 --- a/qualitis-appjoint/appjoint/src/main/resources/log4j.properties +++ /dev/null @@ -1,37 +0,0 @@ -# -# Copyright 2019 WeBank -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# - -### set log levels ### - -log4j.rootCategory=INFO,console - -log4j.appender.console=org.apache.log4j.ConsoleAppender -log4j.appender.console.Threshold=INFO -log4j.appender.console.layout=org.apache.log4j.PatternLayout -#log4j.appender.console.layout.ConversionPattern= %d{ISO8601} %-5p (%t) [%F:%M(%L)] - %m%n -log4j.appender.console.layout.ConversionPattern= %d{ISO8601} %-5p (%t) %p %c{1} - %m%n - - -log4j.appender.com.webank.bdp.ide.core=org.apache.log4j.DailyRollingFileAppender -log4j.appender.com.webank.bdp.ide.core.Threshold=INFO -log4j.additivity.com.webank.bdp.ide.core=false -log4j.appender.com.webank.bdp.ide.core.layout=org.apache.log4j.PatternLayout -log4j.appender.com.webank.bdp.ide.core.Append=true -log4j.appender.com.webank.bdp.ide.core.File=logs/linkis.log -log4j.appender.com.webank.bdp.ide.core.layout.ConversionPattern= %d{ISO8601} %-5p (%t) [%F:%M(%L)] - %m%n - -log4j.logger.org.springframework=INFO diff --git a/qualitis-appjoint/appjoint/src/main/resources/log4j2.xml b/qualitis-appjoint/appjoint/src/main/resources/log4j2.xml deleted file mode 100644 index 3923cd9f39..0000000000 --- a/qualitis-appjoint/appjoint/src/main/resources/log4j2.xml +++ /dev/null @@ -1,39 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - diff --git a/sendemail-appjoint/sendemail-core/pom.xml b/sendemail-appjoint/sendemail-core/pom.xml deleted file mode 100644 index 3739647f0a..0000000000 --- a/sendemail-appjoint/sendemail-core/pom.xml +++ /dev/null @@ -1,129 +0,0 @@ - - - - - - dss - com.webank.wedatasphere.dss - 0.9.1 - - 4.0.0 - - dss-sendmail-appjoint-core - - - - com.webank.wedatasphere.dss - dss-appjoint-core - ${dss.version} - provided - true - - - - com.webank.wedatasphere.linkis - linkis-module - ${linkis.version} - provided - true - - - - org.springframework - spring-context-support - 5.0.7.RELEASE - - - - javax.mail - mail - 1.4 - - - - - - - - - - org.apache.maven.plugins - maven-deploy-plugin - - - - net.alchim31.maven - scala-maven-plugin - - - org.apache.maven.plugins - maven-jar-plugin - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - src/main/java - - **/*.xml - - - - src/main/resources - - **/*.properties - **/application.yml - **/bootstrap.yml - **/log4j2.xml - - - - - - - \ No newline at end of file diff --git a/sendemail-appjoint/sendemail-core/src/main/java/com/webank/wedatasphere/dss/appjoint/sendemail/Email.java b/sendemail-appjoint/sendemail-core/src/main/java/com/webank/wedatasphere/dss/appjoint/sendemail/Email.java deleted file mode 100644 index 61e1021e35..0000000000 --- a/sendemail-appjoint/sendemail-core/src/main/java/com/webank/wedatasphere/dss/appjoint/sendemail/Email.java +++ /dev/null @@ -1,45 +0,0 @@ -/* - * Copyright 2019 WeBank - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package com.webank.wedatasphere.dss.appjoint.sendemail; -/** - * Created by shanhuang on 2019/10/12. - */ -public interface Email { - - String getContent(); - void setContent(String content); - - Attachment[] getAttachments(); - void setAttachments(Attachment[] attachments); - - String getSubject(); - void setSubject(String subject); - - String getFrom(); - void setFrom(String from); - - String getTo(); - void setTo(String to); - - String getCc(); - void setCc(String cc); - - String getBcc(); - void setBcc(String bcc); - -} diff --git a/sendemail-appjoint/sendemail-core/src/main/java/com/webank/wedatasphere/dss/appjoint/sendemail/EmailContent.java b/sendemail-appjoint/sendemail-core/src/main/java/com/webank/wedatasphere/dss/appjoint/sendemail/EmailContent.java deleted file mode 100644 index a024c85df3..0000000000 --- a/sendemail-appjoint/sendemail-core/src/main/java/com/webank/wedatasphere/dss/appjoint/sendemail/EmailContent.java +++ /dev/null @@ -1,29 +0,0 @@ -/* - * Copyright 2019 WeBank - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package com.webank.wedatasphere.dss.appjoint.sendemail; - -/** - * Created by shanhuang on 2019/10/12. - */ -public interface EmailContent { - - T getContent(); - - void setContent(T content); - -} diff --git a/sendemail-appjoint/sendemail-core/src/main/java/com/webank/wedatasphere/dss/appjoint/sendemail/conf/SendEmailAppJointSpringConfiguration.java b/sendemail-appjoint/sendemail-core/src/main/java/com/webank/wedatasphere/dss/appjoint/sendemail/conf/SendEmailAppJointSpringConfiguration.java deleted file mode 100644 index 856c29233e..0000000000 --- a/sendemail-appjoint/sendemail-core/src/main/java/com/webank/wedatasphere/dss/appjoint/sendemail/conf/SendEmailAppJointSpringConfiguration.java +++ /dev/null @@ -1,80 +0,0 @@ -/* - * Copyright 2019 WeBank - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package com.webank.wedatasphere.dss.appjoint.sendemail.conf; - -import com.webank.wedatasphere.dss.appjoint.sendemail.EmailContentGenerator; -import com.webank.wedatasphere.dss.appjoint.sendemail.EmailGenerator; -import com.webank.wedatasphere.dss.appjoint.sendemail.EmailSender; -import com.webank.wedatasphere.dss.appjoint.sendemail.email.generate.MultiContentEmailGenerator; -import com.webank.wedatasphere.dss.appjoint.sendemail.emailcontent.generator.MultiEmailContentGenerator; -import com.webank.wedatasphere.dss.appjoint.sendemail.sender.SpringJavaEmailSender; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** - * Created by enjoyyin on 2019/10/14. - */ - -public class SendEmailAppJointSpringConfiguration { - - - private static final String ESB_EMAIL_SENDER = "com.webank.wedatasphere.dss.appjoint.sendemail.sender.EsbEmailSender"; - - private static final Logger logger = LoggerFactory.getLogger(SendEmailAppJointSpringConfiguration.class); - - private static final EmailGenerator EMAIL_GENERATOR = new MultiContentEmailGenerator(); - - private static final EmailSender SPRING_EMAIL_SENDER = new SpringJavaEmailSender(); - - private static final EmailContentGenerator EMAIL_CONTENT_GENERATOR = new MultiEmailContentGenerator(); - - private static EmailSender emailSender = null; - - static { - try { - Class clazz = Class.forName(ESB_EMAIL_SENDER); - emailSender = (EmailSender) clazz.newInstance(); - } catch (Exception e) { - logger.warn("{} can be instanced", ESB_EMAIL_SENDER, e); - } - } - /** - * 如果是行内就直接返回com.webank.wedatasphere.dss.appjoint.sendemail.sender.EsbEmailSender - * @return - */ - - public static EmailSender createEmailSender() { - - if (null != emailSender) { - return emailSender; - } else{ - return SPRING_EMAIL_SENDER; - } - } - - - public static EmailGenerator createEmailGenerator() { - return EMAIL_GENERATOR; - } - - - public static EmailContentGenerator createEmailContentGenerator() { - return EMAIL_CONTENT_GENERATOR; - } - -} diff --git a/sendemail-appjoint/sendemail-core/src/main/java/com/webank/wedatasphere/dss/appjoint/sendemail/exception/EmailSendFailedException.java b/sendemail-appjoint/sendemail-core/src/main/java/com/webank/wedatasphere/dss/appjoint/sendemail/exception/EmailSendFailedException.java deleted file mode 100644 index 2f8b225a79..0000000000 --- a/sendemail-appjoint/sendemail-core/src/main/java/com/webank/wedatasphere/dss/appjoint/sendemail/exception/EmailSendFailedException.java +++ /dev/null @@ -1,29 +0,0 @@ -/* - * Copyright 2019 WeBank - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package com.webank.wedatasphere.dss.appjoint.sendemail.exception; - - -import com.webank.wedatasphere.linkis.common.exception.ErrorException; -/** - * Created by shanhuang on 2019/10/12. - */ -public class EmailSendFailedException extends ErrorException { - public EmailSendFailedException(int errCode, String desc) { - super(errCode, desc); - } -} diff --git a/sendemail-appjoint/sendemail-core/src/main/java/com/webank/wedatasphere/dss/appjoint/sendemail/sender/AbstractEmailSender.java b/sendemail-appjoint/sendemail-core/src/main/java/com/webank/wedatasphere/dss/appjoint/sendemail/sender/AbstractEmailSender.java deleted file mode 100644 index 29536a1e64..0000000000 --- a/sendemail-appjoint/sendemail-core/src/main/java/com/webank/wedatasphere/dss/appjoint/sendemail/sender/AbstractEmailSender.java +++ /dev/null @@ -1,40 +0,0 @@ -/* - * Copyright 2019 WeBank - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package com.webank.wedatasphere.dss.appjoint.sendemail.sender; - -import com.webank.wedatasphere.dss.appjoint.sendemail.Email; -import com.webank.wedatasphere.dss.appjoint.sendemail.EmailSender; -import com.webank.wedatasphere.linkis.common.utils.Utils; -import scala.runtime.BoxedUnit; - -import java.util.concurrent.Future; - -/** - * Created by enjoyyin on 2019/10/13. - */ -public abstract class AbstractEmailSender implements EmailSender { - - @Override - public Future sendAsync(Email email) { - return Utils.defaultScheduler().submit(() -> { - send(email); - return BoxedUnit.UNIT; - }); - } - -} diff --git a/sendemail-appjoint/sendemail-core/src/main/java/com/webank/wedatasphere/dss/appjoint/sendemail/sender/SpringJavaEmailSender.java b/sendemail-appjoint/sendemail-core/src/main/java/com/webank/wedatasphere/dss/appjoint/sendemail/sender/SpringJavaEmailSender.java deleted file mode 100644 index eb7a6c56df..0000000000 --- a/sendemail-appjoint/sendemail-core/src/main/java/com/webank/wedatasphere/dss/appjoint/sendemail/sender/SpringJavaEmailSender.java +++ /dev/null @@ -1,99 +0,0 @@ -/* - * Copyright 2019 WeBank - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package com.webank.wedatasphere.dss.appjoint.sendemail.sender; - -import com.webank.wedatasphere.dss.appjoint.sendemail.conf.SendEmailAppJointConfiguration; -import com.webank.wedatasphere.dss.appjoint.sendemail.Attachment; -import com.webank.wedatasphere.dss.appjoint.sendemail.Email; -import com.webank.wedatasphere.dss.appjoint.sendemail.exception.EmailSendFailedException; -import org.apache.commons.lang.StringUtils; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.springframework.mail.javamail.JavaMailSender; -import org.springframework.mail.javamail.JavaMailSenderImpl; -import org.springframework.mail.javamail.MimeMessageHelper; - -import javax.mail.internet.MimeMessage; -import javax.mail.util.ByteArrayDataSource; -import java.util.Properties; - -/** - * Created by shanhuang on 2019/10/12. - */ -public class SpringJavaEmailSender extends AbstractEmailSender { - - private static final Logger logger = LoggerFactory.getLogger(SpringJavaEmailSender.class); - - private JavaMailSenderImpl javaMailSender = new JavaMailSenderImpl(); - - public SpringJavaEmailSender() { - javaMailSender.setHost(SendEmailAppJointConfiguration.EMAIL_HOST().getValue()); - javaMailSender.setPort(Integer.parseInt(SendEmailAppJointConfiguration.EMAIL_PORT().getValue())); - javaMailSender.setProtocol(SendEmailAppJointConfiguration.EMAIL_PROTOCOL().getValue()); - javaMailSender.setUsername(SendEmailAppJointConfiguration.EMAIL_USERNAME().getValue()); - javaMailSender.setPassword(SendEmailAppJointConfiguration.EMAIL_PASSWORD().getValue()); - try { - Properties prop = new Properties(); - prop.put("mail.smtp.auth", Boolean.parseBoolean(SendEmailAppJointConfiguration.EMAIL_SMTP_AUTH().getValue())); - prop.put("mail.smtp.starttls.enable", Boolean.parseBoolean(SendEmailAppJointConfiguration.EMAIL_SMTP_STARTTLS_ENABLE().getValue())); - prop.put("mail.smtp.starttls.required", Boolean.parseBoolean(SendEmailAppJointConfiguration.EMAIL_SMTP_STARTTLS_REQUIRED().getValue())); - prop.put("mail.smtp.ssl.enable", Boolean.parseBoolean(SendEmailAppJointConfiguration.EMAIL_SMTP_SSL_ENABLED().getValue())); - prop.put("mail.smtp.timeout", Integer.parseInt(SendEmailAppJointConfiguration.EMAIL_SMTP_TIMEOUT().getValue())); - javaMailSender.setJavaMailProperties(prop); - } catch (Exception e) { - logger.error("Failed to read mail properties, roll back to default values.", e); - } - } - - @Override - public void send(Email email) throws EmailSendFailedException { - logger.info("Begin to send Email(" + email.getSubject() + ")."); - try { - javaMailSender.send(parseToMimeMessage(email)); - } catch (Exception e) { - logger.error("Send email failed: ", e); - EmailSendFailedException ex = new EmailSendFailedException(80001, "Send email failed!"); - ex.initCause(e); - throw ex; - } - logger.info("Send Email(" + email.getSubject() + ") succeed."); - } - - private MimeMessage parseToMimeMessage(Email email) { - MimeMessage message = javaMailSender.createMimeMessage(); - try { - MimeMessageHelper messageHelper = new MimeMessageHelper(message, true); - if(StringUtils.isBlank(email.getFrom())) { - messageHelper.setFrom(SendEmailAppJointConfiguration.DEFAULT_EMAIL_FROM().getValue()); - } else { - messageHelper.setFrom(email.getFrom()); - } - messageHelper.setSubject(email.getSubject()); - messageHelper.setTo(email.getTo()); - messageHelper.setCc(email.getCc()); - messageHelper.setBcc(email.getBcc()); - for(Attachment attachment: email.getAttachments()){ - messageHelper.addAttachment(attachment.getName(), new ByteArrayDataSource(attachment.getBase64Str(), attachment.getMediaType())); - } - messageHelper.setText(email.getContent(), true); - } catch (Exception e) { - logger.error("Send mail failed", e); - } - return message; - } -} diff --git a/sendemail-appjoint/sendemail-core/src/main/scala/com/webank/wedatasphere/dss/appjoint/sendemail/Attachment.scala b/sendemail-appjoint/sendemail-core/src/main/scala/com/webank/wedatasphere/dss/appjoint/sendemail/Attachment.scala deleted file mode 100644 index 5b9c16e6b4..0000000000 --- a/sendemail-appjoint/sendemail-core/src/main/scala/com/webank/wedatasphere/dss/appjoint/sendemail/Attachment.scala +++ /dev/null @@ -1,32 +0,0 @@ -/* - * Copyright 2019 WeBank - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package com.webank.wedatasphere.dss.appjoint.sendemail - -import java.io.File - -/** - * Created by shanhuang on 2019/10/12. - */ -trait Attachment { - - def getName: String - def getBase64Str: String - def getFile: File - def getMediaType: String - -} diff --git a/sendemail-appjoint/sendemail-core/src/main/scala/com/webank/wedatasphere/dss/appjoint/sendemail/EmailContentGenerator.scala b/sendemail-appjoint/sendemail-core/src/main/scala/com/webank/wedatasphere/dss/appjoint/sendemail/EmailContentGenerator.scala deleted file mode 100644 index db6e7ccc0e..0000000000 --- a/sendemail-appjoint/sendemail-core/src/main/scala/com/webank/wedatasphere/dss/appjoint/sendemail/EmailContentGenerator.scala +++ /dev/null @@ -1,27 +0,0 @@ -/* - * Copyright 2019 WeBank - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package com.webank.wedatasphere.dss.appjoint.sendemail - -/** - * Created by shanhuang on 2019/10/12. - */ -trait EmailContentGenerator { - - def generate(email: Email): Unit - -} \ No newline at end of file diff --git a/sendemail-appjoint/sendemail-core/src/main/scala/com/webank/wedatasphere/dss/appjoint/sendemail/EmailContentParser.scala b/sendemail-appjoint/sendemail-core/src/main/scala/com/webank/wedatasphere/dss/appjoint/sendemail/EmailContentParser.scala deleted file mode 100644 index 19b7901468..0000000000 --- a/sendemail-appjoint/sendemail-core/src/main/scala/com/webank/wedatasphere/dss/appjoint/sendemail/EmailContentParser.scala +++ /dev/null @@ -1,27 +0,0 @@ -/* - * Copyright 2019 WeBank - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package com.webank.wedatasphere.dss.appjoint.sendemail - -/** - * Created by shanhuang on 2019/10/12. - */ -trait EmailContentParser { - - def parse(email: Email): Unit - -} diff --git a/sendemail-appjoint/sendemail-core/src/main/scala/com/webank/wedatasphere/dss/appjoint/sendemail/EmailGenerator.scala b/sendemail-appjoint/sendemail-core/src/main/scala/com/webank/wedatasphere/dss/appjoint/sendemail/EmailGenerator.scala deleted file mode 100644 index e6cefee329..0000000000 --- a/sendemail-appjoint/sendemail-core/src/main/scala/com/webank/wedatasphere/dss/appjoint/sendemail/EmailGenerator.scala +++ /dev/null @@ -1,29 +0,0 @@ -/* - * Copyright 2019 WeBank - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package com.webank.wedatasphere.dss.appjoint.sendemail - -import com.webank.wedatasphere.dss.appjoint.execution.core.{AppJointNode, NodeContext} - -/** - * Created by enjoyyin on 2019/10/12. - */ -trait EmailGenerator { - - def generateEmail(node: AppJointNode, nodeContext: NodeContext): Email - -} \ No newline at end of file diff --git a/sendemail-appjoint/sendemail-core/src/main/scala/com/webank/wedatasphere/dss/appjoint/sendemail/EmailSender.scala b/sendemail-appjoint/sendemail-core/src/main/scala/com/webank/wedatasphere/dss/appjoint/sendemail/EmailSender.scala deleted file mode 100644 index 949fb19eec..0000000000 --- a/sendemail-appjoint/sendemail-core/src/main/scala/com/webank/wedatasphere/dss/appjoint/sendemail/EmailSender.scala +++ /dev/null @@ -1,35 +0,0 @@ -/* - * Copyright 2019 WeBank - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package com.webank.wedatasphere.dss.appjoint.sendemail - -import java.util.concurrent.Future - -import com.webank.wedatasphere.dss.appjoint.sendemail.exception.EmailSendFailedException -import com.webank.wedatasphere.dss.appjoint.sendemail.exception.EmailSendFailedException - -/** - * Created by shanhuang on 2019/10/12. - */ -trait EmailSender { - - @throws(classOf[EmailSendFailedException]) - def send(email: Email): Unit - - def sendAsync(email: Email): Future[Unit] - -} diff --git a/sendemail-appjoint/sendemail-core/src/main/scala/com/webank/wedatasphere/dss/appjoint/sendemail/SendEmailAppJoint.scala b/sendemail-appjoint/sendemail-core/src/main/scala/com/webank/wedatasphere/dss/appjoint/sendemail/SendEmailAppJoint.scala deleted file mode 100644 index 97235acf9c..0000000000 --- a/sendemail-appjoint/sendemail-core/src/main/scala/com/webank/wedatasphere/dss/appjoint/sendemail/SendEmailAppJoint.scala +++ /dev/null @@ -1,51 +0,0 @@ -/* - * Copyright 2019 WeBank - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package com.webank.wedatasphere.dss.appjoint.sendemail - -import java.util - -import com.webank.wedatasphere.dss.appjoint.AppJoint -import com.webank.wedatasphere.dss.appjoint.execution.NodeExecution -import com.webank.wedatasphere.dss.appjoint.service.AppJointUrlImpl - -/** - * Created by shanhuang on 2019/10/12. - */ -class SendEmailAppJoint extends AppJointUrlImpl with AppJoint { - - private var params: util.Map[String, AnyRef] = _ - private var nodeExecution: NodeExecution = _ - - override def getAppJointName = "SendEmail" - - override def init(baseUrl: String, params: util.Map[String, AnyRef]): Unit = { - setBaseUrl(baseUrl) - this.params = params - } - - override def getNodeExecution: NodeExecution = { - if(nodeExecution == null) synchronized { - if(nodeExecution == null) { - nodeExecution = new SendEmailNodeExecution - nodeExecution.setBaseUrl(getBaseUrl) - nodeExecution.init(params) - } - } - nodeExecution - } -} diff --git a/sendemail-appjoint/sendemail-core/src/main/scala/com/webank/wedatasphere/dss/appjoint/sendemail/SendEmailNodeExecution.scala b/sendemail-appjoint/sendemail-core/src/main/scala/com/webank/wedatasphere/dss/appjoint/sendemail/SendEmailNodeExecution.scala deleted file mode 100644 index 9f81f3724a..0000000000 --- a/sendemail-appjoint/sendemail-core/src/main/scala/com/webank/wedatasphere/dss/appjoint/sendemail/SendEmailNodeExecution.scala +++ /dev/null @@ -1,85 +0,0 @@ -/* - * Copyright 2019 WeBank - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package com.webank.wedatasphere.dss.appjoint.sendemail - -import java.util - -import com.webank.wedatasphere.dss.appjoint.execution.NodeExecution -import com.webank.wedatasphere.dss.appjoint.execution.common.{CompletedNodeExecutionResponse, NodeExecutionResponse} -import com.webank.wedatasphere.dss.appjoint.execution.core.{AppJointNode, NodeContext} -import com.webank.wedatasphere.dss.appjoint.sendemail.conf.SendEmailAppJointSpringConfiguration -import com.webank.wedatasphere.dss.appjoint.sendemail.emailcontent.parser.{FileEmailContentParser, HtmlEmailContentParser, PictureEmailContentParser, TableEmailContentParser} -import com.webank.wedatasphere.dss.appjoint.service.AppJointUrlImpl -import com.webank.wedatasphere.dss.appjoint.service.session.Session -import com.webank.wedatasphere.linkis.common.exception.ErrorException -import com.webank.wedatasphere.linkis.common.utils.{Logging, Utils} - -/** - * Created by shanhuang on 2019/10/12. - */ -class SendEmailNodeExecution extends AppJointUrlImpl with NodeExecution with Logging{ - - private val sendEmailAppJointHooks = Array.empty[SendEmailNodeExecutionHook] - private val emailContentParsers = - Array[EmailContentParser](FileEmailContentParser, HtmlEmailContentParser, PictureEmailContentParser, TableEmailContentParser) - private val emailContentGenerators = Array[EmailContentGenerator](SendEmailAppJointSpringConfiguration.createEmailContentGenerator()) - - private val emailGenerator = SendEmailAppJointSpringConfiguration.createEmailGenerator() - - private val emailSender = SendEmailAppJointSpringConfiguration.createEmailSender() - - override def canExecute(node: AppJointNode, nodeContext: NodeContext, session: Session): Boolean = node.getNodeType.toLowerCase.contains("sendemail") - - override def execute(node: AppJointNode, nodeContext: NodeContext, seesion:Session): NodeExecutionResponse = { - val response = new CompletedNodeExecutionResponse - val email = Utils.tryCatch { - sendEmailAppJointHooks.foreach(_.preGenerate(node, nodeContext)) - val email = emailGenerator.generateEmail(node, nodeContext) - emailContentParsers.foreach{ - p => Utils.tryQuietly(p.parse(email)) - } - emailContentGenerators.foreach{ - g => Utils.tryQuietly(g.generate(email)) - } - sendEmailAppJointHooks.foreach(_.preSend(node, nodeContext, email)) - email - }{ t => - putErrorMsg("解析邮件内容失败！", t, response) - return response - } - Utils.tryCatch { - emailSender.send(email) - response.setIsSucceed(true) - }(putErrorMsg("发送邮件失败！", _, response)) - response - } - - protected def putErrorMsg(errorMsg: String, t: Throwable, - response: CompletedNodeExecutionResponse): Unit = t match { - case t: Throwable => - response.setErrorMsg(errorMsg) - val exception = new ErrorException(80079, "failed to sendEmail") - exception.initCause(t) - logger.error(s"failed to send email, $errorMsg ", t) - response.setException(exception) - response.setIsSucceed(false) - } - - - override def init(params: util.Map[String, AnyRef]): Unit = {} -} diff --git a/sendemail-appjoint/sendemail-core/src/main/scala/com/webank/wedatasphere/dss/appjoint/sendemail/SendEmailNodeExecutionHook.scala b/sendemail-appjoint/sendemail-core/src/main/scala/com/webank/wedatasphere/dss/appjoint/sendemail/SendEmailNodeExecutionHook.scala deleted file mode 100644 index e13371be29..0000000000 --- a/sendemail-appjoint/sendemail-core/src/main/scala/com/webank/wedatasphere/dss/appjoint/sendemail/SendEmailNodeExecutionHook.scala +++ /dev/null @@ -1,31 +0,0 @@ -/* - * Copyright 2019 WeBank - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package com.webank.wedatasphere.dss.appjoint.sendemail - -import com.webank.wedatasphere.dss.appjoint.execution.core.{AppJointNode, NodeContext} - -/** - * Created by enjoyyin on 2019/10/12. - */ -trait SendEmailNodeExecutionHook { - - def preGenerate(node: AppJointNode, nodeContext: NodeContext): Unit - - def preSend(node: AppJointNode, nodeContext: NodeContext, email: Email): Unit - -} diff --git a/sendemail-appjoint/sendemail-core/src/main/scala/com/webank/wedatasphere/dss/appjoint/sendemail/conf/SendEmailAppJointConfiguration.scala b/sendemail-appjoint/sendemail-core/src/main/scala/com/webank/wedatasphere/dss/appjoint/sendemail/conf/SendEmailAppJointConfiguration.scala deleted file mode 100644 index 1f271c95b5..0000000000 --- a/sendemail-appjoint/sendemail-core/src/main/scala/com/webank/wedatasphere/dss/appjoint/sendemail/conf/SendEmailAppJointConfiguration.scala +++ /dev/null @@ -1,43 +0,0 @@ -/* - * Copyright 2019 WeBank - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package com.webank.wedatasphere.dss.appjoint.sendemail.conf - -import com.webank.wedatasphere.linkis.common.conf.CommonVars - -/** - * Created by shanhuang on 2019/10/12. - */ -object SendEmailAppJointConfiguration { - - val EMAIL_IMAGE_HEIGHT = CommonVars("wds.dataworkis.appjoint.email.image.height", 500) - val EMAIL_IMAGE_WIDTH = CommonVars("wds.dataworkis.appjoint.email.image.width", 1920) - val DEFAULT_EMAIL_FROM = CommonVars("wds.dataworkis.appjoint.email.from.default", "") - - val EMAIL_HOST = CommonVars("wds.dataworkis.appjoint.email.host", "") - val EMAIL_PORT = CommonVars("wds.dataworkis.appjoint.email.port", "") - val EMAIL_PROTOCOL = CommonVars("wds.dataworkis.appjoint.email.protocol", "smtp") - val EMAIL_USERNAME = CommonVars("wds.dataworkis.appjoint.email.username", "") - val EMAIL_PASSWORD = CommonVars("wds.dataworkis.appjoint.email.password", "") - - val EMAIL_SMTP_AUTH = CommonVars("wds.dataworkis.appjoint.email.smtp.auth", "true") - val EMAIL_SMTP_STARTTLS_ENABLE = CommonVars("wds.dataworkis.appjoint.email.smtp.starttls.enable", "true") - val EMAIL_SMTP_STARTTLS_REQUIRED = CommonVars("wds.dataworkis.appjoint.email.smtp.starttls.required", "true") - val EMAIL_SMTP_SSL_ENABLED = CommonVars("wds.dataworkis.appjoint.email.smtp.ssl.enable", "true") - val EMAIL_SMTP_TIMEOUT = CommonVars("wds.dataworkis.appjoint.email.smtp.timeout", "25000") - -} diff --git a/sendemail-appjoint/sendemail-core/src/main/scala/com/webank/wedatasphere/dss/appjoint/sendemail/conf/SendEmailAppJointUtils.scala b/sendemail-appjoint/sendemail-core/src/main/scala/com/webank/wedatasphere/dss/appjoint/sendemail/conf/SendEmailAppJointUtils.scala deleted file mode 100644 index 38ca582104..0000000000 --- a/sendemail-appjoint/sendemail-core/src/main/scala/com/webank/wedatasphere/dss/appjoint/sendemail/conf/SendEmailAppJointUtils.scala +++ /dev/null @@ -1,33 +0,0 @@ -/* - * Copyright 2019 WeBank - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package com.webank.wedatasphere.dss.appjoint.sendemail.conf - -/** - * Created by enjoyyin on 2019/10/12. - */ -object SendEmailAppJointUtils { - -// def getInstance[T](clazz: Class[T]): T = DataWorkCloudApplication.getApplicationContext.getBean(clazz) -// -// def getInstances[T](clazz: Class[T]): Seq[T] = { -// val instances = ArrayBuffer[T]() -// DataWorkCloudApplication.getApplicationContext.getBeansOfType(clazz).foreach(instances += _._2) -// instances -// } - -} \ No newline at end of file diff --git a/sendemail-appjoint/sendemail-core/src/main/scala/com/webank/wedatasphere/dss/appjoint/sendemail/email/AbstractEmail.scala b/sendemail-appjoint/sendemail-core/src/main/scala/com/webank/wedatasphere/dss/appjoint/sendemail/email/AbstractEmail.scala deleted file mode 100644 index 277100807e..0000000000 --- a/sendemail-appjoint/sendemail-core/src/main/scala/com/webank/wedatasphere/dss/appjoint/sendemail/email/AbstractEmail.scala +++ /dev/null @@ -1,60 +0,0 @@ -/* - * Copyright 2019 WeBank - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package com.webank.wedatasphere.dss.appjoint.sendemail.email - -import com.webank.wedatasphere.dss.appjoint.sendemail.{Attachment, Email} -import com.webank.wedatasphere.dss.appjoint.sendemail.{Attachment, Email} - -import scala.collection.mutable.ArrayBuffer - -/** - * Created by shanhuang on 2019/10/12. - */ -class AbstractEmail extends Email { - - private var content: String = _ - private var attachments = ArrayBuffer[Attachment]() - private var subject: String = _ - private var from: String = _ - private var to: String = _ - private var cc: String = _ - private var bcc: String = _ - - override def getContent: String = content - override def setContent(content: String): Unit = this.content = content - - override def getAttachments: Array[Attachment] = attachments.toArray - override def setAttachments(attachments: Array[Attachment]): Unit = - this.attachments ++= attachments - def addAttachment(attachment: Attachment): Unit = this.attachments += attachment - - override def getSubject: String = subject - override def setSubject(subject: String): Unit = this.subject = subject - - override def getFrom: String = from - override def setFrom(from: String): Unit = this.from = from - - override def getTo: String = to - override def setTo(to: String): Unit = this.to = to - - override def getCc: String = cc - override def setCc(cc: String): Unit = this.cc = cc - - override def getBcc: String = bcc - override def setBcc(bcc: String): Unit = this.bcc = bcc -} diff --git a/sendemail-appjoint/sendemail-core/src/main/scala/com/webank/wedatasphere/dss/appjoint/sendemail/email/MultiContentEmail.scala b/sendemail-appjoint/sendemail-core/src/main/scala/com/webank/wedatasphere/dss/appjoint/sendemail/email/MultiContentEmail.scala deleted file mode 100644 index 4fde2ac36d..0000000000 --- a/sendemail-appjoint/sendemail-core/src/main/scala/com/webank/wedatasphere/dss/appjoint/sendemail/email/MultiContentEmail.scala +++ /dev/null @@ -1,38 +0,0 @@ -/* - * Copyright 2019 WeBank - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package com.webank.wedatasphere.dss.appjoint.sendemail.email - -import java.util - -import com.webank.wedatasphere.dss.appjoint.sendemail.EmailContent -import com.webank.wedatasphere.dss.appjoint.sendemail.EmailContent - -import scala.collection.JavaConversions._ - -/** - * Created by shanhuang on 2019/10/12. - */ -class MultiContentEmail extends AbstractEmail { - - private val emailContents = new util.ArrayList[EmailContent[_]]() - - def addEmailContent(emailContent: EmailContent[_]): Unit = emailContents.add(emailContent) - - def getEmailContents: Array[EmailContent[_]] = emailContents.toIterator.toArray - -} diff --git a/sendemail-appjoint/sendemail-core/src/main/scala/com/webank/wedatasphere/dss/appjoint/sendemail/email/PngAttachment.scala b/sendemail-appjoint/sendemail-core/src/main/scala/com/webank/wedatasphere/dss/appjoint/sendemail/email/PngAttachment.scala deleted file mode 100644 index 94c6e12ccf..0000000000 --- a/sendemail-appjoint/sendemail-core/src/main/scala/com/webank/wedatasphere/dss/appjoint/sendemail/email/PngAttachment.scala +++ /dev/null @@ -1,38 +0,0 @@ -/* - * Copyright 2019 WeBank - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package com.webank.wedatasphere.dss.appjoint.sendemail.email - -import java.io.File - -import com.webank.wedatasphere.dss.appjoint.sendemail.Attachment -import com.webank.wedatasphere.dss.appjoint.sendemail.Attachment - -/** - * Created by shanhuang on 2019/10/12. - */ -class PngAttachment(name: String, b64: String) extends Attachment { - - override def getName: String = name - - override def getBase64Str: String = b64 - - override def getFile: File = null //TODO write b64 to file - - override def getMediaType = "image/png" - -} \ No newline at end of file diff --git a/sendemail-appjoint/sendemail-core/src/main/scala/com/webank/wedatasphere/dss/appjoint/sendemail/email/generate/AbstractEmailGenerator.scala b/sendemail-appjoint/sendemail-core/src/main/scala/com/webank/wedatasphere/dss/appjoint/sendemail/email/generate/AbstractEmailGenerator.scala deleted file mode 100644 index 2cc77339e6..0000000000 --- a/sendemail-appjoint/sendemail-core/src/main/scala/com/webank/wedatasphere/dss/appjoint/sendemail/email/generate/AbstractEmailGenerator.scala +++ /dev/null @@ -1,61 +0,0 @@ -/* - * Copyright 2019 WeBank - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package com.webank.wedatasphere.dss.appjoint.sendemail.email.generate - -import com.webank.wedatasphere.dss.appjoint.sendemail.email.AbstractEmail -import com.webank.wedatasphere.dss.appjoint.sendemail.Email -import com.webank.wedatasphere.dss.appjoint.execution.core.{AppJointNode, NodeContext} -import com.webank.wedatasphere.dss.appjoint.sendemail.email.AbstractEmail -import com.webank.wedatasphere.dss.appjoint.sendemail.{Email, EmailGenerator} -import com.webank.wedatasphere.linkis.common.utils.Logging - -/** - * Created by shanhuang on 2019/10/12. - */ -trait AbstractEmailGenerator extends EmailGenerator with Logging{ - - protected def createEmail(): AbstractEmail - - override def generateEmail(node: AppJointNode, nodeContext: NodeContext): Email = { - val email = createEmail() - generateEmailInfo(node, nodeContext, email) - generateEmailContent(node, nodeContext, email) - email - } - - protected def generateEmailInfo(node: AppJointNode, nodeContext: NodeContext, email: AbstractEmail): Unit = { - import scala.collection.JavaConversions._ - nodeContext.getRuntimeMap foreach { - case (k, v) => logger.info(s"K is $k, V is $v") - } - val subject = if (nodeContext.getRuntimeMap.get("subject") != null) nodeContext.getRuntimeMap.get("subject").toString else "This is an email" - email.setSubject(subject) - val bcc = if (nodeContext.getRuntimeMap.get("bcc") != null) nodeContext.getRuntimeMap.get("bcc").toString else "" - email.setBcc(bcc) - val cc = if (nodeContext.getRuntimeMap.get("cc") != null) nodeContext.getRuntimeMap.get("cc").toString else "" - email.setCc(cc) - val from = if (nodeContext.getRuntimeMap.get("from") != null) nodeContext.getRuntimeMap.get("from").toString else - nodeContext.getRuntimeMap.get("user").toString - email.setFrom(from) - val to = if (nodeContext.getRuntimeMap.get("to") != null) nodeContext.getRuntimeMap.get("to").toString else "" - email.setTo(to) - } - - protected def generateEmailContent(node: AppJointNode, nodeContext: NodeContext, email: AbstractEmail): Unit - -} diff --git a/sendemail-appjoint/sendemail-core/src/main/scala/com/webank/wedatasphere/dss/appjoint/sendemail/email/generate/MultiContentEmailGenerator.scala b/sendemail-appjoint/sendemail-core/src/main/scala/com/webank/wedatasphere/dss/appjoint/sendemail/email/generate/MultiContentEmailGenerator.scala deleted file mode 100644 index 3425109e77..0000000000 --- a/sendemail-appjoint/sendemail-core/src/main/scala/com/webank/wedatasphere/dss/appjoint/sendemail/email/generate/MultiContentEmailGenerator.scala +++ /dev/null @@ -1,63 +0,0 @@ -/* - * Copyright 2019 WeBank - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package com.webank.wedatasphere.dss.appjoint.sendemail.email.generate - -import com.webank.wedatasphere.dss.appjoint.sendemail._ -import com.webank.wedatasphere.dss.appjoint.sendemail.email.{AbstractEmail, MultiContentEmail} -import com.webank.wedatasphere.linkis.common.io.FsPath -import com.webank.wedatasphere.dss.appjoint.execution.core.{AppJointNode, NodeContext} -import com.webank.wedatasphere.dss.appjoint.sendemail.EmailContent -import com.webank.wedatasphere.dss.appjoint.sendemail.email.{AbstractEmail, MultiContentEmail} -import com.webank.wedatasphere.dss.appjoint.sendemail.emailcontent._ -import com.webank.wedatasphere.linkis.storage.resultset.ResultSetFactory - -/** - * Created by shanhuang on 2019/10/12. - */ -class MultiContentEmailGenerator extends AbstractEmailGenerator { - - override protected def createEmail() = new MultiContentEmail - - override protected def generateEmailContent(node: AppJointNode, nodeContext: NodeContext, email: AbstractEmail): Unit = email match { - case multiContentEmail: MultiContentEmail => - def addContentEmail(getEmailContent: String => EmailContent[_]): Unit = nodeContext.getRuntimeMap.get("content") match { - case content: Array[String] => content.foreach(c => multiContentEmail.addEmailContent(getEmailContent(c))) - case content: String => multiContentEmail.addEmailContent(getEmailContent(content)) - } - nodeContext.getRuntimeMap.get("category") match { - case "node" => - val resultSetFactory = ResultSetFactory.getInstance - nodeContext.getJobIdsOfShareNode.foreach { jobId => - nodeContext.getResultSetPathsByJobId(jobId).foreach { fsPath => - val resultSet = resultSetFactory.getResultSetByPath(fsPath) - val emailContent = resultSet.resultSetType() match { - case ResultSetFactory.PICTURE_TYPE => new PictureEmailContent(fsPath) - case ResultSetFactory.HTML_TYPE => new HtmlEmailContent(fsPath) - case ResultSetFactory.TABLE_TYPE => new TableEmailContent(fsPath) - case ResultSetFactory.TEXT_TYPE => new FileEmailContent(fsPath) - } - multiContentEmail.addEmailContent(emailContent) - } - } - case "file" => addContentEmail(c => new FileEmailContent(new FsPath(c))) - case "text" => addContentEmail(new TextEmailContent(_)) - case "link" => addContentEmail(new UrlEmailContent(_)) - } - } - -} \ No newline at end of file diff --git a/sendemail-appjoint/sendemail-core/src/main/scala/com/webank/wedatasphere/dss/appjoint/sendemail/emailcontent/ArrayEmailContent.scala b/sendemail-appjoint/sendemail-core/src/main/scala/com/webank/wedatasphere/dss/appjoint/sendemail/emailcontent/ArrayEmailContent.scala deleted file mode 100644 index 2b13828253..0000000000 --- a/sendemail-appjoint/sendemail-core/src/main/scala/com/webank/wedatasphere/dss/appjoint/sendemail/emailcontent/ArrayEmailContent.scala +++ /dev/null @@ -1,34 +0,0 @@ -/* - * Copyright 2019 WeBank - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package com.webank.wedatasphere.dss.appjoint.sendemail.emailcontent - -import com.webank.wedatasphere.dss.appjoint.sendemail.EmailContent -import com.webank.wedatasphere.dss.appjoint.sendemail.EmailContent - -/** - * Created by enjoyyin on 2019/10/13. - */ -class ArrayEmailContent extends EmailContent[Array[String]] { - - private var content: Array[String] = _ - - override def getContent: Array[String] = content - - override def setContent(content: Array[String]): Unit = this.content = content - -} \ No newline at end of file diff --git a/sendemail-appjoint/sendemail-core/src/main/scala/com/webank/wedatasphere/dss/appjoint/sendemail/emailcontent/FsPathStoreEmailContent.scala b/sendemail-appjoint/sendemail-core/src/main/scala/com/webank/wedatasphere/dss/appjoint/sendemail/emailcontent/FsPathStoreEmailContent.scala deleted file mode 100644 index 0599121555..0000000000 --- a/sendemail-appjoint/sendemail-core/src/main/scala/com/webank/wedatasphere/dss/appjoint/sendemail/emailcontent/FsPathStoreEmailContent.scala +++ /dev/null @@ -1,30 +0,0 @@ -/* - * Copyright 2019 WeBank - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package com.webank.wedatasphere.dss.appjoint.sendemail.emailcontent - -import com.webank.wedatasphere.linkis.common.io.FsPath - -/** - * Created by enjoyyin on 2019/10/13. - */ -trait FsPathStoreEmailContent { - private var fsPath: FsPath = _ - - def getFsPath: FsPath = fsPath - def setFsPath(fsPath: FsPath): Unit = this.fsPath = fsPath -} \ No newline at end of file diff --git a/sendemail-appjoint/sendemail-core/src/main/scala/com/webank/wedatasphere/dss/appjoint/sendemail/emailcontent/StringEmailContent.scala b/sendemail-appjoint/sendemail-core/src/main/scala/com/webank/wedatasphere/dss/appjoint/sendemail/emailcontent/StringEmailContent.scala deleted file mode 100644 index c3065f7ba9..0000000000 --- a/sendemail-appjoint/sendemail-core/src/main/scala/com/webank/wedatasphere/dss/appjoint/sendemail/emailcontent/StringEmailContent.scala +++ /dev/null @@ -1,32 +0,0 @@ -/* - * Copyright 2019 WeBank - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package com.webank.wedatasphere.dss.appjoint.sendemail.emailcontent - -import com.webank.wedatasphere.dss.appjoint.sendemail.EmailContent - -/** - * Created by enjoyyin on 2019/10/12. - */ -class StringEmailContent extends EmailContent[String] { - private var content: String = _ - - override def getContent: String = content - - override def setContent(content: String): Unit = this.content = content - -} \ No newline at end of file diff --git a/sendemail-appjoint/sendemail-core/src/main/scala/com/webank/wedatasphere/dss/appjoint/sendemail/emailcontent/generator/AbstractEmailContentGenerator.scala b/sendemail-appjoint/sendemail-core/src/main/scala/com/webank/wedatasphere/dss/appjoint/sendemail/emailcontent/generator/AbstractEmailContentGenerator.scala deleted file mode 100644 index 53cf5a0fb5..0000000000 --- a/sendemail-appjoint/sendemail-core/src/main/scala/com/webank/wedatasphere/dss/appjoint/sendemail/emailcontent/generator/AbstractEmailContentGenerator.scala +++ /dev/null @@ -1,64 +0,0 @@ -/* - * Copyright 2019 WeBank - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package com.webank.wedatasphere.dss.appjoint.sendemail.emailcontent.generator - -import java.text.SimpleDateFormat -import java.util.{Calendar, Date} - -import com.webank.wedatasphere.dss.appjoint.sendemail.{Email, EmailContentGenerator} -import com.webank.wedatasphere.dss.appjoint.sendemail.{Email, EmailContentGenerator} - -/** - * Created by shanhuang on 2019/10/12. - */ -trait AbstractEmailContentGenerator extends EmailContentGenerator { - - /** - * 兼容Visualis老版的变量设置方式 - * @param email - */ - protected def formatSubjectOfOldVersion(email: Email): Unit = { - var title = email.getSubject - if (title.contains("YYYY-MM-DD HH:MM:SS")) { - val sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss") - val timeStr = sdf.format(new Date) - title = title.replace("YYYY-MM-DD HH:MM:SS", timeStr) - } else if (title.contains("YYYY-MM-DD-1")) { - val sdf = new SimpleDateFormat("yyyy-MM-dd") - val calendar = Calendar.getInstance - calendar.add(Calendar.DATE, -1) - val timeStr = sdf.format(calendar.getTime) - title = title.replace("YYYY-MM-DD-1", timeStr) - } else if (title.contains("YYYY-MM-DD")) { - val sdf = new SimpleDateFormat("yyyy-MM-dd") - val timeStr = sdf.format(new Date) - title = title.replace("YYYY-MM-DD", timeStr) - } else { - val sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss") - val timeStr = sdf.format(new Date) - title = title + timeStr - } - // if (!title.contains("试运行版")) title = "【试运行版】" + title - email.setSubject(title) - } - - protected def formatSubject(email: Email): Unit = { - - } - -} diff --git a/sendemail-appjoint/sendemail-core/src/main/scala/com/webank/wedatasphere/dss/appjoint/sendemail/emailcontent/generator/MultiEmailContentGenerator.scala b/sendemail-appjoint/sendemail-core/src/main/scala/com/webank/wedatasphere/dss/appjoint/sendemail/emailcontent/generator/MultiEmailContentGenerator.scala deleted file mode 100644 index c48e0115c0..0000000000 --- a/sendemail-appjoint/sendemail-core/src/main/scala/com/webank/wedatasphere/dss/appjoint/sendemail/emailcontent/generator/MultiEmailContentGenerator.scala +++ /dev/null @@ -1,56 +0,0 @@ -/* - * Copyright 2019 WeBank - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package com.webank.wedatasphere.dss.appjoint.sendemail.emailcontent.generator - -import com.webank.wedatasphere.dss.appjoint.sendemail.Email -import com.webank.wedatasphere.dss.appjoint.sendemail.email.MultiContentEmail -import com.webank.wedatasphere.dss.appjoint.sendemail.emailcontent.{ArrayEmailContent, StringEmailContent} -import com.webank.wedatasphere.dss.appjoint.sendemail.Email -import com.webank.wedatasphere.dss.appjoint.sendemail.email.MultiContentEmail -import com.webank.wedatasphere.dss.appjoint.sendemail.emailcontent.{ArrayEmailContent, StringEmailContent} -import com.webank.wedatasphere.linkis.common.utils.Logging -import org.springframework.stereotype.Component - -/** - * Created by shanhuang on 2019/10/12. - */ - -class MultiEmailContentGenerator extends AbstractEmailContentGenerator with Logging { - - override def generate(email: Email): Unit = email match { - case multiContentEmail: MultiContentEmail => - formatSubjectOfOldVersion(email) - formatSubject(multiContentEmail) - formatContent(multiContentEmail) - } - - protected def formatContent(email: MultiContentEmail): Unit = { - val sb = new StringBuilder("") - sb.append("") - email.getEmailContents.foreach { - case emailContent: ArrayEmailContent => - emailContent.getContent.foreach(content => sb.append("")) - case emailContent: StringEmailContent => - sb.append("") - } - sb.append("
").append(content).append("
").append(emailContent.getContent).append("
") - sb.append("") - email.setContent(sb.toString) - } - -} \ No newline at end of file diff --git a/sendemail-appjoint/sendemail-core/src/main/scala/com/webank/wedatasphere/dss/appjoint/sendemail/emailcontent/package.scala b/sendemail-appjoint/sendemail-core/src/main/scala/com/webank/wedatasphere/dss/appjoint/sendemail/emailcontent/package.scala deleted file mode 100644 index 525a1409e5..0000000000 --- a/sendemail-appjoint/sendemail-core/src/main/scala/com/webank/wedatasphere/dss/appjoint/sendemail/emailcontent/package.scala +++ /dev/null @@ -1,86 +0,0 @@ -/* - * Copyright 2019 WeBank - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package com.webank.wedatasphere.dss.appjoint.sendemail.emailcontent - -import com.webank.wedatasphere.dss.appjoint.sendemail.emailcontent.StringEmailContent -import com.webank.wedatasphere.linkis.common.io.FsPath - -import scala.beans.BeanProperty - -/** - * Created by enjoyyin on 2019/10/13. - */ -package object emailcontent { - -} - -class PictureEmailContent extends ArrayEmailContent with FsPathStoreEmailContent { - - def this(filePath: FsPath) = { - this() - setFsPath(filePath) - } - -} - -class HtmlEmailContent extends StringEmailContent with FsPathStoreEmailContent { - - def this(filePath: FsPath) = { - this() - setFsPath(filePath) - } - -} - -class TableEmailContent extends StringEmailContent with FsPathStoreEmailContent { - - def this(filePath: FsPath) = { - this() - setFsPath(filePath) - } - -} - -class FileEmailContent extends StringEmailContent with FsPathStoreEmailContent { - - def this(filePath: FsPath) = { - this() - setFsPath(filePath) - } - -} - -class TextEmailContent extends StringEmailContent { - - def this(text: String) = { - this() - setContent(text) - } - -} - -class UrlEmailContent extends StringEmailContent { - - @BeanProperty var url: String = _ - - def this(url: String) = { - this() - this.url = url - } - -} \ No newline at end of file diff --git a/sendemail-appjoint/sendemail-core/src/main/scala/com/webank/wedatasphere/dss/appjoint/sendemail/emailcontent/parser/AbstractEmailContentParser.scala b/sendemail-appjoint/sendemail-core/src/main/scala/com/webank/wedatasphere/dss/appjoint/sendemail/emailcontent/parser/AbstractEmailContentParser.scala deleted file mode 100644 index 98249bcdf3..0000000000 --- a/sendemail-appjoint/sendemail-core/src/main/scala/com/webank/wedatasphere/dss/appjoint/sendemail/emailcontent/parser/AbstractEmailContentParser.scala +++ /dev/null @@ -1,71 +0,0 @@ -/* - * Copyright 2019 WeBank - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package com.webank.wedatasphere.dss.appjoint.sendemail.emailcontent.parser - -import java.lang.reflect.{ParameterizedType, Type} - -import com.webank.wedatasphere.dss.appjoint.sendemail.email.MultiContentEmail -import com.webank.wedatasphere.dss.appjoint.sendemail.emailcontent.FsPathStoreEmailContent -import com.webank.wedatasphere.dss.appjoint.sendemail.{Email, EmailContent, EmailContentParser} -import com.webank.wedatasphere.dss.appjoint.sendemail.email.MultiContentEmail -import com.webank.wedatasphere.dss.appjoint.sendemail.emailcontent.FsPathStoreEmailContent -import com.webank.wedatasphere.dss.appjoint.sendemail.{Email, EmailContent, EmailContentParser} -import com.webank.wedatasphere.linkis.common.io.resultset.ResultSetReader -import com.webank.wedatasphere.linkis.common.io.{MetaData, Record} -import com.webank.wedatasphere.linkis.common.utils.Utils -import com.webank.wedatasphere.linkis.storage.LineRecord -import com.webank.wedatasphere.linkis.storage.resultset.ResultSetReader -import org.apache.commons.io.IOUtils - -/** - * Created by shanhuang on 2019/10/12. - */ -abstract class AbstractEmailContentParser[T] extends EmailContentParser { - - override def parse(email: Email): Unit = email match { - case multiContentEmail: MultiContentEmail => - multiContentEmail.getEmailContents.foreach { - case t: EmailContent[_] if t.getClass == getEmailContentClass => - parseEmailContent(t.asInstanceOf[T], multiContentEmail) - case _ => - } - case _ => - } - - protected def getResultSetReader(fsPathStore: FsPathStoreEmailContent): ResultSetReader[_ <: MetaData, _ <: Record] = { - val reader = ResultSetReader.getResultSetReader(fsPathStore.getFsPath.getSchemaPath) - reader.getMetaData - reader - } - - protected def getFirstLineRecord(fsPathStore: FsPathStoreEmailContent): Option[String] = { - val reader = getResultSetReader(fsPathStore) - if(!reader.hasNext) None - else Utils.tryFinally(reader.getRecord match { - case record: LineRecord => Option(record.getLine) - })(IOUtils.closeQuietly(reader)) - } - - protected def getEmailContentClass: Type = getClass.getGenericSuperclass match { - case pType: ParameterizedType => pType.getActualTypeArguments.head - } - - protected def parseEmailContent(emailContent: T, - multiContentEmail: MultiContentEmail): Unit - -} diff --git a/sendemail-appjoint/sendemail-core/src/main/scala/com/webank/wedatasphere/dss/appjoint/sendemail/emailcontent/parser/FileEmailContentParser.scala b/sendemail-appjoint/sendemail-core/src/main/scala/com/webank/wedatasphere/dss/appjoint/sendemail/emailcontent/parser/FileEmailContentParser.scala deleted file mode 100644 index 7181342c89..0000000000 --- a/sendemail-appjoint/sendemail-core/src/main/scala/com/webank/wedatasphere/dss/appjoint/sendemail/emailcontent/parser/FileEmailContentParser.scala +++ /dev/null @@ -1,44 +0,0 @@ -/* - * Copyright 2019 WeBank - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package com.webank.wedatasphere.dss.appjoint.sendemail.emailcontent.parser - -import com.webank.wedatasphere.dss.appjoint.sendemail.email.MultiContentEmail -import com.webank.wedatasphere.dss.appjoint.sendemail.emailcontent.FileEmailContent -import com.webank.wedatasphere.linkis.common.utils.Utils -import com.webank.wedatasphere.linkis.storage.LineRecord -import org.apache.commons.io.IOUtils -import org.springframework.stereotype.Component - -/** - * Created by shanhuang on 2019/10/12. - */ - -object FileEmailContentParser extends AbstractEmailContentParser[FileEmailContent] { - override protected def parseEmailContent(emailContent: FileEmailContent, - multiContentEmail: MultiContentEmail): Unit = { - val reader = getResultSetReader(emailContent) - val content = new StringBuilder - Utils.tryFinally{ - while(reader.hasNext) reader.getRecord match { - case lineRecord: LineRecord => - content.append(lineRecord.getLine).append("
") - } - }(IOUtils.closeQuietly(reader)) - emailContent.setContent(content.toString()) - } -} \ No newline at end of file diff --git a/sendemail-appjoint/sendemail-core/src/main/scala/com/webank/wedatasphere/dss/appjoint/sendemail/emailcontent/parser/HtmlEmailContentParser.scala b/sendemail-appjoint/sendemail-core/src/main/scala/com/webank/wedatasphere/dss/appjoint/sendemail/emailcontent/parser/HtmlEmailContentParser.scala deleted file mode 100644 index 71ef0d0ecc..0000000000 --- a/sendemail-appjoint/sendemail-core/src/main/scala/com/webank/wedatasphere/dss/appjoint/sendemail/emailcontent/parser/HtmlEmailContentParser.scala +++ /dev/null @@ -1,33 +0,0 @@ -/* - * Copyright 2019 WeBank - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package com.webank.wedatasphere.dss.appjoint.sendemail.emailcontent.parser - -import com.webank.wedatasphere.dss.appjoint.sendemail.email.MultiContentEmail -import com.webank.wedatasphere.dss.appjoint.sendemail.emailcontent.HtmlEmailContent -import org.springframework.stereotype.Component - -/** - * Created by shanhuang on 2019/10/12. - */ - -object HtmlEmailContentParser extends AbstractEmailContentParser[HtmlEmailContent] { - override protected def parseEmailContent(emailContent: HtmlEmailContent, - multiContentEmail: MultiContentEmail): Unit = { - getFirstLineRecord(emailContent).foreach(emailContent.setContent) - } -} diff --git a/sendemail-appjoint/sendemail-core/src/main/scala/com/webank/wedatasphere/dss/appjoint/sendemail/emailcontent/parser/PictureEmailContentParser.scala b/sendemail-appjoint/sendemail-core/src/main/scala/com/webank/wedatasphere/dss/appjoint/sendemail/emailcontent/parser/PictureEmailContentParser.scala deleted file mode 100644 index 595c417ac6..0000000000 --- a/sendemail-appjoint/sendemail-core/src/main/scala/com/webank/wedatasphere/dss/appjoint/sendemail/emailcontent/parser/PictureEmailContentParser.scala +++ /dev/null @@ -1,74 +0,0 @@ -/* - * Copyright 2019 WeBank - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package com.webank.wedatasphere.dss.appjoint.sendemail.emailcontent.parser - -import java.awt.image.BufferedImage -import java.io.{ByteArrayInputStream, ByteArrayOutputStream, File} -import java.util.{Base64, UUID} - -import javax.imageio.ImageIO -import com.webank.wedatasphere.dss.appjoint.sendemail.conf.SendEmailAppJointConfiguration.EMAIL_IMAGE_HEIGHT -import com.webank.wedatasphere.dss.appjoint.sendemail.email.{AbstractEmail, MultiContentEmail, PngAttachment} -import com.webank.wedatasphere.dss.appjoint.sendemail.emailcontent.PictureEmailContent -import com.webank.wedatasphere.linkis.common.conf.Configuration -import org.apache.commons.codec.binary.Base64OutputStream -import org.springframework.stereotype.Component - -/** - * Created by shanhuang on 2019/10/12. - */ - -object PictureEmailContentParser extends AbstractEmailContentParser[PictureEmailContent] { - - override protected def parseEmailContent(emailContent: PictureEmailContent, - multiContentEmail: MultiContentEmail): Unit = { - getFirstLineRecord(emailContent).foreach { imageStr => - val decoder = Base64.getDecoder - val byteArr = decoder.decode(imageStr) - val inputStream = new ByteArrayInputStream(byteArr) - val image = ImageIO.read(inputStream) - val contents = generateImage(image, multiContentEmail) - emailContent.setContent(contents) -// val image = ImageIO.read(new ByteArrayInputStream(imageStr.getBytes(Configuration.BDP_ENCODING.getValue))) -// val contents = generateImage(image, multiContentEmail) -// emailContent.setContent(contents) - } - } - - protected def generateImage(bufferedImage: BufferedImage, email: AbstractEmail): Array[String] = { - val imageUUID = UUID.randomUUID.toString - val width = bufferedImage.getWidth - val height = bufferedImage.getHeight - val imagesCuts = if (height > EMAIL_IMAGE_HEIGHT.getValue) { - val numOfCut = Math.ceil(height.toDouble / EMAIL_IMAGE_HEIGHT.getValue).toInt - val realHeight = height / numOfCut - (0 until numOfCut).map(i => bufferedImage.getSubimage(0, i * realHeight, width, realHeight)).toArray - } else Array(bufferedImage) - imagesCuts.indices.map { index => - val image = imagesCuts(index) - val imageName = index + "_" + imageUUID - val os = new ByteArrayOutputStream - val b64Stream = new Base64OutputStream(os) - ImageIO.write(image, "png", b64Stream) - val b64 = os.toString(Configuration.BDP_ENCODING.getValue) - email.addAttachment(new PngAttachment(imageName, b64)) - s"""""" - }.toArray - } - -} diff --git a/sendemail-appjoint/sendemail-core/src/main/scala/com/webank/wedatasphere/dss/appjoint/sendemail/emailcontent/parser/TableEmailContentParser.scala b/sendemail-appjoint/sendemail-core/src/main/scala/com/webank/wedatasphere/dss/appjoint/sendemail/emailcontent/parser/TableEmailContentParser.scala deleted file mode 100644 index 3b832185e3..0000000000 --- a/sendemail-appjoint/sendemail-core/src/main/scala/com/webank/wedatasphere/dss/appjoint/sendemail/emailcontent/parser/TableEmailContentParser.scala +++ /dev/null @@ -1,69 +0,0 @@ -/* - * Copyright 2019 WeBank - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package com.webank.wedatasphere.dss.appjoint.sendemail.emailcontent.parser - -import com.webank.wedatasphere.dss.appjoint.sendemail.email.MultiContentEmail -import com.webank.wedatasphere.dss.appjoint.sendemail.emailcontent.TableEmailContent -import com.webank.wedatasphere.linkis.common.utils.Utils -import com.webank.wedatasphere.linkis.storage.resultset.table.{TableMetaData, TableRecord} -import org.apache.commons.io.IOUtils -import org.apache.commons.lang.StringUtils -import org.springframework.stereotype.Component - -/** - * Created by shanhuang on 2019/10/12. - */ - -object TableEmailContentParser extends AbstractEmailContentParser[TableEmailContent] { - override protected def parseEmailContent(emailContent: TableEmailContent, - multiContentEmail: MultiContentEmail): Unit = { - val reader = getResultSetReader(emailContent) - val content = new StringBuilder - reader.getMetaData match { - case tableMetaData: TableMetaData => - writeTableTH(tableMetaData, content) - } - Utils.tryFinally { - while(reader.hasNext) reader.getRecord match { - case tableRecord: TableRecord => - writeTableTR(tableRecord, content) - } - }(IOUtils.closeQuietly(reader)) - emailContent.setContent(content.toString()) - } - - protected def writeTableTH(tableMetaData: TableMetaData, content: StringBuilder): Unit = { - content.append("") - tableMetaData.columns.foreach{ c => - content.append("").append(c.columnName) - if(StringUtils.isNotBlank(c.comment)) { - val comment = if(c.comment.length < 10) c.comment else c.comment.substring(0, 9) + "..." - content.append("(").append(comment).append(")") - } - content.append("") - } - content.append("") - } - - protected def writeTableTR(tableRecord: TableRecord, content: StringBuilder): Unit = { - content.append("") - tableRecord.row.foreach(v => content.append("").append(v).append("")) - content.append("") - } - -} diff --git a/visualis-appjoint/appjoint/pom.xml b/visualis-appjoint/appjoint/pom.xml deleted file mode 100644 index e572638daf..0000000000 --- a/visualis-appjoint/appjoint/pom.xml +++ /dev/null @@ -1,130 +0,0 @@ - - - - - - dss - com.webank.wedatasphere.dss - 0.9.1 - - 4.0.0 - - dss-visualis-appjoint - - - - com.webank.wedatasphere.dss - dss-appjoint-core - ${dss.version} - provided - true - - - - com.webank.wedatasphere.linkis - linkis-module - ${linkis.version} - provided - true - - - - com.webank.wedatasphere.linkis - linkis-httpclient - ${linkis.version} - provided - true - - - - net.databinder.dispatch - dispatch-core_2.11 - 0.12.3 - - - - - - - - - org.apache.maven.plugins - maven-deploy-plugin - - - - net.alchim31.maven - scala-maven-plugin - - - org.apache.maven.plugins - maven-jar-plugin - - - org.apache.maven.plugins - maven-assembly-plugin - 2.3 - false - - - make-assembly - package - - single - - - - src/main/assembly/distribution.xml - - - - - - false - dss-visualis-appjoint - false - false - - src/main/assembly/distribution.xml - - - - - - - src/main/java - - **/*.xml - - - - src/main/resources - - **/*.properties - **/application.yml - **/bootstrap.yml - **/log4j2.xml - - - - - - - \ No newline at end of file diff --git a/visualis-appjoint/appjoint/src/main/assembly/distribution.xml b/visualis-appjoint/appjoint/src/main/assembly/distribution.xml deleted file mode 100644 index 0d56a79cf8..0000000000 --- a/visualis-appjoint/appjoint/src/main/assembly/distribution.xml +++ /dev/null @@ -1,226 +0,0 @@ - - - - dss-visualis-appjoint - - zip - - true - visualis - - - - - - lib - true - true - false - true - true - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - ${basedir}/conf - - * - - 0777 - conf - unix - - - ${basedir}/bin - - * - - 0777 - bin - unix - - - . - - */** - - logs - - - - - diff --git a/visualis-appjoint/appjoint/src/main/java/com/webank/wedatasphere/dss/appjoint/visualis/VisualisAppJoint.java b/visualis-appjoint/appjoint/src/main/java/com/webank/wedatasphere/dss/appjoint/visualis/VisualisAppJoint.java deleted file mode 100644 index 4dff6a3f11..0000000000 --- a/visualis-appjoint/appjoint/src/main/java/com/webank/wedatasphere/dss/appjoint/visualis/VisualisAppJoint.java +++ /dev/null @@ -1,71 +0,0 @@ -/* - * Copyright 2019 WeBank - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package com.webank.wedatasphere.dss.appjoint.visualis; - -import com.webank.wedatasphere.dss.appjoint.AppJoint; -import com.webank.wedatasphere.dss.appjoint.execution.NodeExecution; -import com.webank.wedatasphere.dss.appjoint.service.AppJointUrlImpl; -import com.webank.wedatasphere.dss.appjoint.service.NodeService; -import com.webank.wedatasphere.dss.appjoint.service.SecurityService; -import com.webank.wedatasphere.dss.appjoint.visualis.service.VisualisNodeService; -import com.webank.wedatasphere.dss.appjoint.visualis.service.VisualisSecurityService; -import com.webank.wedatasphere.dss.appjoint.visualis.execution.VisualisNodeExecution; - -import java.util.Map; - -/** - * Created by enjoyyin on 2019/11/6. - */ -public class VisualisAppJoint extends AppJointUrlImpl implements AppJoint { - - private SecurityService securityService; - - private NodeExecution nodeExecution; - - private NodeService nodeService; - - @Override - public String getAppJointName() { - return "visualis"; - } - - @Override - public void init(String baseUrl, Map params) { - securityService = new VisualisSecurityService(); - securityService.setBaseUrl(baseUrl); - nodeExecution = new VisualisNodeExecution(); - nodeExecution.setBaseUrl(baseUrl); - nodeService = new VisualisNodeService(); - nodeService.setBaseUrl(baseUrl); - } - - @Override - public SecurityService getSecurityService() { - return securityService; - } - - @Override - public NodeService getNodeService() { - return nodeService; - } - - @Override - public NodeExecution getNodeExecution() { - return nodeExecution; - } -} diff --git a/visualis-appjoint/appjoint/src/main/java/com/webank/wedatasphere/dss/appjoint/visualis/service/VisualisNodeService.java b/visualis-appjoint/appjoint/src/main/java/com/webank/wedatasphere/dss/appjoint/visualis/service/VisualisNodeService.java deleted file mode 100644 index ac2c5aa8bd..0000000000 --- a/visualis-appjoint/appjoint/src/main/java/com/webank/wedatasphere/dss/appjoint/visualis/service/VisualisNodeService.java +++ /dev/null @@ -1,70 +0,0 @@ -/* - * Copyright 2019 WeBank - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package com.webank.wedatasphere.dss.appjoint.visualis.service; - - -import com.webank.wedatasphere.dss.appjoint.exception.AppJointErrorException; -import com.webank.wedatasphere.dss.appjoint.execution.core.AppJointNode; -import com.webank.wedatasphere.dss.appjoint.service.AppJointUrlImpl; -import com.webank.wedatasphere.dss.appjoint.service.NodeService; -import com.webank.wedatasphere.dss.appjoint.service.session.Session; -import com.webank.wedatasphere.dss.appjoint.visualis.service.nodeservice.DashboardNodeService; -import com.webank.wedatasphere.dss.appjoint.visualis.service.nodeservice.DisplayNodeService; - -import java.util.Map; - -/** - * Created by enjoyyin on 2019/11/5. - */ -public class VisualisNodeService extends AppJointUrlImpl implements NodeService { - - @Override - public Map createNode(Session session, AppJointNode node, - Map requestBody) throws AppJointErrorException { - if (DisplayNodeService.getNodeType().equals(node.getNodeType())) { - return DisplayNodeService.createNode(session, getBaseUrl(), String.valueOf(node.getProjectId()), node.getNodeType(), requestBody); - } else if (DashboardNodeService.getNodeType().equals(node.getNodeType())) { - return DashboardNodeService.createNode(session, getBaseUrl(), String.valueOf(node.getProjectId()), node.getNodeType(), requestBody); - } else { - throw new AppJointErrorException(42002, "cannot recognize the nodeType " + node.getNodeType()); - } - } - - @Override - public void deleteNode(Session session, AppJointNode node) throws AppJointErrorException { - if (DisplayNodeService.getNodeType().equals(node.getNodeType())) { - DisplayNodeService.deleteNode(session, getBaseUrl(), String.valueOf(node.getProjectId()), node.getNodeType(), node.getJobContent()); - } else if (DashboardNodeService.getNodeType().equals(node.getNodeType())) { - DashboardNodeService.deleteNode(session, getBaseUrl(), String.valueOf(node.getProjectId()), node.getNodeType(), node.getJobContent()); - } else { - throw new AppJointErrorException(42002, "cannot recognize the nodeType " + node.getNodeType()); - } - } - - @Override - public Map updateNode(Session session, AppJointNode node, - Map requestBody) throws AppJointErrorException { - if (DisplayNodeService.getNodeType().equals(node.getNodeType())) { - return DisplayNodeService.updateNode(session, getBaseUrl(), node.getProjectId(), node.getNodeType(), requestBody); - } else if (DashboardNodeService.getNodeType().equals(node.getNodeType())) { - return DashboardNodeService.updateNode(session, getBaseUrl(), node.getProjectId(), node.getNodeType(), requestBody); - } else { - throw new AppJointErrorException(42002, "cannot recognize the nodeType " + node.getNodeType()); - } - } -} \ No newline at end of file diff --git a/visualis-appjoint/appjoint/src/main/java/com/webank/wedatasphere/dss/appjoint/visualis/service/VisualisSecurityService.java b/visualis-appjoint/appjoint/src/main/java/com/webank/wedatasphere/dss/appjoint/visualis/service/VisualisSecurityService.java deleted file mode 100644 index 8b970d9cba..0000000000 --- a/visualis-appjoint/appjoint/src/main/java/com/webank/wedatasphere/dss/appjoint/visualis/service/VisualisSecurityService.java +++ /dev/null @@ -1,42 +0,0 @@ -/* - * Copyright 2019 WeBank - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package com.webank.wedatasphere.dss.appjoint.visualis.service; - -import com.webank.wedatasphere.dss.appjoint.exception.AppJointErrorException; -import com.webank.wedatasphere.dss.appjoint.service.AppJointUrlImpl; -import com.webank.wedatasphere.dss.appjoint.service.SecurityService; -import com.webank.wedatasphere.dss.appjoint.service.session.Session; - -/** - * Created by shanhuang on 2019/10/12. - */ -public class VisualisSecurityService extends AppJointUrlImpl implements SecurityService { - @Override - public Session login(String user) throws AppJointErrorException { - VisualisSession visualisSession = new VisualisSession(); - visualisSession.setUser(user); - visualisSession.getParameters().put("Token-User",user); - visualisSession.getParameters().put("Token-Code","WS-AUTH"); - return visualisSession; - } - - @Override - public void logout(String user) { - - } -} diff --git a/visualis-appjoint/appjoint/src/main/java/com/webank/wedatasphere/dss/appjoint/visualis/service/VisualisSession.java b/visualis-appjoint/appjoint/src/main/java/com/webank/wedatasphere/dss/appjoint/visualis/service/VisualisSession.java deleted file mode 100644 index ae06d1c8ff..0000000000 --- a/visualis-appjoint/appjoint/src/main/java/com/webank/wedatasphere/dss/appjoint/visualis/service/VisualisSession.java +++ /dev/null @@ -1,74 +0,0 @@ -/* - * Copyright 2019 WeBank - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package com.webank.wedatasphere.dss.appjoint.visualis.service; - -import com.webank.wedatasphere.dss.appjoint.service.session.Session; -import org.apache.http.Header; -import org.apache.http.cookie.Cookie; - -import java.util.HashMap; -import java.util.Map; - -/** - * Created by shanhuang on 2019/10/12. - */ -public class VisualisSession implements Session { - private long lastAccessTime; - private Cookie[] cookies; - private Header[] headers; - private String user; - private Map parameters = new HashMap<>(); - - @Override - public String getUser() { - return user; - } - - public void setUser(String user) { - this.user = user; - } - - @Override - public Cookie[] getCookies() { - return this.cookies; - } - - @Override - public Header[] getHeaders() { - return this.headers; - } - - @Override - public Map getParameters() { - return this.parameters; - } - - @Override - public long getLastAccessTime() { - return this.lastAccessTime; - } - - public void setCookies(Cookie[] cookies){ - this.cookies = cookies; - } - - @Override - public void updateLastAccessTime() { - lastAccessTime = System.currentTimeMillis(); - } -} diff --git a/visualis-appjoint/appjoint/src/main/java/com/webank/wedatasphere/dss/appjoint/visualis/service/nodeservice/DashboardNodeService.java b/visualis-appjoint/appjoint/src/main/java/com/webank/wedatasphere/dss/appjoint/visualis/service/nodeservice/DashboardNodeService.java deleted file mode 100644 index 9f2b42efb4..0000000000 --- a/visualis-appjoint/appjoint/src/main/java/com/webank/wedatasphere/dss/appjoint/visualis/service/nodeservice/DashboardNodeService.java +++ /dev/null @@ -1,151 +0,0 @@ -/* - * Copyright 2019 WeBank - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package com.webank.wedatasphere.dss.appjoint.visualis.service.nodeservice; - -import com.fasterxml.jackson.core.JsonProcessingException; -import com.google.gson.Gson; -import com.google.gson.JsonElement; -import com.google.gson.JsonObject; -import com.webank.wedatasphere.dss.appjoint.exception.AppJointErrorException; -import com.webank.wedatasphere.dss.appjoint.service.session.Session; -import com.webank.wedatasphere.linkis.server.BDPJettyServerHelper; -import com.webank.wedatasphere.linkis.server.conf.ServerConfiguration; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.IOException; -import java.util.HashMap; -import java.util.Map; -/** - * Created by shanhuang on 2019/10/12. - */ -public class DashboardNodeService { - private final static String dashboardUrl = "/api/rest_s/" + ServerConfiguration.BDP_SERVER_VERSION() + "/visualis/dashboardPortals"; - private final static Logger logger = LoggerFactory.getLogger(DashboardNodeService.class); - - public static Map createNode(Session session, String url, String projectId, String nodeType, - Map requestBody) throws AppJointErrorException{ - Map element; - try { - HttpUtils httpUtils = new HttpUtils(); - requestBody.put("projectId", Integer.valueOf(projectId)); - String jsonParams = BDPJettyServerHelper.jacksonJson().writeValueAsString(requestBody); - String resultString = httpUtils.sendPostReq(session, url + dashboardUrl, jsonParams, session.getUser()); - Map jsonObject = BDPJettyServerHelper.jacksonJson().readValue(resultString, Map.class); - Map header = (Map) jsonObject.get("header"); - int code = (int) header.get("code"); - if (code != 200) { - String errorMsg = header.toString(); - throw new AppJointErrorException(code, errorMsg); - } - int dashboardId = (int) ((Map) jsonObject.get("payload")).get("id"); - Map reqMap = new HashMap<>(); - reqMap.put("config", ""); - reqMap.put("dashboardPortalId", dashboardId); - reqMap.put("index", 0); - reqMap.put("name", requestBody.get("name").toString()); - reqMap.put("parentId", 0); - reqMap.put("type", 1); - String reqJsonObject = BDPJettyServerHelper.jacksonJson().writeValueAsString(reqMap); - String result = httpUtils.sendPostReq(session, url + dashboardUrl + "/" + dashboardId + "/dashboards", reqJsonObject, session.getUser()); - element = BDPJettyServerHelper.jacksonJson().readValue(result, Map.class); - Map header1 = (Map) element.get("header"); - code = (int) header1.get("code"); - if (code != 200) { - String errorMsg = header1.toString(); - throw new AppJointErrorException(code, errorMsg); - } - element.put("dashboardPortalId", dashboardId); - }catch (Exception ex){ - throw new AppJointErrorException(90155,"Update Display AppJointNode Exception",ex); - } - - return element; - } - - public static Map deleteNode(Session session, String url, String projectId, String nodeType, - Map requestBody) throws AppJointErrorException{ - Map element; - try { - HttpUtils httpUtils = new HttpUtils(); - requestBody.put("projectId", Integer.valueOf(projectId)); - logger.info("DashboardNodeServiceImpl request params is " + requestBody + ",nodeType:" + nodeType); - logger.info("Dashboard url is " + url); - String nodeId = requestBody.get("id").toString(); - String resultString = httpUtils.sendHttpDelete(session, url + dashboardUrl + "/" + nodeId, session.getUser()); - element = BDPJettyServerHelper.jacksonJson().readValue(resultString, Map.class); - Map header = (Map) element.get("header"); - int code = (int) header.get("code"); - if (code != 200) { - String errorMsg = header.toString(); - throw new AppJointErrorException(code, errorMsg); - } - }catch (Exception ex){ - throw new AppJointErrorException(90156,"Update Display AppJointNode Exception",ex); - } - return element; - } - - public static Map updateNode(Session session, String url, Long projectId, String nodeType, - Map requestBody) throws AppJointErrorException{ - Map element; - try { - HttpUtils httpUtils = new HttpUtils(); - requestBody.put("projectId", projectId.intValue()); - String jsonParams = BDPJettyServerHelper.jacksonJson().writeValueAsString(requestBody); - logger.info("updateNode request params is " + jsonParams + ",nodeType:" + nodeType); - logger.info("Dashboard url is " + url); - String dashboardPortalId = requestBody.get("dashboardPortalId").toString(); - String resultString = httpUtils.sendHttpPut(session, url + dashboardUrl + "/" + dashboardPortalId + "/dashboards", jsonParams, session.getUser()); - element = BDPJettyServerHelper.jacksonJson().readValue(resultString, Map.class); - Map header = (Map) element.get("header"); - int code = (int) header.get("code"); - if (code != 200) { - String errorMsg = header.toString(); - throw new AppJointErrorException(code, errorMsg); - } -// int dashboardId = jsonObject.get("payload").getAsJsonObject().get("id").getAsInt(); - Map reqMap = new HashMap<>(); - reqMap.put("avatar", "9"); - reqMap.put("description", ""); - reqMap.put("id", dashboardPortalId); - reqMap.put("name", requestBody.get("name").toString()); - reqMap.put("publish", true); - reqMap.put("roleIds", "[]"); - String reqJson = BDPJettyServerHelper.jacksonJson().writeValueAsString(reqMap); - logger.info("updateNode request slides params is: " + reqJson); - String result = httpUtils.sendHttpPut(session, url + dashboardUrl + "/" + dashboardPortalId, reqJson, session.getUser()); - element = BDPJettyServerHelper.jacksonJson().readValue(result, Map.class); - header = (Map) element.get("header"); - code = (int) header.get("code"); - if (code != 200) { - String errorMsg = header.toString(); - throw new AppJointErrorException(code, errorMsg); - } - element.put("dashboardPortalId", dashboardPortalId); - }catch (Exception ex){ - throw new AppJointErrorException(90157,"Update Display AppJointNode Exception",ex); - } - - return element; - } - - public static String getNodeType() { - return "linkis.appjoint.visualis.dashboard"; - } -} diff --git a/visualis-appjoint/appjoint/src/main/java/com/webank/wedatasphere/dss/appjoint/visualis/service/nodeservice/DisplayNodeService.java b/visualis-appjoint/appjoint/src/main/java/com/webank/wedatasphere/dss/appjoint/visualis/service/nodeservice/DisplayNodeService.java deleted file mode 100644 index 1f21dd3f62..0000000000 --- a/visualis-appjoint/appjoint/src/main/java/com/webank/wedatasphere/dss/appjoint/visualis/service/nodeservice/DisplayNodeService.java +++ /dev/null @@ -1,125 +0,0 @@ -/* - * Copyright 2019 WeBank - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package com.webank.wedatasphere.dss.appjoint.visualis.service.nodeservice; - - -import com.webank.wedatasphere.dss.appjoint.exception.AppJointErrorException; -import com.webank.wedatasphere.dss.appjoint.service.session.Session; -import com.webank.wedatasphere.linkis.server.BDPJettyServerHelper; -import com.webank.wedatasphere.linkis.server.conf.ServerConfiguration; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import java.util.HashMap; -import java.util.Map; -/** - * Created by shanhuang on 2019/10/12. - */ -public class DisplayNodeService { - private final static String displayUrl = "/api/rest_s/" + ServerConfiguration.BDP_SERVER_VERSION() + "/visualis/displays"; - private final static String config = "{\"slideParams\":{\"width\":1920,\"height\":1080,\"backgroundColor\":[255,255,255],\"scaleMode\":\"noScale\",\"backgroundImage\":null}}"; - private final static Logger logger = LoggerFactory.getLogger(DisplayNodeService.class); - - public static Map createNode(Session session, String url, String projectId, - String nodeType, Map requestBody) throws AppJointErrorException { - Map element; - try { - requestBody.put("projectId", Integer.valueOf(projectId)); - String jsonParams = BDPJettyServerHelper.jacksonJson().writeValueAsString(requestBody); - logger.info("DisplayNodeServiceImpl request params is " + jsonParams + ",nodeType:" + nodeType); - String displayRes = HttpUtils.sendPostReq(session, url + displayUrl, jsonParams, session.getUser()); - logger.info("Create Display AppJointNode First Return Value: " + displayRes); - Map jsonObject = BDPJettyServerHelper.jacksonJson().readValue(displayRes, Map.class); - Map header = (Map) jsonObject.get("header"); - int code = (int) header.get("code"); - if (code != 200) { - String errorMsg = header.toString(); - throw new AppJointErrorException(code, errorMsg); - } - int displayId = (int) ((Map) jsonObject.get("payload")).get("id"); - Map reqMap = new HashMap<>(); - reqMap.put("config", config); - reqMap.put("displayId", displayId); - reqMap.put("index", 0); - String reqJsonObject = BDPJettyServerHelper.jacksonJson().writeValueAsString(reqMap); - logger.info("request slides params is: " + reqJsonObject); - String result = HttpUtils.sendPostReq(session, url + displayUrl + "/" + displayId + "/slides", reqJsonObject, session.getUser()); - element = BDPJettyServerHelper.jacksonJson().readValue(result, Map.class); - header = (Map) element.get("header"); - code = (int) header.get("code"); - if (code != 200) { - String errorMsg = header.toString(); - throw new AppJointErrorException(code, errorMsg); - } - - }catch (Exception ex){ - throw new AppJointErrorException(90173,"Create Display AppJointNode Exception",ex); - } - - return element; - - } - - - public static Map deleteNode(Session session, String url, String projectId, - String nodeType, Map requestBody) throws AppJointErrorException{ - Map element; - try { - requestBody.put("projectId", Integer.valueOf(projectId)); - logger.info("request params is " + requestBody + ",nodeType:" + nodeType); - logger.info("Delete display url is " + url); - String nodeId = requestBody.get("id").toString(); - String resultString = HttpUtils.sendHttpDelete(session, url + displayUrl + "/" + nodeId, session.getUser()); - element = BDPJettyServerHelper.jacksonJson().readValue(resultString, Map.class); - Map header = (Map) element.get("header"); - int code = (int) header.get("code"); - if (code != 200) { - String errorMsg = header.toString(); - throw new AppJointErrorException(code, errorMsg); - } - }catch (Exception ex){ - throw new AppJointErrorException(90174,"Delete Display AppJointNode Exception",ex); - } - return element; - } - - - public static Map updateNode(Session session, String url, Long projectId, - String nodeType, Map requestBody) throws AppJointErrorException{ - Map element; - try { - logger.info("[updateNode]: request projectId is {}, nodeType is {}, params: {}.", projectId, nodeType, requestBody); - String nodeId = requestBody.get("id").toString(); - String jsonParams = BDPJettyServerHelper.jacksonJson().writeValueAsString(requestBody); - String displayRes = HttpUtils.sendHttpPut(session, url + displayUrl + "/" + nodeId, jsonParams, session.getUser()); - element = BDPJettyServerHelper.jacksonJson().readValue(displayRes, Map.class); - Map header = (Map) element.get("header"); - int code = (int) header.get("code"); - if (code != 200) { - String errorMsg = header.toString(); - throw new AppJointErrorException(code, errorMsg); - } - }catch (Exception ex){ - throw new AppJointErrorException(90175,"Update Display AppJointNode Exception",ex); - } - return element; - } - - public static String getNodeType() { - return "linkis.appjoint.visualis.display"; - } -} diff --git a/visualis-appjoint/appjoint/src/main/java/com/webank/wedatasphere/dss/appjoint/visualis/service/nodeservice/HttpUtils.java b/visualis-appjoint/appjoint/src/main/java/com/webank/wedatasphere/dss/appjoint/visualis/service/nodeservice/HttpUtils.java deleted file mode 100644 index 4a030dc681..0000000000 --- a/visualis-appjoint/appjoint/src/main/java/com/webank/wedatasphere/dss/appjoint/visualis/service/nodeservice/HttpUtils.java +++ /dev/null @@ -1,153 +0,0 @@ -/* - * Copyright 2019 WeBank - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package com.webank.wedatasphere.dss.appjoint.visualis.service.nodeservice; - -import com.webank.wedatasphere.dss.appjoint.exception.AppJointErrorException; -import com.webank.wedatasphere.dss.appjoint.service.session.Session; -import com.webank.wedatasphere.dss.appjoint.visualis.service.VisualisSession; -import org.apache.commons.io.IOUtils; -import org.apache.http.HttpEntity; -import org.apache.http.client.CookieStore; -import org.apache.http.client.methods.CloseableHttpResponse; -import org.apache.http.client.methods.HttpDelete; -import org.apache.http.client.methods.HttpPost; -import org.apache.http.client.methods.HttpPut; -import org.apache.http.entity.StringEntity; -import org.apache.http.impl.client.BasicCookieStore; -import org.apache.http.impl.client.CloseableHttpClient; -import org.apache.http.impl.client.HttpClients; -import org.apache.http.protocol.HTTP; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.UnsupportedEncodingException; -/** - * Created by shanhuang on 2019/10/12. - */ -public class HttpUtils { - private static Logger logger = LoggerFactory.getLogger(HttpUtils.class); - public static String sendPostReq(Session session, String url, String params, - String user) throws AppJointErrorException { - String resultString = "{}"; - VisualisSession visualisSession = (VisualisSession)session; - String token = visualisSession.getParameters().get("Token-Code"); - logger.info("sendPostReq url is: "+url+",session:"+token ); - HttpPost httpPost = new HttpPost(url); - httpPost.addHeader(HTTP.CONTENT_ENCODING, "UTF-8"); - httpPost.addHeader("Token-User",user); - httpPost.addHeader("Token-Code",token); - CookieStore cookieStore = new BasicCookieStore(); -// cookieStore.addCookie(session.getCookies()[0]); - logger.info("Http request params is :"+params); - StringEntity entity = null; - try { - entity = new StringEntity(params); - } catch (UnsupportedEncodingException e) { - throw new AppJointErrorException(42000, "create StringEntity failed!", e); - } - entity.setContentEncoding("UTF-8"); - entity.setContentType("application/json");//发送json数据需要设置contentType - httpPost.setEntity(entity); - CloseableHttpClient httpClient = null; - CloseableHttpResponse response = null; - try { - httpClient = HttpClients.custom().setDefaultCookieStore(cookieStore).build(); - response = httpClient.execute(httpPost); - HttpEntity ent = response.getEntity(); - resultString = IOUtils.toString(ent.getContent(), "utf-8"); - logger.info("Send Http Request Success", resultString); - } catch (Exception e) { - logger.error("Send Http Request Failed", e); - throw new AppJointErrorException(42000, e.getMessage(), e); - } finally { - IOUtils.closeQuietly(response); - IOUtils.closeQuietly(httpClient); - } - return resultString; - } - - public static String sendHttpDelete(Session session,String url,String user) throws AppJointErrorException { - String resultString = "{}"; - HttpDelete httpdelete = new HttpDelete(url); - //设置header - VisualisSession visualisSession = (VisualisSession)session; - String token = visualisSession.getParameters().get("Token-Code"); - logger.info("sendDeleteReq url is: "+url+",session:"+token ); - httpdelete.addHeader(HTTP.CONTENT_ENCODING, "UTF-8"); - httpdelete.addHeader("Token-User",user); - httpdelete.addHeader("Token-Code",token); - CookieStore cookieStore = new BasicCookieStore(); - CloseableHttpClient httpClient = null; - CloseableHttpResponse response = null; - try { - httpClient = HttpClients.custom().setDefaultCookieStore(cookieStore).build(); - response = httpClient.execute(httpdelete); - HttpEntity ent = response.getEntity(); - resultString = IOUtils.toString(ent.getContent(), "utf-8"); - logger.info("Send Http Delete Request Success", resultString); - } catch (Exception e) { - logger.error("Send Http Delete Request Failed", e); - throw new AppJointErrorException(42001, e.getMessage(), e); - }finally{ - IOUtils.closeQuietly(response); - IOUtils.closeQuietly(httpClient); - } - return resultString; - } - - public static String sendHttpPut(Session session, String url, String params, - String user) throws AppJointErrorException { - String resultString = "{}"; - VisualisSession visualisSession = (VisualisSession)session; - String token = visualisSession.getParameters().get("Token-Code"); - logger.info("sendPostReq url is: "+url+",session:"+token ); - HttpPut httpPut = new HttpPut(url); - httpPut.addHeader(HTTP.CONTENT_ENCODING, "UTF-8"); - httpPut.addHeader("Token-User",user); - httpPut.addHeader("Token-Code",token); - CookieStore cookieStore = new BasicCookieStore(); -// cookieStore.addCookie(session.getCookies()[0]); - logger.info("Http put params is :"+params); - StringEntity entity = null; - try { - entity = new StringEntity(params); - } catch (UnsupportedEncodingException e) { - throw new AppJointErrorException(42000, "create StringEntity failed!", e); - } - entity.setContentEncoding("UTF-8"); - entity.setContentType("application/json");//发送json数据需要设置contentType - httpPut.setEntity(entity); - CloseableHttpClient httpClient = null; - CloseableHttpResponse response = null; - try { - httpClient = HttpClients.custom().setDefaultCookieStore(cookieStore).build(); - response = httpClient.execute(httpPut); - HttpEntity ent = response.getEntity(); - resultString = IOUtils.toString(ent.getContent(), "utf-8"); - logger.info("Send Http Put Success", resultString); - } catch (Exception e) { - logger.error("Send Http Put Failed", e); - throw new AppJointErrorException(42000, e.getMessage(), e); - } finally { - IOUtils.closeQuietly(response); - IOUtils.closeQuietly(httpClient); - } - return resultString; - } - -} diff --git a/visualis-appjoint/appjoint/src/main/scala/com/webank/wedatasphere/dss/appjoint/visualis/execution/VisualisNodeExecution.scala b/visualis-appjoint/appjoint/src/main/scala/com/webank/wedatasphere/dss/appjoint/visualis/execution/VisualisNodeExecution.scala deleted file mode 100644 index fa25757263..0000000000 --- a/visualis-appjoint/appjoint/src/main/scala/com/webank/wedatasphere/dss/appjoint/visualis/execution/VisualisNodeExecution.scala +++ /dev/null @@ -1,142 +0,0 @@ -/* - * Copyright 2019 WeBank - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package com.webank.wedatasphere.dss.appjoint.visualis.execution - -import java.io.{ByteArrayOutputStream, InputStream} -import java.util -import java.util.Base64 - -import com.webank.wedatasphere.dss.appjoint.execution.NodeExecution -import com.webank.wedatasphere.dss.appjoint.execution.common.{CompletedNodeExecutionResponse, NodeExecutionResponse} -import com.webank.wedatasphere.dss.appjoint.execution.core.{AppJointNode, CommonAppJointNode, NodeContext} -import com.webank.wedatasphere.dss.appjoint.service.session.Session -import com.webank.wedatasphere.dss.appjoint.visualis.execution.VisualisNodeExecutionConfiguration._ -import com.webank.wedatasphere.linkis.common.exception.ErrorException -import com.webank.wedatasphere.linkis.common.log.LogUtils -import com.webank.wedatasphere.linkis.common.utils.{Logging, Utils} -import com.webank.wedatasphere.linkis.storage.{LineMetaData, LineRecord} -import org.apache.commons.io.IOUtils - -import scala.collection.JavaConversions.mapAsScalaMap -import scala.concurrent.duration.Duration -import scala.concurrent.{Await, ExecutionContext} -import dispatch._ -import org.json4s.{DefaultFormats, Formats} - -/** - * Created by enjoyyin on 2019/10/12. - */ -class VisualisNodeExecution extends NodeExecution with Logging { - - private val DISPLAY = "display" - private val DASHBOARD = "dashboard" - - var basicUrl:String = _ - - protected implicit val executors: ExecutionContext = Utils.newCachedExecutionContext(VISUALIS_THREAD_MAX.getValue, getName + "-NodeExecution-Thread", true) - protected implicit val formats: Formats = DefaultFormats - - private implicit def svc(url: String): Req = - dispatch.url(url) - - - - override def getBaseUrl: String = this.basicUrl - - override def setBaseUrl(basicUrl: String): Unit = this.basicUrl = basicUrl - - def getName:String = "visualis" - - override def canExecute(node: AppJointNode, nodeContext: NodeContext, session: Session): Boolean = node.getNodeType.toLowerCase.contains(getName.toLowerCase) - - override def execute(node: AppJointNode, nodeContext: NodeContext, session: Session): NodeExecutionResponse = node match { - case commonAppJointNode: CommonAppJointNode => - val appJointResponse = new CompletedNodeExecutionResponse() - val idMap = commonAppJointNode.getJobContent - val id = idMap.values().iterator().next().toString - val url = if(commonAppJointNode.getNodeType.toLowerCase.contains(DISPLAY)) getDisplayPreviewUrl(nodeContext.getGatewayUrl, id) - else if(commonAppJointNode.getNodeType.toLowerCase.contains(DASHBOARD)) getDashboardPreviewUrl(nodeContext.getGatewayUrl, id) - else { - appJointResponse.setIsSucceed(false) - appJointResponse.setErrorMsg("不支持的appJoint类型：" + node.getNodeType) - return appJointResponse - } - var response = "" - val headers = nodeContext.getTokenHeader(nodeContext.getUser) - nodeContext.appendLog(LogUtils.generateInfo(s"Ready to download preview picture from $url.")) - Utils.tryCatch(download(url, null, headers.toMap, - input => Utils.tryFinally{ - val os = new ByteArrayOutputStream() - IOUtils.copy(input, os) - response = new String(Base64.getEncoder.encode(os.toByteArray)) - //response = IOUtils.toString(input, ServerConfiguration.BDP_SERVER_ENCODING.getValue) - }(IOUtils.closeQuietly(input)))){ t => - val errException = new ErrorException(70063, "failed to do visualis request") - errException.initCause(t) - appJointResponse.setException(errException) - appJointResponse.setIsSucceed(false) - appJointResponse.setErrorMsg(s"用户${nodeContext.getUser}请求Visualis失败！URL为: " + url) - return appJointResponse - } - nodeContext.appendLog(LogUtils.generateInfo("Preview picture downloaded, now ready to write results.")) - val imagesBytes = response - val resultSetWriter = nodeContext.createPictureResultSetWriter() - Utils.tryFinally{ - resultSetWriter.addMetaData(new LineMetaData()) - resultSetWriter.addRecord(new LineRecord(imagesBytes)) - }(IOUtils.closeQuietly(resultSetWriter)) - appJointResponse.setIsSucceed(true) - appJointResponse - } - - def download(url: String, queryParams: Map[String, String], headerParams: Map[String, String], - write: InputStream => Unit, - paths: String*): Unit = { - var req = url.GET - if(headerParams != null && headerParams.nonEmpty) req = req <:< headerParams - if(queryParams != null) queryParams.foreach{ case (k, v) => req = req.addQueryParameter(k, v)} - if(paths != null) paths.filter(_ != null).foreach(p => req = req / p) - val response = Http(req OK as.Response(_.getResponseBodyAsStream)).map(write) - Await.result(response, Duration.Inf) - } - - private def getRealId(displayId:String):Int = { - Utils.tryCatch{ - val f = java.lang.Float.parseFloat(displayId) - f.asInstanceOf[Int] - }{ - _ => 0 - } - } - - def getPreviewUrl(gatewayUrl: String, uri: String, displayId: String): String = { - val realId = getRealId(displayId) - val realGatewayUrl = "http://" + gatewayUrl - if(!realGatewayUrl.endsWith("/") && !uri.startsWith("/")) - realGatewayUrl + "/" + String.format(uri, realId.toString) - else if(realGatewayUrl.endsWith("/") && uri.startsWith("/")) - realGatewayUrl.substring(0, realGatewayUrl.length - 1) + String.format(uri, realId.toString) - else realGatewayUrl + String.format(uri, realId.toString) - } - - def getDisplayPreviewUrl(gatewayUrl: String, displayId: String): String = getPreviewUrl(gatewayUrl, DISPLAY_PREVIEW_URL_FORMAT.getValue, displayId) - - def getDashboardPreviewUrl(gatewayUrl: String, dashboardId: String): String = getPreviewUrl(gatewayUrl, DASHBOARD_PREVIEW_URL_FORMAT.getValue, dashboardId) - - override def init(map: util.Map[String, AnyRef]): Unit = {} -} diff --git a/visualis-appjoint/appjoint/src/main/scala/com/webank/wedatasphere/dss/appjoint/visualis/execution/VisualisNodeExecutionConfiguration.scala b/visualis-appjoint/appjoint/src/main/scala/com/webank/wedatasphere/dss/appjoint/visualis/execution/VisualisNodeExecutionConfiguration.scala deleted file mode 100644 index 95c3648ea1..0000000000 --- a/visualis-appjoint/appjoint/src/main/scala/com/webank/wedatasphere/dss/appjoint/visualis/execution/VisualisNodeExecutionConfiguration.scala +++ /dev/null @@ -1,35 +0,0 @@ -/* - * Copyright 2019 WeBank - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package com.webank.wedatasphere.dss.appjoint.visualis.execution - -import com.webank.wedatasphere.linkis.common.conf.CommonVars -import com.webank.wedatasphere.linkis.server.conf.ServerConfiguration - -/** - * Created by enjoyyin on 2019/10/12. - */ -object VisualisNodeExecutionConfiguration { - - val DISPLAY_PREVIEW_URL_FORMAT = CommonVars("wds.dss.appjoint.visualis.display.preview.url", - "/api/rest_s/" + ServerConfiguration.BDP_SERVER_VERSION + "/visualis/displays/%s/preview") - val DASHBOARD_PREVIEW_URL_FORMAT = CommonVars("wds.dss.appjoint.visualis.dashboard.preview.url", - "/api/rest_s/" + ServerConfiguration.BDP_SERVER_VERSION + "/visualis/dashboard/%s/preview") - - val VISUALIS_THREAD_MAX = CommonVars("wds.dss.appjoint.visualis.thread.max", 20) - -} diff --git a/visualis-appjoint/appjoint/src/main/scala/com/webank/wedatasphere/linkis/server/JSONUtils.scala b/visualis-appjoint/appjoint/src/main/scala/com/webank/wedatasphere/linkis/server/JSONUtils.scala deleted file mode 100644 index c5b348ff0d..0000000000 --- a/visualis-appjoint/appjoint/src/main/scala/com/webank/wedatasphere/linkis/server/JSONUtils.scala +++ /dev/null @@ -1,27 +0,0 @@ -/* - * Copyright 2019 WeBank - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package com.webank.wedatasphere.linkis.server - -/** - * Created by enjoyyin on 2019/10/12. - */ -object JSONUtils { - - val gson = BDPJettyServerHelper.gson - -} From 0455164a10ee60d1038215f145d93cc3480347e6 Mon Sep 17 00:00:00 2001 From: wushengyeyouya <690574002@qq.com> Date: Mon, 23 Aug 2021 11:25:40 +0800 Subject: [PATCH 2/2] Update the version of Linkis. --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 6c1abde2ed..19e2f4e7dd 100644 --- a/pom.xml +++ b/pom.xml @@ -39,7 +39,7 @@ 1.0.0 - 1.0.1 + 1.0.2 2.11.12 1.8 3.3.3