From f0ff451904d55c98b22e6ab1b48f29a0556bc48a Mon Sep 17 00:00:00 2001 From: Eric Gao Date: Wed, 19 Oct 2022 11:08:04 +0800 Subject: [PATCH] [Improvement][Style] Fix remaining style errors and wildcard imports (#12412) * Fix remaining style errors and wildcard imports * Upgrade spotless to the latest stable version, exclude .github folder from spotless check and fix dead link * Fix spotless matching scope of markdown files --- docs/docs/en/about/glossary.md | 2 +- docs/docs/en/about/hardware.md | 3 +- .../en/contribute/join/become-a-committer.md | 3 +- docs/docs/en/guide/datasource/oracle.md | 24 +- docs/docs/en/guide/installation/standalone.md | 4 +- docs/docs/en/guide/upgrade/incompatible.md | 1 + docs/docs/zh/about/hardware.md | 3 +- .../zh/contribute/join/become-a-committer.md | 1 + docs/docs/zh/guide/datasource/oracle.md | 1 + docs/docs/zh/guide/parameter/built-in.md | 61 ++- docs/docs/zh/guide/upgrade/incompatible.md | 1 + .../dolphinscheduler-alert-api/pom.xml | 11 +- .../alert/api/AlertChannel.java | 1 + .../alert/api/AlertChannelFactory.java | 3 +- .../alert/api/AlertConstants.java | 1 + .../dolphinscheduler/alert/api/ShowType.java | 1 + .../dolphinscheduler-alert-dingtalk/pom.xml | 7 +- .../dingtalk/DingTalkAlertChannelFactory.java | 19 +- .../dingtalk/DingTalkParamsConstants.java | 1 + .../plugin/alert/dingtalk/DingTalkSender.java | 17 +- .../dolphinscheduler-alert-email/pom.xml | 7 +- .../plugin/alert/email/EmailAlertChannel.java | 1 + .../alert/email/EmailAlertChannelFactory.java | 142 ++--- .../plugin/alert/email/EmailConstants.java | 28 +- .../plugin/alert/email/ExcelUtils.java | 18 +- .../alert/email/MailParamsConstants.java | 1 + .../plugin/alert/email/MailSender.java | 20 +- .../email/exception/AlertEmailException.java | 1 + .../email/template/DefaultHTMLTemplate.java | 10 +- .../dolphinscheduler-alert-feishu/pom.xml | 7 +- .../alert/feishu/FeiShuAlertChannel.java | 1 + .../feishu/FeiShuAlertChannelFactory.java | 62 ++- .../alert/feishu/FeiShuParamsConstants.java | 1 + .../plugin/alert/feishu/FeiShuSender.java | 14 +- .../plugin/alert/feishu/HttpRequestUtil.java | 4 +- .../dolphinscheduler-alert-http/pom.xml | 7 +- .../plugin/alert/http/HttpAlertChannel.java | 1 + .../alert/http/HttpAlertChannelFactory.java | 63 ++- .../plugin/alert/http/HttpAlertConstants.java | 1 + .../dolphinscheduler-alert-pagerduty/pom.xml | 7 +- .../pagerduty/PagerDutyAlertChannel.java | 1 + .../PagerDutyAlertChannelFactory.java | 9 +- .../pagerduty/PagerDutyParamsConstants.java | 1 + .../alert/pagerduty/PagerDutySender.java | 20 +- .../dolphinscheduler-alert-script/pom.xml | 7 +- .../plugin/alert/script/OSUtils.java | 1 + .../plugin/alert/script/ProcessUtils.java | 1 + .../alert/script/ScriptAlertChannel.java | 1 + .../script/ScriptAlertChannelFactory.java | 41 +- .../alert/script/ScriptParamsConstants.java | 1 + .../plugin/alert/script/ScriptType.java | 1 + .../plugin/alert/script/StreamGobbler.java | 1 + .../dolphinscheduler-alert-slack/pom.xml | 7 +- .../plugin/alert/slack/SlackAlertChannel.java | 1 + .../alert/slack/SlackAlertChannelFactory.java | 24 +- .../alert/slack/SlackParamsConstants.java | 1 + .../plugin/alert/slack/SlackSender.java | 16 +- .../dolphinscheduler-alert-telegram/pom.xml | 7 +- .../alert/telegram/TelegramAlertChannel.java | 1 + .../telegram/TelegramAlertChannelFactory.java | 74 ++- .../telegram/TelegramParamsConstants.java | 1 - .../plugin/alert/telegram/TelegramSender.java | 14 +- .../dolphinscheduler-alert-webexteams/pom.xml | 7 +- .../plugin/alert/webexteams/WebexMessage.java | 1 + .../webexteams/WebexTeamsAlertChannel.java | 1 + .../WebexTeamsAlertChannelFactory.java | 90 +-- .../webexteams/WebexTeamsDestination.java | 4 +- .../alert/webexteams/WebexTeamsSender.java | 9 +- .../dolphinscheduler-alert-wechat/pom.xml | 7 +- .../alert/wechat/WeChatAlertChannel.java | 1 + .../wechat/WeChatAlertChannelFactory.java | 74 +-- .../alert/wechat/WeChatAlertConstants.java | 4 +- .../wechat/WeChatAlertParamsConstants.java | 1 + .../plugin/alert/wechat/WeChatSender.java | 28 +- .../plugin/alert/wechat/WeChatType.java | 2 +- .../alert/wechat/WechatAppChatMessage.java | 4 +- .../plugin/alert/wechat/WechatAppMessage.java | 3 +- .../dolphinscheduler-alert-plugins/pom.xml | 7 +- .../alert/AlertPluginManager.java | 32 +- .../alert/AlertRequestProcessor.java | 14 +- .../alert/AlertServerMetrics.java | 2 +- dolphinscheduler-alert/pom.xml | 10 +- .../api/ApiApplicationServer.java | 1 - .../api/aspect/AccessLogAnnotation.java | 1 + .../api/aspect/AccessLogAspect.java | 16 +- .../api/audit/AuditMessage.java | 4 +- .../api/configuration/AuditConfiguration.java | 3 +- .../OpenAPITranslationConfiguration.java | 44 +- .../PythonGatewayConfiguration.java | 1 + .../configuration/TaskTypeConfiguration.java | 12 +- .../configuration/TrafficConfiguration.java | 4 +- .../ApiFuncIdentificationConstant.java | 23 +- .../api/controller/AccessTokenController.java | 2 +- .../api/controller/AlertGroupController.java | 2 +- .../AlertPluginInstanceController.java | 1 + .../api/controller/ClusterController.java | 2 +- .../api/controller/DataQualityController.java | 2 +- .../api/controller/DataSourceController.java | 2 +- .../api/controller/DsErrorController.java | 2 + .../api/controller/EnvironmentController.java | 2 +- .../api/controller/ExecutorController.java | 26 +- .../api/controller/FavTaskController.java | 22 +- .../controller/K8sNamespaceController.java | 2 +- .../api/controller/MonitorController.java | 3 +- .../ProcessDefinitionController.java | 12 +- .../controller/ProcessInstanceController.java | 42 +- .../api/controller/ProjectController.java | 2 +- .../api/controller/ProjectV2Controller.java | 2 +- .../api/controller/QueueController.java | 2 +- .../api/controller/QueueV2Controller.java | 24 +- .../api/controller/ResourcesController.java | 2 +- .../api/controller/SchedulerController.java | 1 + .../controller/TaskDefinitionController.java | 2 +- .../controller/TaskInstanceController.java | 2 +- .../api/controller/TenantController.java | 2 +- .../api/controller/UsersController.java | 2 +- .../controller/WorkFlowLineageController.java | 2 +- .../api/controller/WorkerGroupController.java | 20 +- .../api/dto/CommandStateCount.java | 3 +- .../api/dto/CreateTokenRequest.java | 2 +- .../api/dto/CreateTokenResponse.java | 1 + .../api/dto/RuleDefinition.java | 2 +- .../api/dto/TaskCountDto.java | 8 +- .../api/dto/TaskStateCount.java | 3 +- .../api/dto/gantt/GanttDto.java | 9 +- .../dolphinscheduler/api/dto/gantt/Task.java | 1 + .../api/dto/project/ProjectCreateRequest.java | 2 +- .../dto/project/ProjectDeleteResponse.java | 2 +- .../api/dto/project/ProjectQueryRequest.java | 3 +- .../api/dto/project/ProjectUpdateRequest.java | 4 +- .../api/dto/queue/QueueCreateRequest.java | 2 +- .../api/dto/queue/QueueQueryRequest.java | 3 +- .../api/dto/queue/QueueUpdateRequest.java | 3 +- .../api/dto/queue/QueueVerifyRequest.java | 2 +- .../api/dto/resources/FileLeaf.java | 2 +- .../api/dto/resources/filter/IFilter.java | 1 + .../dto/resources/filter/ResourceFilter.java | 17 +- .../visitor/ResourceTreeVisitor.java | 27 +- .../api/dto/resources/visitor/Visitor.java | 1 + .../dto/schedule/ScheduleCreateRequest.java | 3 - .../api/enums/ExecuteType.java | 2 +- .../api/exceptions/ApiException.java | 1 + .../api/exceptions/ApiExceptionHandler.java | 1 - .../api/interceptor/RateLimitInterceptor.java | 16 +- .../api/k8s/K8sClientService.java | 35 +- .../dolphinscheduler/api/k8s/K8sManager.java | 4 +- .../api/permission/PermissionCheck.java | 10 +- .../ResourcePermissionCheckService.java | 11 +- .../ResourcePermissionCheckServiceImpl.java | 46 +- .../api/python/PythonGateway.java | 144 +++-- .../api/security/Authenticator.java | 1 + .../api/security/SecurityConfig.java | 1 + .../security/impl/AbstractAuthenticator.java | 3 +- .../security/impl/ldap/LdapAuthenticator.java | 2 +- .../api/service/AccessTokenService.java | 1 - .../api/service/AlertGroupService.java | 3 +- .../service/AlertPluginInstanceService.java | 5 +- .../api/service/BaseService.java | 9 +- .../api/service/ClusterService.java | 1 - .../api/service/DataAnalysisService.java | 4 +- .../api/service/DataSourceService.java | 2 +- .../api/service/DqRuleService.java | 2 +- .../api/service/EnvironmentService.java | 5 +- ...EnvironmentWorkerGroupRelationService.java | 1 - .../api/service/ExecutorService.java | 3 +- .../api/service/K8sNamespaceService.java | 8 +- .../api/service/LoggerService.java | 1 - .../api/service/MonitorService.java | 12 +- .../api/service/ProcessDefinitionService.java | 4 +- .../api/service/ProcessInstanceService.java | 10 +- .../api/service/QueueService.java | 2 - .../api/service/ResourcesService.java | 17 +- .../api/service/TaskGroupQueueService.java | 8 +- .../api/service/TaskGroupService.java | 2 +- .../api/service/TenantService.java | 2 +- .../api/service/UdfFuncService.java | 22 +- .../api/service/UsersService.java | 10 +- .../api/service/WorkerGroupService.java | 3 +- .../service/impl/AccessTokenServiceImpl.java | 7 +- .../service/impl/AlertGroupServiceImpl.java | 44 +- .../impl/AlertPluginInstanceServiceImpl.java | 46 +- .../api/service/impl/AuditServiceImpl.java | 2 +- .../api/service/impl/BaseServiceImpl.java | 55 +- .../api/service/impl/ClusterServiceImpl.java | 4 +- .../service/impl/DataAnalysisServiceImpl.java | 16 +- .../service/impl/DataSourceServiceImpl.java | 77 +-- .../impl/DqExecuteResultServiceImpl.java | 1 - .../service/impl/EnvironmentServiceImpl.java | 45 +- ...ronmentWorkerGroupRelationServiceImpl.java | 12 +- .../api/service/impl/FavTaskServiceImpl.java | 6 +- .../service/impl/K8SNamespaceServiceImpl.java | 4 +- .../api/service/impl/LoggerServiceImpl.java | 17 +- .../api/service/impl/MonitorServiceImpl.java | 50 +- .../impl/ProcessDefinitionServiceImpl.java | 6 +- .../impl/ProcessInstanceServiceImpl.java | 144 +++-- .../api/service/impl/ProjectServiceImpl.java | 34 +- .../api/service/impl/QueueServiceImpl.java | 29 +- .../service/impl/ResourcesServiceImpl.java | 70 ++- .../api/service/impl/SessionServiceImpl.java | 2 +- .../impl/TaskGroupQueueServiceImpl.java | 21 +- .../service/impl/TaskGroupServiceImpl.java | 3 +- .../service/impl/TaskInstanceServiceImpl.java | 45 +- .../api/service/impl/TenantServiceImpl.java | 64 ++- .../api/service/impl/UdfFuncServiceImpl.java | 76 +-- .../api/service/impl/UsersServiceImpl.java | 22 +- .../impl/WorkFlowLineageServiceImpl.java | 60 +- .../service/impl/WorkerGroupServiceImpl.java | 3 +- .../api/utils/CheckUtils.java | 8 +- .../dolphinscheduler/api/utils/FileUtils.java | 1 + .../dolphinscheduler/api/utils/PageInfo.java | 4 +- .../controller/WorkerGroupControllerTest.java | 4 +- .../api/service/BaseServiceTest.java | 1 + .../service/ProcessInstanceServiceTest.java | 23 +- .../ProcessTaskRelationServiceTest.java | 1 + .../api/service/TenantServiceTest.java | 3 +- .../api/service/WorkerGroupServiceTest.java | 8 +- .../config/YamlPropertySourceFactory.java | 6 +- .../common/enums/AlertEvent.java | 2 +- .../common/enums/AlertStatus.java | 1 + .../common/enums/AlertType.java | 4 +- .../common/enums/AlertWarnLevel.java | 2 +- .../common/enums/AuditResourceType.java | 1 + .../common/enums/AuthorizationType.java | 17 +- .../common/enums/BlockingOpportunity.java | 2 +- .../common/enums/CacheType.java | 1 + .../common/enums/CommandType.java | 2 +- .../common/enums/ComplementDependentMode.java | 5 +- .../common/enums/ConditionType.java | 1 + .../dolphinscheduler/common/enums/Flag.java | 1 + .../common/enums/HttpCheckCondition.java | 4 +- .../common/enums/HttpMethod.java | 2 +- .../common/enums/HttpParametersType.java | 2 +- .../common/enums/PluginType.java | 3 +- .../common/enums/Priority.java | 1 + .../common/enums/ReleaseState.java | 4 +- .../common/enums/RunMode.java | 3 +- .../common/enums/SlotCheckState.java | 2 +- .../common/enums/SqoopJobType.java | 3 +- .../common/enums/TaskDependType.java | 1 + .../common/enums/TaskExecuteType.java | 1 + .../common/enums/TaskRecordStatus.java | 3 +- .../common/enums/TimeoutFlag.java | 1 + .../common/enums/UdfType.java | 14 +- .../common/enums/UserType.java | 2 +- .../common/enums/WarningType.java | 3 +- .../common/enums/WorkflowExecutionStatus.java | 8 +- .../StorageOperateNoConfiguredException.java | 3 +- .../dolphinscheduler/common/graph/DAG.java | 19 +- .../common/model/BaseHeartBeatTask.java | 6 +- .../common/model/MasterHeartBeat.java | 1 + .../common/model/PreviousTaskNode.java | 2 +- .../dolphinscheduler/common/model/Server.java | 4 +- .../common/model/TaskNodeRelation.java | 2 +- .../common/model/WorkerHeartBeat.java | 1 - .../common/process/HttpProperty.java | 9 +- .../common/shell/AbstractShell.java | 28 +- .../common/shell/ShellExecutor.java | 12 +- .../common/thread/ThreadUtils.java | 4 +- .../common/utils/CollectionUtils.java | 3 +- .../common/utils/DateUtils.java | 19 +- .../common/utils/FileUtils.java | 12 +- .../common/utils/HttpUtils.java | 18 +- .../common/utils/KerberosHttpClient.java | 15 +- .../common/utils/RetryerUtils.java | 4 +- .../utils/placeholder/BusinessTimeUtils.java | 4 +- .../common/ConstantsTest.java | 1 + .../common/exception/ExceptionTest.java | 3 +- .../common/graph/DAGTest.java | 511 +++++++++--------- .../common/os/OSUtilsTest.java | 8 +- .../common/utils/CodeGenerateUtilsTest.java | 5 +- .../common/utils/DateUtilsTest.java | 17 +- .../common/utils/EncryptionUtilsTest.java | 1 - .../common/utils/FileUtilsTest.java | 11 +- .../common/utils/LocalJettyHttpServer.java | 21 +- .../utils/LocalServerHttpUtilsTest.java | 16 +- .../common/utils/NetUtilsTest.java | 3 +- .../common/utils/PropertyUtilsTest.java | 1 + .../common/utils/RetryerUtilsTest.java | 7 +- .../common/utils/ScriptRunnerTest.java | 7 +- .../common/utils/StringTest.java | 12 +- .../apache/dolphinscheduler/dao/AlertDao.java | 4 +- .../dao/entity/CycleDependency.java | 1 + .../dolphinscheduler/dao/entity/DagData.java | 3 +- .../dao/entity/DataSource.java | 2 +- .../entity/DependentProcessDefinition.java | 6 +- .../entity/DqExecuteResultAlertContent.java | 5 +- .../dao/entity/ErrorCommand.java | 2 +- .../dao/entity/ExecuteStatusCount.java | 3 +- .../dolphinscheduler/dao/entity/FavTask.java | 7 +- .../dao/entity/ProcessAlertContent.java | 20 +- .../dao/entity/ResponseTaskLog.java | 1 + .../dao/entity/ServerAlertContent.java | 1 + .../dolphinscheduler/dao/entity/Session.java | 6 +- .../dao/entity/TaskAlertContent.java | 9 +- .../dao/entity/WorkFlowLineage.java | 1 + .../dao/entity/WorkFlowRelation.java | 3 +- .../dao/entity/ZookeeperRecord.java | 6 +- .../dao/mapper/AccessTokenMapper.java | 7 +- .../dao/mapper/AlertGroupMapper.java | 10 +- .../dao/mapper/AuditLogMapper.java | 1 + .../dao/mapper/CommandMapper.java | 11 +- .../dao/mapper/DataSourceMapper.java | 15 +- .../dao/mapper/DataSourceUserMapper.java | 1 - .../dao/mapper/EnvironmentMapper.java | 6 +- .../dao/mapper/ErrorCommandMapper.java | 6 +- .../dao/mapper/FavTaskMapper.java | 4 +- .../dao/mapper/K8sNamespaceMapper.java | 167 +++--- .../mapper/ProcessDefinitionLogMapper.java | 7 +- .../dao/mapper/ProcessInstanceMapMapper.java | 4 +- .../dao/mapper/ProjectMapper.java | 4 +- .../dao/mapper/QueueMapper.java | 10 +- .../dao/mapper/ResourceMapper.java | 12 +- .../dao/mapper/ResourceUserMapper.java | 2 +- .../dao/mapper/SessionMapper.java | 6 +- .../dao/mapper/TaskDefinitionLogMapper.java | 3 +- .../dao/mapper/TaskGroupMapper.java | 9 +- .../dao/mapper/TaskGroupQueueMapper.java | 19 +- .../dao/mapper/TaskInstanceMapper.java | 31 +- .../dao/mapper/TenantMapper.java | 12 +- .../dao/mapper/UDFUserMapper.java | 5 +- .../dao/mapper/UdfFuncMapper.java | 10 +- .../dao/mapper/UserMapper.java | 1 - .../dao/mapper/WorkFlowLineageMapper.java | 7 +- .../dao/upgrade/JsonSplitDao.java | 45 +- .../dao/upgrade/ProcessDefinitionDao.java | 7 +- .../dao/upgrade/ScheduleDao.java | 3 +- .../dao/upgrade/SchemaUtils.java | 7 +- .../dao/upgrade/WorkerGroupDao.java | 16 +- .../dao/utils/BeanContext.java | 4 +- .../dao/utils/PostgreSQLPerformance.java | 4 +- .../utils/ResourceProcessDefinitionUtils.java | 4 +- .../dao/utils/TaskInstanceUtils.java | 3 +- .../dao/utils/WorkflowUtils.java | 12 +- .../dolphinscheduler/dao/AlertDaoTest.java | 11 +- .../dao/entity/ProcessDefinitionTest.java | 8 +- .../dao/entity/TaskInstanceTest.java | 8 +- .../dao/mapper/AccessTokenMapperTest.java | 9 +- .../dao/mapper/AlertGroupMapperTest.java | 13 +- .../dao/mapper/AlertSendStatusMapperTest.java | 1 + .../dao/mapper/AuditLogMapperTest.java | 2 +- .../dao/mapper/CommandMapperTest.java | 1 - .../dao/mapper/DataSourceUserMapperTest.java | 10 +- .../mapper/DqRuleInputEntryMapperTest.java | 9 +- ...ironmentWorkerGroupRelationMapperTest.java | 10 +- .../dao/mapper/ErrorCommandMapperTest.java | 12 +- .../dao/mapper/PluginDefineTest.java | 3 +- .../mapper/ProcessInstanceMapMapperTest.java | 17 +- .../dao/mapper/ProcessInstanceMapperTest.java | 15 +- .../ProcessTaskRelationLogMapperTest.java | 2 +- .../mapper/ProcessTaskRelationMapperTest.java | 2 +- .../dao/mapper/ProjectMapperTest.java | 27 +- .../dao/mapper/ProjectUserMapperTest.java | 13 +- .../dao/mapper/QueueMapperTest.java | 14 +- .../dao/mapper/ResourceUserMapperTest.java | 23 +- .../dao/mapper/SessionMapperTest.java | 14 +- .../dao/mapper/TaskDefinitionMapperTest.java | 2 +- .../dao/mapper/TaskGroupMapperTest.java | 1 - .../dao/mapper/TaskInstanceMapperTest.java | 4 +- .../dao/mapper/TenantMapperTest.java | 22 +- .../dao/mapper/UDFUserMapperTest.java | 45 +- .../dao/mapper/UserMapperTest.java | 48 +- .../dao/mapper/WorkFlowLineageMapperTest.java | 20 +- .../dao/mapper/WorkerGroupMapperTest.java | 11 +- .../dao/upgrade/ProcessDefinitionDaoTest.java | 18 +- .../dao/upgrade/SchemaUtilsTest.java | 2 +- .../dao/upgrade/WorkerGroupDaoTest.java | 9 +- .../ResourceProcessDefinitionUtilsTest.java | 19 +- .../dao/utils/TaskInstanceUtilsTest.java | 2 +- .../dao/utils/WorkflowUtilsTest.java | 6 +- .../data/quality/DataQualityApplication.java | 10 +- .../data/quality/config/BaseConfig.java | 4 +- .../data/quality/config/Config.java | 8 +- .../config/DataQualityConfiguration.java | 5 +- .../data/quality/config/EnvConfig.java | 4 +- .../data/quality/config/ReaderConfig.java | 5 +- .../quality/config/TransformerConfig.java | 5 +- .../data/quality/config/WriterConfig.java | 5 +- .../quality/context/DataQualityContext.java | 8 +- .../data/quality/enums/ReaderType.java | 3 +- .../data/quality/enums/TransformerType.java | 3 +- .../data/quality/enums/WriterType.java | 3 +- .../data/quality/execution/Execution.java | 1 + .../execution/SparkBatchExecution.java | 12 +- .../execution/SparkRuntimeEnvironment.java | 4 +- .../data/quality/flow/Component.java | 4 +- .../quality/flow/batch/reader/HiveReader.java | 2 +- .../quality/flow/batch/reader/JdbcReader.java | 4 +- .../flow/batch/reader/ReaderFactory.java | 4 +- .../batch/transformer/TransformerFactory.java | 6 +- .../quality/flow/batch/writer/JdbcWriter.java | 18 +- .../flow/batch/writer/WriterFactory.java | 6 +- .../batch/writer/file/BaseFileWriter.java | 13 +- .../batch/writer/file/HdfsFileWriter.java | 2 +- .../batch/writer/file/LocalFileWriter.java | 2 +- .../data/quality/utils/JsonUtils.java | 4 +- .../dolphinscheduler-datasource-all/pom.xml | 7 +- .../api/client/CommonDataSourceClient.java | 5 +- .../datasource/BaseHDFSConnectionParam.java | 1 + .../api/datasource/DataSourceProcessor.java | 1 - .../api/plugin/DataSourcePluginManager.java | 11 +- .../plugin/DataSourceProcessorManager.java | 1 + .../plugin/DataSourceProcessorProvider.java | 6 +- .../datasource/api/utils/CommonUtils.java | 10 +- .../datasource/api/utils/DataSourceUtils.java | 11 +- .../api/datasource/MySQLConnectionParam.java | 1 + .../pom.xml | 10 +- .../athena/AthenaDataSourceChannel.java | 3 +- .../AthenaDataSourceChannelFactory.java | 1 + .../athena/AthenaDataSourceClient.java | 1 + .../param/AthenaDataSourceParamDTO.java | 1 + .../param/AthenaDataSourceProcessor.java | 20 +- .../ClickHouseDataSourceChannelFactory.java | 1 + .../param/ClickHouseConnectionParam.java | 1 + .../param/ClickHouseDataSourceProcessor.java | 9 +- .../db2/param/Db2ConnectionParam.java | 1 + .../db2/param/Db2DataSourceProcessor.java | 3 +- .../hive/HiveDataSourceChannelFactory.java | 1 + .../hive/param/HiveConnectionParam.java | 1 + .../hive/param/HiveDataSourceProcessor.java | 3 +- .../mysql/MySQLDataSourceChannelFactory.java | 1 + .../mysql/param/MySQLConnectionParam.java | 1 + .../mysql/param/MySQLDataSourceProcessor.java | 18 +- .../OracleDataSourceChannelFactory.java | 1 + .../oracle/param/OracleConnectionParam.java | 1 + .../PostgreSQLDataSourceChannelFactory.java | 1 + .../param/PostgreSQLConnectionParam.java | 1 + .../param/PostgreSQLDataSourceProcessor.java | 9 +- .../PrestoDataSourceChannelFactory.java | 1 + .../presto/param/PrestoConnectionParam.java | 1 + .../param/PrestoDataSourceProcessor.java | 3 +- .../pom.xml | 9 +- .../redshift/RedshiftDataSourceChannel.java | 3 +- .../RedshiftDataSourceChannelFactory.java | 1 + .../redshift/RedshiftDataSourceClient.java | 1 + .../param/RedshiftConnectionParam.java | 1 + .../param/RedshiftDataSourceProcessor.java | 15 +- .../spark/SparkDataSourceChannelFactory.java | 1 + .../spark/param/SparkConnectionParam.java | 1 + .../spark/param/SparkDataSourceProcessor.java | 15 +- .../SQLServerDataSourceChannelFactory.java | 1 + .../param/SQLServerConnectionParam.java | 1 + .../param/SQLServerDataSourceProcessor.java | 3 +- .../server/master/MasterServer.java | 5 +- .../builder/TaskExecutionContextBuilder.java | 5 +- .../ProcessInstanceExecCacheManager.java | 3 +- .../StreamTaskInstanceExecCacheManager.java | 3 +- .../ProcessInstanceExecCacheManagerImpl.java | 6 +- ...treamTaskInstanceExecCacheManagerImpl.java | 11 +- .../server/master/config/MasterConfig.java | 12 +- .../consumer/TaskPriorityQueueConsumer.java | 15 +- .../controller/WorkflowExecuteController.java | 3 +- .../master/dispatch/ExecutorDispatcher.java | 2 +- .../executor/NettyExecutorManager.java | 2 - .../dispatch/host/CommonHostManager.java | 5 +- .../dispatch/host/LowerWeightHostManager.java | 13 +- .../host/assign/AbstractSelector.java | 5 +- .../dispatch/host/assign/HostWeight.java | 15 +- .../host/assign/LowerWeightRoundRobin.java | 12 +- .../host/assign/RoundRobinSelector.java | 4 +- .../server/master/event/StateEvent.java | 6 +- .../master/event/StateEventHandler.java | 4 +- .../event/StateEventHandlerManager.java | 4 +- .../event/TaskRetryStateEventHandler.java | 6 +- .../master/event/TaskRunningEventHandler.java | 5 +- .../server/master/event/TaskStateEvent.java | 7 +- .../event/TaskTimeoutStateEventHandler.java | 26 +- .../event/TaskWaitTaskGroupStateHandler.java | 3 +- .../event/WorkflowBlockStateEventHandler.java | 8 +- .../event/WorkflowStartEventHandler.java | 28 +- .../master/event/WorkflowStateEvent.java | 9 +- .../event/WorkflowStateEventHandler.java | 4 +- .../WorkflowTimeoutStateEventHandler.java | 3 +- .../master/metrics/MasterServerMetrics.java | 2 +- .../metrics/ProcessInstanceMetrics.java | 26 +- .../server/master/metrics/TaskMetrics.java | 14 +- .../master/processor/CacheProcessor.java | 4 +- .../HostUpdateResponseProcessor.java | 7 +- .../master/processor/StateEventProcessor.java | 6 +- .../master/processor/TaskEventProcessor.java | 6 +- .../TaskExecuteResponseProcessor.java | 12 +- .../TaskExecuteRunningProcessor.java | 11 +- .../processor/TaskExecuteStartProcessor.java | 15 +- .../processor/TaskKillResponseProcessor.java | 9 +- .../master/processor/TaskRecallProcessor.java | 8 +- ...WorkflowExecutingDataRequestProcessor.java | 10 +- .../queue/StateEventResponseService.java | 17 +- .../processor/queue/TaskEventService.java | 16 +- .../processor/queue/TaskExecuteRunnable.java | 10 +- .../queue/TaskExecuteThreadPool.java | 21 +- .../MasterConnectionStateListener.java | 4 +- .../master/registry/MasterRegistryClient.java | 10 +- .../registry/MasterRegistryDataListener.java | 4 +- .../master/registry/MasterStopStrategy.java | 1 + .../registry/MasterWaitingStrategy.java | 5 +- .../master/runner/EventExecuteService.java | 1 + .../master/runner/FailoverExecuteThread.java | 1 + .../runner/MasterSchedulerBootstrap.java | 4 +- .../runner/StateWheelExecuteThread.java | 15 +- .../runner/StreamTaskExecuteThreadPool.java | 4 +- .../master/runner/WorkflowEventLooper.java | 12 +- .../runner/WorkflowExecuteRunnable.java | 20 +- .../runner/WorkflowExecuteThreadPool.java | 14 +- .../runner/task/CommonTaskProcessor.java | 16 +- .../master/runner/task/TaskInstanceKey.java | 7 +- .../runner/task/TaskProcessorFactory.java | 19 +- .../master/service/ExecutingService.java | 3 +- .../master/service/FailoverService.java | 5 +- .../master/task/MasterHeartBeatTask.java | 5 +- .../server/master/utils/DependentExecute.java | 6 +- .../server/master/utils/SwitchTaskUtils.java | 3 +- .../server/master/ConditionsTaskTest.java | 11 +- .../server/master/DependentTaskTest.java | 9 +- .../server/master/ParamsTest.java | 10 +- .../server/master/SwitchTaskTest.java | 13 +- ...ocessInstanceExecCacheManagerImplTest.java | 2 +- .../TaskPriorityQueueConsumerTest.java | 2 +- .../dispatch/ExecutorDispatcherTest.java | 2 +- .../executor/NettyExecutorManagerTest.java | 20 +- .../host/RoundRobinHostManagerTest.java | 3 +- .../assign/LowerWeightRoundRobinTest.java | 30 +- .../host/assign/RandomSelectorTest.java | 6 +- .../host/assign/RoundRobinSelectorTest.java | 13 +- .../processor/TaskAckProcessorTest.java | 2 +- .../TaskResponseProcessorTestConfig.java | 2 +- .../queue/TaskResponseServiceTest.java | 2 +- .../runner/task/CommonTaskProcessorTest.java | 36 +- dolphinscheduler-meter/pom.xml | 33 +- .../meter/MeterConfiguration.java | 1 + dolphinscheduler-microbench/pom.xml | 32 +- .../base/AbstractBaseBenchmark.java | 1 - .../microbench/common/EnumBenchMark.java | 23 +- .../microbench/common/RpcTest.java | 1 + .../microbench/common/UserCallback.java | 1 + dolphinscheduler-python/pom.xml | 17 +- .../pydolphinscheduler/DEVELOP.md | 4 +- .../dolphinscheduler-registry-all/pom.xml | 7 +- .../dolphinscheduler-registry-api/pom.xml | 8 +- .../api/ConnectStrategyProperties.java | 4 +- .../registry/api/ConnectionListener.java | 1 + .../dolphinscheduler/registry/api/Event.java | 8 +- .../registry/api/SubscribeListener.java | 1 + .../dolphinscheduler-registry-etcd/pom.xml | 9 +- .../etcd/EtcdConnectionStateListener.java | 2 +- .../plugin/registry/etcd/EtcdRegistry.java | 44 +- .../registry/etcd/EtcdRegistryProperties.java | 5 +- .../dolphinscheduler-registry-mysql/pom.xml | 7 +- .../plugin/registry/mysql/MysqlRegistry.java | 14 +- .../mysql/MysqlRegistryProperties.java | 4 +- .../plugin/registry/mysql/model/DataType.java | 1 + .../mysql/task/RegistryLockManager.java | 8 +- .../mysql/task/SubscribeDataManager.java | 11 +- .../ZookeeperConnectionStateListener.java | 2 +- .../registry/zookeeper/ZookeeperRegistry.java | 27 +- .../ZookeeperRegistryProperties.java | 3 + .../dolphinscheduler-registry-plugins/pom.xml | 6 +- dolphinscheduler-registry/pom.xml | 41 +- .../remote/NettyRemotingClient.java | 61 ++- .../remote/NettyRemotingServer.java | 15 +- .../remote/codec/NettyDecoder.java | 1 + .../remote/codec/NettyEncoder.java | 1 - .../remote/command/Command.java | 4 +- .../remote/command/CommandType.java | 1 - .../remote/command/HostUpdateCommand.java | 3 +- .../command/StateEventResponseCommand.java | 5 +- .../remote/command/TaskDispatchCommand.java | 1 - .../command/TaskEventChangeCommand.java | 5 +- .../command/TaskExecuteResultCommand.java | 4 - .../command/TaskExecuteRunningAckMessage.java | 7 +- .../command/TaskExecuteRunningCommand.java | 2 +- .../command/TaskKillResponseCommand.java | 9 +- .../command/TaskSavePointResponseCommand.java | 8 +- .../WorkflowStateEventChangeCommand.java | 5 +- .../alert/AlertSendRequestCommand.java | 7 +- .../alert/AlertSendResponseCommand.java | 7 +- .../alert/AlertSendResponseResult.java | 4 +- .../command/log/GetAppIdRequestCommand.java | 7 +- .../command/log/GetAppIdResponseCommand.java | 7 +- .../log/GetLogBytesRequestCommand.java | 7 +- .../log/GetLogBytesResponseCommand.java | 7 +- .../log/RemoveTaskLogRequestCommand.java | 7 +- .../log/RemoveTaskLogResponseCommand.java | 7 +- .../log/RollViewLogRequestCommand.java | 7 +- .../log/RollViewLogResponseCommand.java | 7 +- .../command/log/ViewLogRequestCommand.java | 7 +- .../command/log/ViewLogResponseCommand.java | 7 +- .../remote/config/NettyClientConfig.java | 3 +- .../remote/dto/TaskInstanceExecuteDto.java | 5 +- .../remote/dto/WorkflowExecuteDto.java | 8 +- .../remote/exceptions/RemotingException.java | 1 - .../exceptions/RemotingTimeoutException.java | 4 +- .../RemotingTooMuchRequestException.java | 2 +- .../remote/future/ReleaseSemaphore.java | 6 +- .../remote/future/ResponseFuture.java | 3 +- .../remote/handler/NettyClientHandler.java | 6 +- .../remote/handler/NettyServerHandler.java | 7 +- .../remote/processor/NettyRemoteChannel.java | 5 +- .../processor/NettyRequestProcessor.java | 3 +- .../processor/StateEventCallbackService.java | 3 +- .../utils/CallerThreadExecutePolicy.java | 6 +- .../remote/utils/Constants.java | 1 - .../remote/utils/JsonSerializer.java | 1 + .../remote/utils/NamedThreadFactory.java | 4 +- .../apache/dolphinscheduler/rpc/base/Rpc.java | 5 +- .../dolphinscheduler/rpc/base/RpcService.java | 1 + .../rpc/client/ConsumerInterceptor.java | 8 +- .../rpc/codec/NettyDecoder.java | 3 +- .../rpc/codec/NettyEncoder.java | 3 +- .../rpc/common/RequestEventType.java | 5 +- .../rpc/common/ResponseEventType.java | 30 +- .../rpc/common/ThreadPoolManager.java | 3 +- .../rpc/protocol/EventType.java | 6 +- .../rpc/remote/NettyClient.java | 8 +- .../rpc/remote/NettyClientHandler.java | 4 +- .../rpc/remote/NettyServer.java | 23 +- .../rpc/serializer/ProtoStuffSerializer.java | 30 +- .../rpc/serializer/RpcSerializer.java | 31 +- .../remote/NettyRemotingClientTest.java | 33 +- .../alert/AlertSendRequestCommandTest.java | 4 +- .../alert/AlertSendResponseCommandTest.java | 5 +- .../command/cache/CacheExpireCommandTest.java | 1 + .../command/future/ResponseFutureTest.java | 10 +- .../log/GetLogBytesRequestCommandTest.java | 1 + .../log/GetLogBytesResponseCommandTest.java | 1 + .../log/RemoveTaskLogRequestCommandTest.java | 1 + .../log/RemoveTaskLogResponseCommandTest.java | 1 + .../log/RollViewLogRequestCommandTest.java | 1 + .../apache/dolphinscheduler/rpc/RpcTest.java | 1 + .../dolphinscheduler-scheduler-api/pom.xml | 7 +- .../scheduler/api/SchedulerApi.java | 2 +- .../dolphinscheduler-scheduler-quartz/pom.xml | 7 +- .../scheduler/quartz/ProcessScheduleTask.java | 14 +- .../scheduler/quartz/QuartzScheduler.java | 19 +- dolphinscheduler-scheduler-plugin/pom.xml | 11 +- .../bean/SpringApplicationContext.java | 2 +- .../service/cache/CacheNotifyService.java | 1 + .../service/cron/AbstractCycle.java | 2 +- .../service/cron/CronUtils.java | 38 +- .../service/cron/CycleFactory.java | 348 ++++++------ .../service/cron/CycleLinks.java | 86 +-- .../service/expand/CuringParamsService.java | 12 +- .../service/json/DateJsonSerializer.java | 1 + .../service/queue/MasterPriorityQueue.java | 1 + .../queue/PeerTaskInstancePriorityQueue.java | 11 +- .../service/queue/TaskPriority.java | 73 +-- .../service/queue/TaskPriorityQueue.java | 3 +- .../service/queue/TaskPriorityQueueImpl.java | 2 +- .../service/task/TaskPluginManager.java | 8 +- .../service/utils/ProcessData.java | 6 +- .../service/alert/AlertClientServiceTest.java | 9 +- .../alert/ProcessAlertManagerTest.java | 9 +- .../service/cache/CacheNotifyServiceTest.java | 7 +- .../service/cron/CronUtilsTest.java | 80 +-- .../expand/CuringGlobalParamsServiceTest.java | 52 +- ...ePlaceholderResolverExpandServiceTest.java | 4 +- .../service/log/LogClientTest.java | 5 +- .../log/LoggerRequestProcessorTest.java | 4 +- .../service/log/MasterLogFilterTest.java | 8 +- .../log/SensitiveDataConverterTest.java | 9 +- .../service/log/TaskLogDiscriminatorTest.java | 6 +- .../service/log/TaskLogFilterTest.java | 8 +- .../service/log/WorkerLogFilterTest.java | 8 +- .../service/process/ProcessServiceTest.java | 70 ++- .../PeerTaskInstancePriorityQueueTest.java | 8 +- .../queue/TaskPriorityQueueImplTest.java | 49 +- .../service/storage/impl/HadoopUtilsTest.java | 1 + .../service/storage/impl/OssOperatorTest.java | 22 +- .../service/utils/CommonUtilsTest.java | 7 +- .../service/utils/LogUtilsTest.java | 12 +- .../service/utils/ProcessUtilsTest.java | 11 +- .../datasource/DataSourceChannelFactory.java | 1 + .../spi/enums/CommandType.java | 2 +- .../dolphinscheduler/spi/enums/DbType.java | 8 +- .../dolphinscheduler/spi/enums/Flag.java | 1 + .../spi/enums/ResourceType.java | 1 + .../spi/params/PluginParamsTransfer.java | 10 +- .../spi/params/base/ParamsOptions.java | 3 +- .../spi/params/base/PluginParams.java | 15 +- .../spi/params/base/Validate.java | 1 + .../spi/params/fswitch/SwitchParam.java | 2 +- .../spi/params/group/GroupParam.java | 6 +- .../spi/params/input/InputParam.java | 2 +- .../spi/params/radio/RadioParam.java | 4 +- .../spi/plugin/PrioritySPIFactory.java | 9 +- .../dolphinscheduler/spi/utils/Constants.java | 8 +- .../dolphinscheduler/spi/utils/DateUtils.java | 4 +- .../dolphinscheduler/spi/utils/JSONUtils.java | 21 +- .../spi/utils/StringUtils.java | 27 +- .../spi/params/PluginParamsTransferTest.java | 67 +-- .../spi/plugin/PrioritySPIFactoryTest.java | 12 +- .../spi/utils/JSONUtilsTest.java | 26 +- .../dolphinscheduler/StandaloneServer.java | 4 +- .../dolphinscheduler-task-all/pom.xml | 2 +- .../plugin/task/api/AbstractShell.java | 19 +- .../task/api/K8sTaskExecutionContext.java | 5 +- .../plugin/task/api/ProcessUtils.java | 5 +- .../task/api/SQLTaskExecutionContext.java | 1 - .../plugin/task/api/ShellCommandExecutor.java | 6 +- .../plugin/task/api/ShellExecutor.java | 4 +- .../plugin/task/api/TaskChannelFactory.java | 2 +- .../plugin/task/api/TaskConstants.java | 2 - .../plugin/task/api/enums/DataType.java | 2 +- .../plugin/task/api/enums/DependResult.java | 1 - .../task/api/enums/DependentRelation.java | 2 +- .../plugin/task/api/enums/Direct.java | 2 +- .../task/api/enums/TaskTimeoutStrategy.java | 1 + .../plugin/task/api/enums/UdfType.java | 1 + .../plugin/task/api/enums/dp/CheckType.java | 13 +- .../task/api/enums/dp/ConnectorType.java | 9 +- .../task/api/enums/dp/DqFailureStrategy.java | 3 +- .../plugin/task/api/enums/dp/DqTaskState.java | 11 +- .../task/api/enums/dp/ExecuteSqlType.java | 11 +- .../plugin/task/api/enums/dp/InputType.java | 13 +- .../task/api/enums/dp/OperatorType.java | 17 +- .../task/api/enums/dp/OptionSourceType.java | 13 +- .../plugin/task/api/enums/dp/RuleType.java | 13 +- .../plugin/task/api/enums/dp/ValueType.java | 13 +- .../plugin/task/api/k8s/AbstractK8sTask.java | 6 +- .../task/api/k8s/AbstractK8sTaskExecutor.java | 1 + .../task/api/k8s/K8sTaskMainParameters.java | 14 +- .../task/api/loop/LoopTaskDefinition.java | 4 +- .../task/api/loop/LoopTaskInstanceStatus.java | 1 + .../api/loop/LoopTaskMethodDefinition.java | 3 +- .../loop/template/LoopTaskYamlDefinition.java | 5 + .../template/TemplateMethodTransformer.java | 3 +- .../template/http/HttpLoopTaskDefinition.java | 5 +- .../http/HttpLoopTaskMethodDefinition.java | 1 + ...ttpLoopTaskCancelTaskMethodDefinition.java | 5 +- ...tpLoopTaskQueryStatusMethodDefinition.java | 5 +- ...ttpLoopTaskSubmitTaskMethodDefinition.java | 11 +- .../CancelTemplateMethodTransformer.java | 15 +- .../http/parser/HttpTaskDefinitionParser.java | 16 +- .../QueryStateTemplateMethodTransformer.java | 17 +- .../SubmitTemplateMethodTransformer.java | 17 +- .../plugin/task/api/model/DateInterval.java | 1 + .../plugin/task/api/model/DependentItem.java | 3 +- .../task/api/model/DependentTaskModel.java | 3 +- .../plugin/task/api/model/Property.java | 2 +- .../plugin/task/api/model/SwitchResultVo.java | 2 +- .../api/parameters/AbstractParameters.java | 3 +- .../api/parameters/ConditionsParameters.java | 8 +- .../api/parameters/DependentParameters.java | 5 +- .../task/api/parameters/IParameters.java | 31 +- .../api/parameters/K8sTaskParameters.java | 15 +- .../task/api/parameters/ParametersNode.java | 1 + .../api/parameters/SubProcessParameters.java | 2 +- .../task/api/parameters/SwitchParameters.java | 2 +- .../dataquality/DataQualityParameters.java | 4 +- .../resource/AbstractResourceParameters.java | 1 - .../resource/DataSourceParameters.java | 3 +- .../task/api/parser/BusinessTimeUtils.java | 1 + .../plugin/task/api/parser/ParamUtils.java | 8 +- .../task/api/parser/ParameterUtils.java | 13 +- .../task/api/parser/PlaceholderUtils.java | 10 +- .../api/parser/PropertyPlaceholderHelper.java | 27 +- .../task/api/parser/TimePlaceholderUtils.java | 69 +-- .../plugin/task/api/stream/StreamTask.java | 1 + .../task/api/stream/StreamTaskChannel.java | 3 - .../task/api/utils/DependentDateUtils.java | 2 +- .../plugin/task/api/utils/JsonPathUtils.java | 8 +- .../plugin/task/api/utils/OkHttpUtils.java | 6 +- .../plugin/task/api/utils/RetryUtils.java | 1 + .../plugin/task/api/model/JSONUtilsTest.java | 6 +- .../task/api/utils/ParameterUtilsTest.java | 3 +- .../dolphinscheduler-task-blocking/pom.xml | 7 +- .../blocking/BlockingTaskChannelFactory.java | 1 + .../dolphinscheduler-task-chunjun/pom.xml | 7 +- .../task/chunjun/ChunJunParameters.java | 4 +- .../plugin/task/chunjun/ChunJunTask.java | 29 +- .../dolphinscheduler-task-conditions/pom.xml | 9 +- .../ConditionsTaskChannelFactory.java | 1 + .../dolphinscheduler-task-dataquality/pom.xml | 7 +- .../task/dq/DataQualityTaskChannel.java | 2 +- .../dq/exception/DataQualityException.java | 3 +- .../plugin/task/dq/rule/RuleManager.java | 14 +- .../task/dq/rule/parameter/BaseConfig.java | 4 +- .../parameter/DataQualityConfiguration.java | 3 +- .../task/dq/rule/parameter/EnvConfig.java | 4 +- .../task/dq/rule/parser/IRuleParser.java | 2 +- .../task/dq/rule/parser/MappingColumn.java | 3 +- .../parser/MultiTableAccuracyRuleParser.java | 23 +- .../MultiTableComparisonRuleParser.java | 7 +- .../SingleTableCustomSqlRuleParser.java | 22 +- .../dq/rule/parser/SingleTableRuleParser.java | 22 +- .../plugin/task/dq/utils/RuleParserUtils.java | 293 +++++----- .../plugin/task/dq/utils/SparkArgsUtils.java | 5 +- .../dolphinscheduler-task-datasync/pom.xml | 9 +- .../task/datasync/DatasyncParameters.java | 27 +- .../task/datasync/DatasyncTaskChannel.java | 2 +- .../datasync/DatasyncTaskChannelFactory.java | 5 +- .../dolphinscheduler-task-datax/pom.xml | 7 +- .../plugin/task/datax/DataxParameters.java | 7 +- .../plugin/task/datax/DataxTask.java | 58 +- .../plugin/task/datax/DataxUtils.java | 2 +- .../plugin/task/datax/DataxTaskTest.java | 48 +- .../dolphinscheduler-task-dependent/pom.xml | 9 +- .../task/dependent/DependentTaskChannel.java | 3 +- .../DependentTaskChannelFactory.java | 1 + .../dolphinscheduler-task-dinky/pom.xml | 11 +- .../plugin/task/dinky/DinkyParameters.java | 8 +- .../task/dinky/DinkyTaskChannelFactory.java | 5 +- .../plugin/task/dinky/DinkyTaskConstants.java | 5 +- .../dolphinscheduler-task-dms/pom.xml | 7 +- .../plugin/task/dms/DmsParameters.java | 6 +- .../plugin/task/dms/DmsTask.java | 19 +- .../task/dms/DmsTaskChannelFactory.java | 1 - .../dolphinscheduler-task-dvc/pom.xml | 7 +- .../plugin/task/dvc/DvcConstants.java | 18 +- .../plugin/task/dvc/DvcParameters.java | 21 +- .../plugin/task/dvc/DvcTaskChannel.java | 2 - .../task/dvc/DvcTaskChannelFactory.java | 9 +- .../dolphinscheduler-task-emr/pom.xml | 7 +- .../plugin/task/emr/AbstractEmrTask.java | 1 + .../plugin/task/emr/EmrParameters.java | 15 +- .../plugin/task/emr/EmrTaskChannel.java | 1 + .../task/emr/EmrTaskChannelFactory.java | 1 + .../pom.xml | 9 +- .../task/flink/FlinkStreamTaskChannel.java | 2 +- .../flink/FlinkStreamTaskChannelFactory.java | 1 + .../dolphinscheduler-task-flink/pom.xml | 27 +- .../plugin/task/flink/FileUtils.java | 28 +- .../plugin/task/flink/FlinkArgsUtils.java | 65 ++- .../plugin/task/flink/FlinkTaskChannel.java | 1 + .../task/flink/FlinkTaskChannelFactory.java | 1 + .../dolphinscheduler-task-http/pom.xml | 8 +- .../plugin/task/http/HttpCheckCondition.java | 2 +- .../plugin/task/http/HttpParameters.java | 1 + .../plugin/task/http/HttpParametersType.java | 2 +- .../plugin/task/http/HttpProperty.java | 1 + .../plugin/task/http/HttpTask.java | 17 +- .../dolphinscheduler-task-java/pom.xml | 11 +- .../plugin/task/java/JavaConstants.java | 2 - .../plugin/task/java/JavaParameters.java | 1 + .../plugin/task/java/JavaTask.java | 14 +- .../task/java/JavaTaskChannelFactory.java | 1 + .../JavaSourceFileExistException.java | 4 +- .../PublicClassNotFoundException.java | 4 +- .../dolphinscheduler-task-jupyter/pom.xml | 7 +- .../jupyter/JupyterTaskChannelFactory.java | 1 + .../dolphinscheduler-task-k8s/pom.xml | 7 +- .../plugin/task/k8s/K8sTaskChannel.java | 1 + .../task/k8s/K8sTaskChannelFactory.java | 1 + .../dolphinscheduler-task-mlflow/pom.xml | 7 +- .../plugin/task/mlflow/MlflowConstants.java | 31 +- .../plugin/task/mlflow/MlflowParameters.java | 11 +- .../plugin/task/mlflow/MlflowTaskChannel.java | 1 - .../task/mlflow/MlflowTaskChannelFactory.java | 8 +- .../dolphinscheduler-task-mr/pom.xml | 7 +- .../plugin/task/mr/MapReduceParameters.java | 3 +- .../plugin/task/mr/MapReduceTask.java | 14 +- .../plugin/task/mr/MapReduceTaskChannel.java | 1 + .../dolphinscheduler-task-openmldb/pom.xml | 7 +- .../task/openmldb/OpenmldbTaskChannel.java | 1 + .../openmldb/OpenmldbTaskChannelFactory.java | 1 + .../plugin/task/pigeon/PigeonConfig.java | 3 +- .../plugin/task/pigeon/PigeonTaskChannel.java | 1 + .../task/pigeon/PigeonTaskChannelFactory.java | 8 +- .../dolphinscheduler-task-procedure/pom.xml | 7 +- .../task/procedure/ProcedureParameters.java | 6 +- .../dolphinscheduler-task-python/pom.xml | 7 +- .../plugin/task/python/PythonParameters.java | 1 + .../plugin/task/python/PythonTaskChannel.java | 1 + .../task/python/PythonTaskChannelFactory.java | 31 +- .../dolphinscheduler-task-pytorch/pom.xml | 7 +- .../task/pytorch/GitProjectManager.java | 8 +- .../task/pytorch/PytorchParameters.java | 2 - .../plugin/task/pytorch/PytorchTask.java | 10 +- .../pytorch/PytorchTaskChannelFactory.java | 1 + .../task/sagemaker/SagemakerConstants.java | 1 + .../SagemakerTaskChannelFactory.java | 1 + .../dolphinscheduler-task-seatunnel/pom.xml | 7 +- .../plugin/task/seatunnel/DeployModeEnum.java | 1 + .../task/seatunnel/SeatunnelParameters.java | 7 +- .../task/seatunnel/SeatunnelTaskChannel.java | 3 +- .../flink/SeatunnelFlinkParameters.java | 1 + .../seatunnel/flink/SeatunnelFlinkTask.java | 7 +- .../spark/SeatunnelSparkParameters.java | 1 + .../seatunnel/spark/SeatunnelSparkTask.java | 9 +- .../dolphinscheduler-task-shell/pom.xml | 7 +- .../plugin/task/shell/ShellTask.java | 9 +- .../task/shell/ShellTaskChannelFactory.java | 1 + .../dolphinscheduler-task-spark/pom.xml | 7 +- .../task/spark/SparkTaskChannelFactory.java | 1 + .../dolphinscheduler-task-sql/pom.xml | 7 +- .../plugin/task/sql/SqlBinds.java | 1 + .../plugin/task/sql/SqlTask.java | 60 +- .../plugin/task/sql/SqlTaskChannel.java | 1 + .../task/sql/SqlTaskChannelFactory.java | 1 + .../dolphinscheduler-task-sqoop/pom.xml | 7 +- .../plugin/task/sqoop/SqoopConstants.java | 14 +- .../plugin/task/sqoop/SqoopJobType.java | 2 +- .../plugin/task/sqoop/SqoopTask.java | 11 +- .../task/sqoop/generator/CommonGenerator.java | 28 +- .../sqoop/generator/SqoopJobGenerator.java | 7 +- .../sources/HdfsSourceGenerator.java | 6 +- .../sources/HiveSourceGenerator.java | 16 +- .../sources/MySQLSourceGenerator.java | 29 +- .../targets/HdfsTargetGenerator.java | 12 +- .../targets/HiveTargetGenerator.java | 22 +- .../targets/MySQLTargetGenerator.java | 24 +- .../task/sqoop/parameter/SqoopParameters.java | 34 +- .../plugin/task/sqoop/EntityTestUtils.java | 12 +- .../dolphinscheduler-task-subprocess/pom.xml | 9 +- .../SubProcessTaskChannelFactory.java | 1 + .../dolphinscheduler-task-switch/pom.xml | 9 +- .../task/switchtask/SwitchParameters.java | 7 +- .../plugin/task/switchtask/SwitchResult.java | 1 + .../switchtask/SwitchTaskChannelFactory.java | 4 +- .../task/zeppelin/ZeppelinParameters.java | 7 +- .../zeppelin/ZeppelinTaskChannelFactory.java | 1 + dolphinscheduler-tools/pom.xml | 23 +- .../datasource/DolphinSchedulerManager.java | 19 +- .../datasource/InitDolphinScheduler.java | 2 + .../datasource/UpgradeDolphinScheduler.java | 2 + .../tools/datasource/dao/MySQLUpgradeDao.java | 13 +- .../datasource/dao/PostgreSQLUpgradeDao.java | 2 +- .../tools/datasource/dao/ResourceDao.java | 10 +- dolphinscheduler-ui/pom.xml | 128 ++--- .../server/worker/WorkerServer.java | 11 +- .../worker/message/MessageRetryRunner.java | 17 +- .../TaskExecuteResultMessageSender.java | 8 +- .../message/TaskRejectMessageSender.java | 4 +- .../server/worker/metrics/TaskMetrics.java | 6 +- .../worker/metrics/WorkerServerMetrics.java | 28 +- .../worker/processor/HostUpdateProcessor.java | 3 +- .../processor/TaskDispatchProcessor.java | 16 +- .../TaskExecuteResultAckProcessor.java | 6 +- .../TaskExecuteRunningAckProcessor.java | 2 +- .../processor/TaskRejectAckProcessor.java | 6 +- .../processor/TaskSavePointProcessor.java | 31 +- .../WorkerConnectionStateListener.java | 4 +- .../worker/registry/WorkerStopStrategy.java | 1 + .../registry/WorkerWaitingStrategy.java | 5 +- .../worker/rpc/WorkerMessageSender.java | 6 +- .../server/worker/rpc/WorkerRpcClient.java | 2 +- ...DefaultWorkerDelayTaskExecuteRunnable.java | 6 +- ...WorkerDelayTaskExecuteRunnableFactory.java | 10 +- .../WorkerDelayTaskExecuteRunnable.java | 12 +- ...WorkerDelayTaskExecuteRunnableFactory.java | 22 +- .../worker/runner/WorkerExecService.java | 2 +- .../worker/runner/WorkerManagerThread.java | 13 +- .../runner/WorkerTaskExecuteRunnable.java | 13 +- ...rkerTaskExecuteRunnableFactoryBuilder.java | 5 +- .../utils/TaskExecutionCheckerUtils.java | 2 +- .../server/worker/config/BeanConfigTest.java | 3 +- .../processor/TaskDispatchProcessorTest.java | 4 +- ...ultWorkerDelayTaskExecuteRunnableTest.java | 12 +- pom.xml | 15 +- tools/release/README.md | 1 + 948 files changed, 5893 insertions(+), 4725 deletions(-) diff --git a/docs/docs/en/about/glossary.md b/docs/docs/en/about/glossary.md index a8f844c06f28..29f0da5a5ad8 100644 --- a/docs/docs/en/about/glossary.md +++ b/docs/docs/en/about/glossary.md @@ -17,7 +17,7 @@ there are no subsequent nodes. Examples are as follows: manual start or scheduled scheduling. Each time the process definition runs, a process instance is generated **Task instance**: The task instance is the instantiation of the task node in the process definition, which identifies -the specific task +the specific task **Task type**: Currently supports SHELL, SQL, SUB_PROCESS (sub-process), PROCEDURE, MR, SPARK, PYTHON, DEPENDENT ( depends), and plans to support dynamic plug-in expansion, note: **SUB_PROCESS** need relation with another workflow definition which also a separate process diff --git a/docs/docs/en/about/hardware.md b/docs/docs/en/about/hardware.md index 97488332d4c2..604bb6a90eeb 100644 --- a/docs/docs/en/about/hardware.md +++ b/docs/docs/en/about/hardware.md @@ -49,4 +49,5 @@ DolphinScheduler provides the following network port configurations for normal o The minimum supported version of Google Chrome is version 85, but version 90 or above is recommended. ## Synchronize clocks -To avoid problems with internal cluster communications that can impact your task execution, make sure that the clocks on all of the cluster nodes are synchronized from a common clock source, such as using Chrony and/or NTP. Synchronizing the time ensures that every node in the cluster has the same time. \ No newline at end of file + +To avoid problems with internal cluster communications that can impact your task execution, make sure that the clocks on all of the cluster nodes are synchronized from a common clock source, such as using Chrony and/or NTP. Synchronizing the time ensures that every node in the cluster has the same time. diff --git a/docs/docs/en/contribute/join/become-a-committer.md b/docs/docs/en/contribute/join/become-a-committer.md index a64c854a0ee6..f748a1638c8b 100644 --- a/docs/docs/en/contribute/join/become-a-committer.md +++ b/docs/docs/en/contribute/join/become-a-committer.md @@ -20,7 +20,7 @@ In Dolphinscheduler, **new committer nomination** could only be officially start The following steps are recommended (to be initiated only by an existing PMC member): 1. Send an email titled `[DISCUSS] Promote xxx as new committer` to `private@dolphinscheduler.apache.org`. List the important contributions of the candidate, - so you could gather support from other PMC members for your proposal. +so you could gather support from other PMC members for your proposal. 2. Keep the discussion open for more than 3 days but no more than 1 week, unless there is any express objection or concern. 3. If the PMC generally agrees to the proposal, send an email titled `[VOTE] Promote xxx as new committer` to `private@dolphinscheduler.apache.org`. 4. Keep the voting process open for more than 3 days, but no more than 1 week. Consider the result as `Consensus Approval` if there are `3 + 1` votes with `NO` vetos. Please note that +1 votes > -1 votes. @@ -31,6 +31,7 @@ The following steps are recommended (to be initiated only by an existing PMC mem The PMC member who starts the promotion is responsible for sending an invitation to the new committer and guiding him/her to set up the ASF env. The PMC member should send an email using the following template to the new committer: + ``` To: @gmail.com Cc: private@dolphinscheduler.apache.org diff --git a/docs/docs/en/guide/datasource/oracle.md b/docs/docs/en/guide/datasource/oracle.md index f7ab96e1dd80..122801a2d986 100644 --- a/docs/docs/en/guide/datasource/oracle.md +++ b/docs/docs/en/guide/datasource/oracle.md @@ -4,18 +4,18 @@ ## Datasource Parameters -| **Datasource** | **Description** | -|-------------------------|---------------------------------------------------------------------------------------------------------------------------------------------| -| Datasource | Select Oracle. | -| Datasource Name | Enter the name of the datasource. | -| Description | Enter a description of the datasource. | -| IP/Host Name | Enter the Oracle service IP. | -| Port | Enter the Oracle service port. | -| Username | Set the username for Oracle connection. | -| Password | Set the password for Oracle connection. | -| Database Name | Enter the ServiceName or SID of the Oracle connection. | -| ServiceName or SID | Choose ServiceName or SID according to your entry in Database Name column. | -| jdbc connect parameters | Parameter settings for Oracle connection, in JSON format. For example, you can use {"schema": "abc"} to specify database abc for using. | +| **Datasource** | **Description** | +|-------------------------|-----------------------------------------------------------------------------------------------------------------------------------------| +| Datasource | Select Oracle. | +| Datasource Name | Enter the name of the datasource. | +| Description | Enter a description of the datasource. | +| IP/Host Name | Enter the Oracle service IP. | +| Port | Enter the Oracle service port. | +| Username | Set the username for Oracle connection. | +| Password | Set the password for Oracle connection. | +| Database Name | Enter the ServiceName or SID of the Oracle connection. | +| ServiceName or SID | Choose ServiceName or SID according to your entry in Database Name column. | +| jdbc connect parameters | Parameter settings for Oracle connection, in JSON format. For example, you can use {"schema": "abc"} to specify database abc for using. | ## Native Supported diff --git a/docs/docs/en/guide/installation/standalone.md b/docs/docs/en/guide/installation/standalone.md index 30ee2a138f22..d11f70f41e1b 100644 --- a/docs/docs/en/guide/installation/standalone.md +++ b/docs/docs/en/guide/installation/standalone.md @@ -2,8 +2,8 @@ Standalone only for quick experience for DolphinScheduler. -If you are a newbie and want to experience DolphinScheduler functions, we recommend you install follow [Standalone deployment](standalone.md). -If you want to experience more complete functions and schedule massive tasks, we recommend you install follow [pseudo-cluster deployment](pseudo-cluster.md). +If you are a newbie and want to experience DolphinScheduler functions, we recommend you install follow [Standalone deployment](standalone.md). +If you want to experience more complete functions and schedule massive tasks, we recommend you install follow [pseudo-cluster deployment](pseudo-cluster.md). If you want to deploy DolphinScheduler in production, we recommend you follow [cluster deployment](cluster.md) or [Kubernetes deployment](kubernetes.md). > **_Note:_** Standalone only recommends the usage of fewer than 20 workflows, because it uses in-memory H2 Database in default, ZooKeeper Testing Server, too many tasks may cause instability. diff --git a/docs/docs/en/guide/upgrade/incompatible.md b/docs/docs/en/guide/upgrade/incompatible.md index 0fd2c409ead2..ba1af107ce2b 100644 --- a/docs/docs/en/guide/upgrade/incompatible.md +++ b/docs/docs/en/guide/upgrade/incompatible.md @@ -11,3 +11,4 @@ This document records the incompatible updates between each version. You need to * Copy and import workflow without 'copy' suffix [#10607](https://github.com/apache/dolphinscheduler/pull/10607) * Use semicolon as default sql segment separator [#10869](https://github.com/apache/dolphinscheduler/pull/10869) + diff --git a/docs/docs/zh/about/hardware.md b/docs/docs/zh/about/hardware.md index 2ad98f4a6a09..bba384925ee5 100644 --- a/docs/docs/zh/about/hardware.md +++ b/docs/docs/zh/about/hardware.md @@ -47,4 +47,5 @@ DolphinScheduler正常运行提供如下的网络端口配置： DolphinScheduler 推荐 Chrome 以及使用 Chromium 内核的较新版本浏览器访问前端可视化操作界面 ## 时钟同步 -为避免可能影响任务执行的内部集群通信问题，请确保所有集群节点上的时钟与公共时钟源同步，例如使用 Chrony 和/或 NTP。 同步时间确保集群中的每个节点都有相同的时间 \ No newline at end of file + +为避免可能影响任务执行的内部集群通信问题，请确保所有集群节点上的时钟与公共时钟源同步，例如使用 Chrony 和/或 NTP。 同步时间确保集群中的每个节点都有相同的时间 diff --git a/docs/docs/zh/contribute/join/become-a-committer.md b/docs/docs/zh/contribute/join/become-a-committer.md index 720b31d6f165..a1b58499720e 100644 --- a/docs/docs/zh/contribute/join/become-a-committer.md +++ b/docs/docs/zh/contribute/join/become-a-committer.md @@ -28,6 +28,7 @@ PMC会定期基于活跃贡献者们对Dolphinscheduler的贡献从他们中提 发起提名的PMC成员负责向新的committer发出邀请，并指导他/她建立ASF精神思想。 PMC成员应使用以下模板向新的committer发送一封电子邮件: + ``` To: @gmail.com Cc: private@dolphinscheduler.apache.org diff --git a/docs/docs/zh/guide/datasource/oracle.md b/docs/docs/zh/guide/datasource/oracle.md index 581b0d8dd8e7..de5f07086f11 100644 --- a/docs/docs/zh/guide/datasource/oracle.md +++ b/docs/docs/zh/guide/datasource/oracle.md @@ -12,3 +12,4 @@ - 数据库名：输入连接 ORACLE 的 ServiceName 或 SID - Jdbc 连接参数：用于 ORACLE 连接的参数设置，以 JSON 形式填写 - 比如 `{"schema": "abc"}` 指定使用 `username` 用户下的 `abc` 数据库 + diff --git a/docs/docs/zh/guide/parameter/built-in.md b/docs/docs/zh/guide/parameter/built-in.md index 4838c382d987..abf5ade78945 100644 --- a/docs/docs/zh/guide/parameter/built-in.md +++ b/docs/docs/zh/guide/parameter/built-in.md @@ -29,33 +29,34 @@ - 也可以通过以下两种方式： - 1.使用add_months()函数，该函数用于加减月份， - 第一个入口参数为[yyyyMMdd]，表示返回时间的格式 - 第二个入口参数为月份偏移量，表示加减多少个月 - * 后 N 年：$[add_months(yyyyMMdd,12*N)] - * 前 N 年：$[add_months(yyyyMMdd,-12*N)] - * 后 N 月：$[add_months(yyyyMMdd,N)] - * 前 N 月：$[add_months(yyyyMMdd,-N)] - ******************************************* - 2.直接加减数字 - 在自定义格式后直接“+/-”数字 - * 后 N 周：$[yyyyMMdd+7*N] - * 前 N 周：$[yyyyMMdd-7*N] - * 后 N 天：$[yyyyMMdd+N] - * 前 N 天：$[yyyyMMdd-N] - * 后 N 小时：$[HHmmss+N/24] - * 前 N 小时：$[HHmmss-N/24] - * 后 N 分钟：$[HHmmss+N/24/60] - * 前 N 分钟：$[HHmmss-N/24/60] - ******************************************* - 3.业务属性方式 - 在自定义格式后直接“+/-”数字 - 支持日志格式：所有日期表达式，例如：yyyy-MM-dd/yyyyMMddHHmmss - * 当天：$[this_day(yyyy-MM-dd)]，如：2022-08-26 => 2022-08-26 - * 昨天：$[last_day(yyyy-MM-dd)]，如：2022-08-26 => 2022-08-25 - * 年的第N周，以周一为起点：$[year_week(yyyy-MM-dd)]，如：2022-08-26 => 2022-34 - * 年的第N周，以周N为起点：$[year_week(yyyy-MM-dd,N)]，如：N=5时 2022-08-26 => 2022-35 - * 前(-)/后(+) N 月第一天：$[month_first_day(yyyy-MM-dd,-N)]，如：N=1时 2022-08-26 => 2022-07-01 - * 前(-)/后(+) N 月最后一天：$[month_last_day(yyyy-MM-dd,-N)]，如：N=1时 2022-08-28 => 2022-07-31 - * 前(-)/后(+) N 周的周一：$[week_first_day(yyyy-MM-dd,-N)]，如：N=1 2022-08-26 => 2022-08-15 - * 前(-)/后(+) N 周的周日：$[week_last_day(yyyy-MM-dd,-N)]，如：N=1 2022-08-26 => 2022-08-21 + 1.使用add_months()函数，该函数用于加减月份， + 第一个入口参数为[yyyyMMdd]，表示返回时间的格式 + 第二个入口参数为月份偏移量，表示加减多少个月 + * 后 N 年：$[add_months(yyyyMMdd,12*N)] + * 前 N 年：$[add_months(yyyyMMdd,-12*N)] + * 后 N 月：$[add_months(yyyyMMdd,N)] + * 前 N 月：$[add_months(yyyyMMdd,-N)] + ******************************************* + 2.直接加减数字 + 在自定义格式后直接“+/-”数字 + * 后 N 周：$[yyyyMMdd+7*N] + * 前 N 周：$[yyyyMMdd-7*N] + * 后 N 天：$[yyyyMMdd+N] + * 前 N 天：$[yyyyMMdd-N] + * 后 N 小时：$[HHmmss+N/24] + * 前 N 小时：$[HHmmss-N/24] + * 后 N 分钟：$[HHmmss+N/24/60] + * 前 N 分钟：$[HHmmss-N/24/60] + ******************************************* + 3.业务属性方式 + 在自定义格式后直接“+/-”数字 + 支持日志格式：所有日期表达式，例如：yyyy-MM-dd/yyyyMMddHHmmss + * 当天：$[this_day(yyyy-MM-dd)]，如：2022-08-26 => 2022-08-26 + * 昨天：$[last_day(yyyy-MM-dd)]，如：2022-08-26 => 2022-08-25 + * 年的第N周，以周一为起点：$[year_week(yyyy-MM-dd)]，如：2022-08-26 => 2022-34 + * 年的第N周，以周N为起点：$[year_week(yyyy-MM-dd,N)]，如：N=5时 2022-08-26 => 2022-35 + * 前(-)/后(+) N 月第一天：$[month_first_day(yyyy-MM-dd,-N)]，如：N=1时 2022-08-26 => 2022-07-01 + * 前(-)/后(+) N 月最后一天：$[month_last_day(yyyy-MM-dd,-N)]，如：N=1时 2022-08-28 => 2022-07-31 + * 前(-)/后(+) N 周的周一：$[week_first_day(yyyy-MM-dd,-N)]，如：N=1 2022-08-26 => 2022-08-15 + * 前(-)/后(+) N 周的周日：$[week_last_day(yyyy-MM-dd,-N)]，如：N=1 2022-08-26 => 2022-08-21 + diff --git a/docs/docs/zh/guide/upgrade/incompatible.md b/docs/docs/zh/guide/upgrade/incompatible.md index 0bc0a7e675f7..4e294f954d10 100644 --- a/docs/docs/zh/guide/upgrade/incompatible.md +++ b/docs/docs/zh/guide/upgrade/incompatible.md @@ -11,3 +11,4 @@ * Copy and import workflow without 'copy' suffix [#10607](https://github.com/apache/dolphinscheduler/pull/10607) * Use semicolon as default sql segment separator [#10869](https://github.com/apache/dolphinscheduler/pull/10869) + diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-api/pom.xml b/dolphinscheduler-alert/dolphinscheduler-alert-api/pom.xml index 45ec5268598a..9719689f3902 100644 --- a/dolphinscheduler-alert/dolphinscheduler-alert-api/pom.xml +++ b/dolphinscheduler-alert/dolphinscheduler-alert-api/pom.xml @@ -17,22 +17,21 @@ ~ specific language governing permissions and limitations ~ under the License. --> - - + 4.0.0 - dolphinscheduler-alert org.apache.dolphinscheduler + dolphinscheduler-alert dev-SNAPSHOT - 4.0.0 dolphinscheduler-alert-api org.apache.dolphinscheduler - dolphinscheduler-spi + dolphinscheduler-spi + diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-api/src/main/java/org/apache/dolphinscheduler/alert/api/AlertChannel.java b/dolphinscheduler-alert/dolphinscheduler-alert-api/src/main/java/org/apache/dolphinscheduler/alert/api/AlertChannel.java index b461555a0ed5..530a5483425d 100644 --- a/dolphinscheduler-alert/dolphinscheduler-alert-api/src/main/java/org/apache/dolphinscheduler/alert/api/AlertChannel.java +++ b/dolphinscheduler-alert/dolphinscheduler-alert-api/src/main/java/org/apache/dolphinscheduler/alert/api/AlertChannel.java @@ -25,6 +25,7 @@ * alert channel for sending alerts */ public interface AlertChannel { + /** * process and send alert * diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-api/src/main/java/org/apache/dolphinscheduler/alert/api/AlertChannelFactory.java b/dolphinscheduler-alert/dolphinscheduler-alert-api/src/main/java/org/apache/dolphinscheduler/alert/api/AlertChannelFactory.java index 0f9878485c83..55097ee3af15 100644 --- a/dolphinscheduler-alert/dolphinscheduler-alert-api/src/main/java/org/apache/dolphinscheduler/alert/api/AlertChannelFactory.java +++ b/dolphinscheduler-alert/dolphinscheduler-alert-api/src/main/java/org/apache/dolphinscheduler/alert/api/AlertChannelFactory.java @@ -20,8 +20,8 @@ package org.apache.dolphinscheduler.alert.api; import org.apache.dolphinscheduler.spi.params.base.PluginParams; -import org.apache.dolphinscheduler.spi.plugin.SPIIdentify; import org.apache.dolphinscheduler.spi.plugin.PrioritySPI; +import org.apache.dolphinscheduler.spi.plugin.SPIIdentify; import java.util.List; @@ -29,6 +29,7 @@ * alert channel factory */ public interface AlertChannelFactory extends PrioritySPI { + /** * Returns the name of the alert channel * diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-api/src/main/java/org/apache/dolphinscheduler/alert/api/AlertConstants.java b/dolphinscheduler-alert/dolphinscheduler-alert-api/src/main/java/org/apache/dolphinscheduler/alert/api/AlertConstants.java index 6c0848fc9d3f..a94a31a8a926 100644 --- a/dolphinscheduler-alert/dolphinscheduler-alert-api/src/main/java/org/apache/dolphinscheduler/alert/api/AlertConstants.java +++ b/dolphinscheduler-alert/dolphinscheduler-alert-api/src/main/java/org/apache/dolphinscheduler/alert/api/AlertConstants.java @@ -20,6 +20,7 @@ package org.apache.dolphinscheduler.alert.api; public final class AlertConstants { + /** * the field name of alert show type **/ diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-api/src/main/java/org/apache/dolphinscheduler/alert/api/ShowType.java b/dolphinscheduler-alert/dolphinscheduler-alert-api/src/main/java/org/apache/dolphinscheduler/alert/api/ShowType.java index 95dccae7ebe8..46b87320c616 100644 --- a/dolphinscheduler-alert/dolphinscheduler-alert-api/src/main/java/org/apache/dolphinscheduler/alert/api/ShowType.java +++ b/dolphinscheduler-alert/dolphinscheduler-alert-api/src/main/java/org/apache/dolphinscheduler/alert/api/ShowType.java @@ -25,6 +25,7 @@ @AllArgsConstructor @Getter public enum ShowType { + /** * 0 TABLE; * 1 TEXT; diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-dingtalk/pom.xml b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-dingtalk/pom.xml index 34b1ef3335d4..5ce2f6d9d191 100644 --- a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-dingtalk/pom.xml +++ b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-dingtalk/pom.xml @@ -15,15 +15,14 @@ ~ See the License for the specific language governing permissions and ~ limitations under the License. --> - + 4.0.0 - dolphinscheduler-alert-plugins org.apache.dolphinscheduler + dolphinscheduler-alert-plugins dev-SNAPSHOT - 4.0.0 dolphinscheduler-alert-dingtalk jar diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-dingtalk/src/main/java/org/apache/dolphinscheduler/plugin/alert/dingtalk/DingTalkAlertChannelFactory.java b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-dingtalk/src/main/java/org/apache/dolphinscheduler/plugin/alert/dingtalk/DingTalkAlertChannelFactory.java index 235f61c4d394..bfa694cd28a4 100644 --- a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-dingtalk/src/main/java/org/apache/dolphinscheduler/plugin/alert/dingtalk/DingTalkAlertChannelFactory.java +++ b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-dingtalk/src/main/java/org/apache/dolphinscheduler/plugin/alert/dingtalk/DingTalkAlertChannelFactory.java @@ -38,6 +38,7 @@ @AutoService(AlertChannelFactory.class) public final class DingTalkAlertChannelFactory implements AlertChannelFactory { + @Override public String name() { return "DingTalk"; @@ -67,8 +68,10 @@ public List params() { RadioParam msgTypeParam = RadioParam .newBuilder(DingTalkParamsConstants.NAME_DING_TALK_MSG_TYPE, DingTalkParamsConstants.DING_TALK_MSG_TYPE) - .addParamsOptions(new ParamsOptions(DingTalkParamsConstants.DING_TALK_MSG_TYPE_TEXT, DingTalkParamsConstants.DING_TALK_MSG_TYPE_TEXT, false)) - .addParamsOptions(new ParamsOptions(DingTalkParamsConstants.DING_TALK_MSG_TYPE_MARKDOWN, DingTalkParamsConstants.DING_TALK_MSG_TYPE_MARKDOWN, false)) + .addParamsOptions(new ParamsOptions(DingTalkParamsConstants.DING_TALK_MSG_TYPE_TEXT, + DingTalkParamsConstants.DING_TALK_MSG_TYPE_TEXT, false)) + .addParamsOptions(new ParamsOptions(DingTalkParamsConstants.DING_TALK_MSG_TYPE_MARKDOWN, + DingTalkParamsConstants.DING_TALK_MSG_TYPE_MARKDOWN, false)) .setValue(DingTalkParamsConstants.DING_TALK_MSG_TYPE_TEXT) .addValidate(Validate.newBuilder() .setRequired(false) @@ -76,13 +79,15 @@ public List params() { .build(); InputParam atMobilesParam = InputParam - .newBuilder(DingTalkParamsConstants.NAME_DING_TALK_AT_MOBILES, DingTalkParamsConstants.DING_TALK_AT_MOBILES) + .newBuilder(DingTalkParamsConstants.NAME_DING_TALK_AT_MOBILES, + DingTalkParamsConstants.DING_TALK_AT_MOBILES) .addValidate(Validate.newBuilder() .setRequired(false) .build()) .build(); InputParam atUserIdsParam = InputParam - .newBuilder(DingTalkParamsConstants.NAME_DING_TALK_AT_USERIDS, DingTalkParamsConstants.DING_TALK_AT_USERIDS) + .newBuilder(DingTalkParamsConstants.NAME_DING_TALK_AT_USERIDS, + DingTalkParamsConstants.DING_TALK_AT_USERIDS) .addValidate(Validate.newBuilder() .setRequired(false) .build()) @@ -98,7 +103,8 @@ public List params() { .build(); RadioParam isEnableProxy = RadioParam - .newBuilder(DingTalkParamsConstants.NAME_DING_TALK_PROXY_ENABLE, DingTalkParamsConstants.DING_TALK_PROXY_ENABLE) + .newBuilder(DingTalkParamsConstants.NAME_DING_TALK_PROXY_ENABLE, + DingTalkParamsConstants.DING_TALK_PROXY_ENABLE) .addParamsOptions(new ParamsOptions(STRING_YES, STRING_TRUE, false)) .addParamsOptions(new ParamsOptions(STRING_NO, STRING_FALSE, false)) .setValue(STRING_FALSE) @@ -131,7 +137,8 @@ public List params() { .setPlaceholder("if enable use authentication, you need input password") .build(); - return Arrays.asList(webHookParam, keywordParam, secretParam, msgTypeParam, atMobilesParam, atUserIdsParam, isAtAll, isEnableProxy, proxyParam, portParam, userParam, passwordParam); + return Arrays.asList(webHookParam, keywordParam, secretParam, msgTypeParam, atMobilesParam, atUserIdsParam, + isAtAll, isEnableProxy, proxyParam, portParam, userParam, passwordParam); } @Override diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-dingtalk/src/main/java/org/apache/dolphinscheduler/plugin/alert/dingtalk/DingTalkParamsConstants.java b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-dingtalk/src/main/java/org/apache/dolphinscheduler/plugin/alert/dingtalk/DingTalkParamsConstants.java index e5b667ae30dc..d68a1bde5208 100644 --- a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-dingtalk/src/main/java/org/apache/dolphinscheduler/plugin/alert/dingtalk/DingTalkParamsConstants.java +++ b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-dingtalk/src/main/java/org/apache/dolphinscheduler/plugin/alert/dingtalk/DingTalkParamsConstants.java @@ -18,6 +18,7 @@ package org.apache.dolphinscheduler.plugin.alert.dingtalk; public final class DingTalkParamsConstants { + static final String DING_TALK_PROXY_ENABLE = "$t('isEnableProxy')"; static final String NAME_DING_TALK_PROXY_ENABLE = "IsEnableProxy"; diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-dingtalk/src/main/java/org/apache/dolphinscheduler/plugin/alert/dingtalk/DingTalkSender.java b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-dingtalk/src/main/java/org/apache/dolphinscheduler/plugin/alert/dingtalk/DingTalkSender.java index f8a824aec80f..617d0517949a 100644 --- a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-dingtalk/src/main/java/org/apache/dolphinscheduler/plugin/alert/dingtalk/DingTalkSender.java +++ b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-dingtalk/src/main/java/org/apache/dolphinscheduler/plugin/alert/dingtalk/DingTalkSender.java @@ -171,7 +171,8 @@ private String sendMsg(String title, String content) throws IOException { String msg = generateMsgJson(title, content); - HttpPost httpPost = constructHttpPost(org.apache.dolphinscheduler.spi.utils.StringUtils.isBlank(secret) ? url : generateSignedUrl(), msg); + HttpPost httpPost = constructHttpPost( + org.apache.dolphinscheduler.spi.utils.StringUtils.isBlank(secret) ? url : generateSignedUrl(), msg); CloseableHttpClient httpClient; if (Boolean.TRUE.equals(enableProxy)) { @@ -289,8 +290,12 @@ private void generateMarkdownMsg(String title, String content, Map items) { Map at = new HashMap<>(); - String[] atMobileArray = org.apache.dolphinscheduler.spi.utils.StringUtils.isNotBlank(atMobiles) ? atMobiles.split(",") : new String[0]; - String[] atUserArray = org.apache.dolphinscheduler.spi.utils.StringUtils.isNotBlank(atUserIds) ? atUserIds.split(",") : new String[0]; + String[] atMobileArray = + org.apache.dolphinscheduler.spi.utils.StringUtils.isNotBlank(atMobiles) ? atMobiles.split(",") + : new String[0]; + String[] atUserArray = + org.apache.dolphinscheduler.spi.utils.StringUtils.isNotBlank(atUserIds) ? atUserIds.split(",") + : new String[0]; boolean isAtAll = Objects.isNull(atAll) ? false : atAll; at.put("atMobiles", atMobileArray); @@ -313,7 +318,7 @@ private String generateSignedUrl() { Mac mac = Mac.getInstance("HmacSHA256"); mac.init(new SecretKeySpec(secret.getBytes("UTF-8"), "HmacSHA256")); byte[] signData = mac.doFinal(stringToSign.getBytes("UTF-8")); - sign = URLEncoder.encode(new String(Base64.encodeBase64(signData)),"UTF-8"); + sign = URLEncoder.encode(new String(Base64.encodeBase64(signData)), "UTF-8"); } catch (Exception e) { logger.error("generate sign error, message:{}", e); } @@ -321,6 +326,7 @@ private String generateSignedUrl() { } static final class DingTalkSendMsgResponse { + private Integer errcode; private String errmsg; @@ -378,7 +384,8 @@ public int hashCode() { @Override public String toString() { - return "DingTalkSender.DingTalkSendMsgResponse(errcode=" + this.getErrcode() + ", errmsg=" + this.getErrmsg() + ")"; + return "DingTalkSender.DingTalkSendMsgResponse(errcode=" + this.getErrcode() + ", errmsg=" + + this.getErrmsg() + ")"; } } } diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-email/pom.xml b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-email/pom.xml index 1ec23020226d..4669d7eac09c 100644 --- a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-email/pom.xml +++ b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-email/pom.xml @@ -15,15 +15,14 @@ ~ See the License for the specific language governing permissions and ~ limitations under the License. --> - + 4.0.0 - dolphinscheduler-alert-plugins org.apache.dolphinscheduler + dolphinscheduler-alert-plugins dev-SNAPSHOT - 4.0.0 dolphinscheduler-alert-email jar diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-email/src/main/java/org/apache/dolphinscheduler/plugin/alert/email/EmailAlertChannel.java b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-email/src/main/java/org/apache/dolphinscheduler/plugin/alert/email/EmailAlertChannel.java index 3c29ca24f574..663d251ea360 100644 --- a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-email/src/main/java/org/apache/dolphinscheduler/plugin/alert/email/EmailAlertChannel.java +++ b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-email/src/main/java/org/apache/dolphinscheduler/plugin/alert/email/EmailAlertChannel.java @@ -28,6 +28,7 @@ import org.slf4j.LoggerFactory; public final class EmailAlertChannel implements AlertChannel { + private static final Logger logger = LoggerFactory.getLogger(EmailAlertChannel.class); @Override diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-email/src/main/java/org/apache/dolphinscheduler/plugin/alert/email/EmailAlertChannelFactory.java b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-email/src/main/java/org/apache/dolphinscheduler/plugin/alert/email/EmailAlertChannelFactory.java index 2f5d6aae2826..8a38cfa37069 100644 --- a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-email/src/main/java/org/apache/dolphinscheduler/plugin/alert/email/EmailAlertChannelFactory.java +++ b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-email/src/main/java/org/apache/dolphinscheduler/plugin/alert/email/EmailAlertChannelFactory.java @@ -40,6 +40,7 @@ @AutoService(AlertChannelFactory.class) public final class EmailAlertChannelFactory implements AlertChannelFactory { + @Override public String name() { return "Email"; @@ -48,73 +49,88 @@ public String name() { @Override public List params() { List paramsList = new ArrayList<>(); - InputParam receivesParam = InputParam.newBuilder(MailParamsConstants.NAME_PLUGIN_DEFAULT_EMAIL_RECEIVERS, MailParamsConstants.PLUGIN_DEFAULT_EMAIL_RECEIVERS) - .setPlaceholder("please input receives") - .addValidate(Validate.newBuilder() - .setRequired(true) - .build()) - .build(); - - InputParam receiveCcsParam = InputParam.newBuilder(MailParamsConstants.NAME_PLUGIN_DEFAULT_EMAIL_RECEIVERCCS, MailParamsConstants.PLUGIN_DEFAULT_EMAIL_RECEIVERCCS) - .build(); - - InputParam mailSmtpHost = InputParam.newBuilder(MailParamsConstants.NAME_MAIL_SMTP_HOST, MailParamsConstants.MAIL_SMTP_HOST) - .addValidate(Validate.newBuilder().setRequired(true).build()) - .build(); - - InputParam mailSmtpPort = InputParam.newBuilder(MailParamsConstants.NAME_MAIL_SMTP_PORT, MailParamsConstants.MAIL_SMTP_PORT) - .setValue("25") - .addValidate(Validate.newBuilder() - .setRequired(true) - .build()) - .build(); - - InputParam mailSender = InputParam.newBuilder(MailParamsConstants.NAME_MAIL_SENDER, MailParamsConstants.MAIL_SENDER) - .addValidate(Validate.newBuilder().setRequired(true).build()) - .build(); - - RadioParam enableSmtpAuth = RadioParam.newBuilder(MailParamsConstants.NAME_MAIL_SMTP_AUTH, MailParamsConstants.MAIL_SMTP_AUTH) - .addParamsOptions(new ParamsOptions(STRING_YES, STRING_TRUE, false)) - .addParamsOptions(new ParamsOptions(STRING_NO, STRING_FALSE, false)) - .setValue(STRING_TRUE) - .addValidate(Validate.newBuilder().setRequired(true).build()) - .build(); + InputParam receivesParam = InputParam + .newBuilder(MailParamsConstants.NAME_PLUGIN_DEFAULT_EMAIL_RECEIVERS, + MailParamsConstants.PLUGIN_DEFAULT_EMAIL_RECEIVERS) + .setPlaceholder("please input receives") + .addValidate(Validate.newBuilder() + .setRequired(true) + .build()) + .build(); + + InputParam receiveCcsParam = InputParam + .newBuilder(MailParamsConstants.NAME_PLUGIN_DEFAULT_EMAIL_RECEIVERCCS, + MailParamsConstants.PLUGIN_DEFAULT_EMAIL_RECEIVERCCS) + .build(); + + InputParam mailSmtpHost = + InputParam.newBuilder(MailParamsConstants.NAME_MAIL_SMTP_HOST, MailParamsConstants.MAIL_SMTP_HOST) + .addValidate(Validate.newBuilder().setRequired(true).build()) + .build(); + + InputParam mailSmtpPort = + InputParam.newBuilder(MailParamsConstants.NAME_MAIL_SMTP_PORT, MailParamsConstants.MAIL_SMTP_PORT) + .setValue("25") + .addValidate(Validate.newBuilder() + .setRequired(true) + .build()) + .build(); + + InputParam mailSender = + InputParam.newBuilder(MailParamsConstants.NAME_MAIL_SENDER, MailParamsConstants.MAIL_SENDER) + .addValidate(Validate.newBuilder().setRequired(true).build()) + .build(); + + RadioParam enableSmtpAuth = + RadioParam.newBuilder(MailParamsConstants.NAME_MAIL_SMTP_AUTH, MailParamsConstants.MAIL_SMTP_AUTH) + .addParamsOptions(new ParamsOptions(STRING_YES, STRING_TRUE, false)) + .addParamsOptions(new ParamsOptions(STRING_NO, STRING_FALSE, false)) + .setValue(STRING_TRUE) + .addValidate(Validate.newBuilder().setRequired(true).build()) + .build(); InputParam mailUser = InputParam.newBuilder(MailParamsConstants.NAME_MAIL_USER, MailParamsConstants.MAIL_USER) - .setPlaceholder("if enable use authentication, you need input user") - .build(); - - PasswordParam mailPassword = PasswordParam.newBuilder(MailParamsConstants.NAME_MAIL_PASSWD, MailParamsConstants.MAIL_PASSWD) - .setPlaceholder("if enable use authentication, you need input password") - .build(); - - RadioParam enableTls = RadioParam.newBuilder(MailParamsConstants.NAME_MAIL_SMTP_STARTTLS_ENABLE, MailParamsConstants.MAIL_SMTP_STARTTLS_ENABLE) - .addParamsOptions(new ParamsOptions(STRING_YES, STRING_TRUE, false)) - .addParamsOptions(new ParamsOptions(STRING_NO, STRING_FALSE, false)) - .setValue(STRING_FALSE) - .addValidate(Validate.newBuilder().setRequired(true).build()) - .build(); - - RadioParam enableSsl = RadioParam.newBuilder(MailParamsConstants.NAME_MAIL_SMTP_SSL_ENABLE, MailParamsConstants.MAIL_SMTP_SSL_ENABLE) - .addParamsOptions(new ParamsOptions(STRING_YES, STRING_TRUE, false)) - .addParamsOptions(new ParamsOptions(STRING_NO, STRING_FALSE, false)) - .setValue(STRING_FALSE) - .addValidate(Validate.newBuilder().setRequired(true).build()) - .build(); - - InputParam sslTrust = InputParam.newBuilder(MailParamsConstants.NAME_MAIL_SMTP_SSL_TRUST, MailParamsConstants.MAIL_SMTP_SSL_TRUST) - .setValue("*") - .addValidate(Validate.newBuilder().setRequired(true).build()) - .build(); + .setPlaceholder("if enable use authentication, you need input user") + .build(); + + PasswordParam mailPassword = + PasswordParam.newBuilder(MailParamsConstants.NAME_MAIL_PASSWD, MailParamsConstants.MAIL_PASSWD) + .setPlaceholder("if enable use authentication, you need input password") + .build(); + + RadioParam enableTls = RadioParam + .newBuilder(MailParamsConstants.NAME_MAIL_SMTP_STARTTLS_ENABLE, + MailParamsConstants.MAIL_SMTP_STARTTLS_ENABLE) + .addParamsOptions(new ParamsOptions(STRING_YES, STRING_TRUE, false)) + .addParamsOptions(new ParamsOptions(STRING_NO, STRING_FALSE, false)) + .setValue(STRING_FALSE) + .addValidate(Validate.newBuilder().setRequired(true).build()) + .build(); + + RadioParam enableSsl = RadioParam + .newBuilder(MailParamsConstants.NAME_MAIL_SMTP_SSL_ENABLE, MailParamsConstants.MAIL_SMTP_SSL_ENABLE) + .addParamsOptions(new ParamsOptions(STRING_YES, STRING_TRUE, false)) + .addParamsOptions(new ParamsOptions(STRING_NO, STRING_FALSE, false)) + .setValue(STRING_FALSE) + .addValidate(Validate.newBuilder().setRequired(true).build()) + .build(); + + InputParam sslTrust = InputParam + .newBuilder(MailParamsConstants.NAME_MAIL_SMTP_SSL_TRUST, MailParamsConstants.MAIL_SMTP_SSL_TRUST) + .setValue("*") + .addValidate(Validate.newBuilder().setRequired(true).build()) + .build(); RadioParam showType = RadioParam.newBuilder(AlertConstants.NAME_SHOW_TYPE, AlertConstants.SHOW_TYPE) - .addParamsOptions(new ParamsOptions(ShowType.TABLE.getDescp(), ShowType.TABLE.getDescp(), false)) - .addParamsOptions(new ParamsOptions(ShowType.TEXT.getDescp(), ShowType.TEXT.getDescp(), false)) - .addParamsOptions(new ParamsOptions(ShowType.ATTACHMENT.getDescp(), ShowType.ATTACHMENT.getDescp(), false)) - .addParamsOptions(new ParamsOptions(ShowType.TABLE_ATTACHMENT.getDescp(), ShowType.TABLE_ATTACHMENT.getDescp(), false)) - .setValue(ShowType.TABLE.getDescp()) - .addValidate(Validate.newBuilder().setRequired(true).build()) - .build(); + .addParamsOptions(new ParamsOptions(ShowType.TABLE.getDescp(), ShowType.TABLE.getDescp(), false)) + .addParamsOptions(new ParamsOptions(ShowType.TEXT.getDescp(), ShowType.TEXT.getDescp(), false)) + .addParamsOptions( + new ParamsOptions(ShowType.ATTACHMENT.getDescp(), ShowType.ATTACHMENT.getDescp(), false)) + .addParamsOptions(new ParamsOptions(ShowType.TABLE_ATTACHMENT.getDescp(), + ShowType.TABLE_ATTACHMENT.getDescp(), false)) + .setValue(ShowType.TABLE.getDescp()) + .addValidate(Validate.newBuilder().setRequired(true).build()) + .build(); paramsList.add(receivesParam); paramsList.add(receiveCcsParam); diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-email/src/main/java/org/apache/dolphinscheduler/plugin/alert/email/EmailConstants.java b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-email/src/main/java/org/apache/dolphinscheduler/plugin/alert/email/EmailConstants.java index 4c6e1d37eea2..94eb4efa39bd 100644 --- a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-email/src/main/java/org/apache/dolphinscheduler/plugin/alert/email/EmailConstants.java +++ b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-email/src/main/java/org/apache/dolphinscheduler/plugin/alert/email/EmailConstants.java @@ -18,6 +18,7 @@ package org.apache.dolphinscheduler.plugin.alert.email; public final class EmailConstants { + public static final String XLS_FILE_PATH = "xls.file.path"; public static final String MAIL_TRANSPORT_PROTOCOL = "mail.transport.protocol"; @@ -38,19 +39,20 @@ public final class EmailConstants { public static final String TH_END = ""; - public static final String HTML_HEADER_PREFIX = "" - + "" - + "" - + "dolphinscheduler" - + "" - + "" - + "" - + "" - + " "; + public static final String HTML_HEADER_PREFIX = + "" + + "" + + "" + + "dolphinscheduler" + + "" + + "" + + "" + + "" + + "
"; public static final String TABLE_BODY_HTML_TAIL = "
"; diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-email/src/main/java/org/apache/dolphinscheduler/plugin/alert/email/ExcelUtils.java b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-email/src/main/java/org/apache/dolphinscheduler/plugin/alert/email/ExcelUtils.java index 6e02178f191e..fbb22cde6556 100644 --- a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-email/src/main/java/org/apache/dolphinscheduler/plugin/alert/email/ExcelUtils.java +++ b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-email/src/main/java/org/apache/dolphinscheduler/plugin/alert/email/ExcelUtils.java @@ -39,6 +39,7 @@ import org.slf4j.LoggerFactory; public final class ExcelUtils { + private static final int XLSX_WINDOW_ROW = 10000; private static final Logger logger = LoggerFactory.getLogger(ExcelUtils.class); @@ -74,32 +75,33 @@ public static void genExcelFile(String content, String title, String xlsFilePath for (Map.Entry en : headerMap.entrySet()) { headerList.add(en.getKey()); } - try (SXSSFWorkbook wb = new SXSSFWorkbook(XLSX_WINDOW_ROW); - FileOutputStream fos = new FileOutputStream(String.format("%s/%s.xlsx", xlsFilePath, title))) { + try ( + SXSSFWorkbook wb = new SXSSFWorkbook(XLSX_WINDOW_ROW); + FileOutputStream fos = new FileOutputStream(String.format("%s/%s.xlsx", xlsFilePath, title))) { // declare a workbook // generate a table Sheet sheet = wb.createSheet(); Row row = sheet.createRow(0); - //set the height of the first line + // set the height of the first line row.setHeight((short) 500); - //set Horizontal right + // set Horizontal right CellStyle cellStyle = wb.createCellStyle(); cellStyle.setAlignment(HorizontalAlignment.RIGHT); - //setting excel headers + // setting excel headers for (int i = 0; i < headerList.size(); i++) { Cell cell = row.createCell(i); cell.setCellStyle(cellStyle); cell.setCellValue(headerList.get(i)); } - //setting excel body + // setting excel body int rowIndex = 1; for (LinkedHashMap itemsMap : itemsList) { Object[] values = itemsMap.values().toArray(); row = sheet.createRow(rowIndex); - //setting excel body height + // setting excel body height row.setHeight((short) 500); rowIndex++; for (int j = 0; j < values.length; j++) { @@ -117,7 +119,7 @@ public static void genExcelFile(String content, String title, String xlsFilePath sheet.setColumnWidth(i, headerList.get(i).length() * 800); } - //setting file output + // setting file output wb.write(fos); wb.dispose(); } catch (Exception e) { diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-email/src/main/java/org/apache/dolphinscheduler/plugin/alert/email/MailParamsConstants.java b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-email/src/main/java/org/apache/dolphinscheduler/plugin/alert/email/MailParamsConstants.java index bf84fd634d61..5fa8c36da32e 100644 --- a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-email/src/main/java/org/apache/dolphinscheduler/plugin/alert/email/MailParamsConstants.java +++ b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-email/src/main/java/org/apache/dolphinscheduler/plugin/alert/email/MailParamsConstants.java @@ -18,6 +18,7 @@ package org.apache.dolphinscheduler.plugin.alert.email; public final class MailParamsConstants { + public static final String PLUGIN_DEFAULT_EMAIL_RECEIVERS = "$t('receivers')"; public static final String NAME_PLUGIN_DEFAULT_EMAIL_RECEIVERS = "receivers"; diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-email/src/main/java/org/apache/dolphinscheduler/plugin/alert/email/MailSender.java b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-email/src/main/java/org/apache/dolphinscheduler/plugin/alert/email/MailSender.java index a7305cf741eb..5f805f14adec 100644 --- a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-email/src/main/java/org/apache/dolphinscheduler/plugin/alert/email/MailSender.java +++ b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-email/src/main/java/org/apache/dolphinscheduler/plugin/alert/email/MailSender.java @@ -60,6 +60,7 @@ import com.sun.mail.smtp.SMTPProvider; public final class MailSender { + private static final Logger logger = LoggerFactory.getLogger(MailSender.class); private final List receivers; @@ -178,7 +179,7 @@ public AlertResult sendMails(List receivers, List receiverCcs, S } if (CollectionUtils.isNotEmpty(receiverCcs)) { - //cc + // cc for (String receiverCc : receiverCcs) { email.addCc(receiverCc); } @@ -188,12 +189,13 @@ public AlertResult sendMails(List receivers, List receiverCcs, S } catch (Exception e) { handleException(alertResult, e); } - } else if (showType.equals(ShowType.ATTACHMENT.getDescp()) || showType.equals(ShowType.TABLE_ATTACHMENT.getDescp())) { + } else if (showType.equals(ShowType.ATTACHMENT.getDescp()) + || showType.equals(ShowType.TABLE_ATTACHMENT.getDescp())) { try { String partContent = (showType.equals(ShowType.ATTACHMENT.getDescp()) - ? "Please see the attachment " + title + EmailConstants.EXCEL_SUFFIX_XLSX - : htmlTable(content, false)); + ? "Please see the attachment " + title + EmailConstants.EXCEL_SUFFIX_XLSX + : htmlTable(content, false)); attachment(title, content, partContent); @@ -294,6 +296,7 @@ private Session getSession() { props.setProperty(MailParamsConstants.MAIL_SMTP_SSL_TRUST, sslTrust); Authenticator auth = new Authenticator() { + @Override protected PasswordAuthentication getPasswordAuthentication() { // mail username and password @@ -309,7 +312,8 @@ protected PasswordAuthentication getPasswordAuthentication() { /** * attach content */ - private void attachContent(String title, String content, String partContent, MimeMessage msg) throws MessagingException, IOException { + private void attachContent(String title, String content, String partContent, + MimeMessage msg) throws MessagingException, IOException { /* * set receiverCc */ @@ -329,7 +333,8 @@ private void attachContent(String title, String content, String partContent, Mim MimeBodyPart part2 = new MimeBodyPart(); // add random uuid to filename to avoid potential issue String randomFilename = title + UUID.randomUUID(); - File file = new File(xlsFilePath + EmailConstants.SINGLE_SLASH + randomFilename + EmailConstants.EXCEL_SUFFIX_XLSX); + File file = + new File(xlsFilePath + EmailConstants.SINGLE_SLASH + randomFilename + EmailConstants.EXCEL_SUFFIX_XLSX); if (!file.getParentFile().exists()) { file.getParentFile().mkdirs(); } @@ -352,7 +357,8 @@ private void attachContent(String title, String content, String partContent, Mim /** * the string object map */ - private AlertResult getStringObjectMap(String title, String content, AlertResult alertResult, HtmlEmail email) throws EmailException { + private AlertResult getStringObjectMap(String title, String content, AlertResult alertResult, + HtmlEmail email) throws EmailException { /* * the subject of the message to be sent diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-email/src/main/java/org/apache/dolphinscheduler/plugin/alert/email/exception/AlertEmailException.java b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-email/src/main/java/org/apache/dolphinscheduler/plugin/alert/email/exception/AlertEmailException.java index 5b64d8de1efc..6b4577169275 100644 --- a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-email/src/main/java/org/apache/dolphinscheduler/plugin/alert/email/exception/AlertEmailException.java +++ b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-email/src/main/java/org/apache/dolphinscheduler/plugin/alert/email/exception/AlertEmailException.java @@ -18,6 +18,7 @@ package org.apache.dolphinscheduler.plugin.alert.email.exception; public class AlertEmailException extends RuntimeException { + public AlertEmailException(String errMsg) { super(errMsg); } diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-email/src/main/java/org/apache/dolphinscheduler/plugin/alert/email/template/DefaultHTMLTemplate.java b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-email/src/main/java/org/apache/dolphinscheduler/plugin/alert/email/template/DefaultHTMLTemplate.java index 4167e77235bb..8d2f1ab14ee4 100644 --- a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-email/src/main/java/org/apache/dolphinscheduler/plugin/alert/email/template/DefaultHTMLTemplate.java +++ b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-email/src/main/java/org/apache/dolphinscheduler/plugin/alert/email/template/DefaultHTMLTemplate.java @@ -33,15 +33,14 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.springframework.boot.configurationprocessor.json.JSONArray; +import org.springframework.boot.configurationprocessor.json.JSONException; +import org.springframework.boot.configurationprocessor.json.JSONTokener; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.node.ArrayNode; import com.fasterxml.jackson.databind.node.ObjectNode; -import org.springframework.boot.configurationprocessor.json.JSONArray; -import org.springframework.boot.configurationprocessor.json.JSONException; -import org.springframework.boot.configurationprocessor.json.JSONTokener; - public class DefaultHTMLTemplate implements AlertTemplate { public static final Logger logger = LoggerFactory.getLogger(DefaultHTMLTemplate.class); @@ -55,7 +54,8 @@ public String getMessageFromTemplate(String content, ShowType showType, boolean case TEXT: return getTextTypeMessage(content); default: - throw new IllegalArgumentException(String.format("not support showType: %s in DefaultHTMLTemplate", showType)); + throw new IllegalArgumentException( + String.format("not support showType: %s in DefaultHTMLTemplate", showType)); } } diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-feishu/pom.xml b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-feishu/pom.xml index 93450e08f6b2..1a56cdc0942b 100644 --- a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-feishu/pom.xml +++ b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-feishu/pom.xml @@ -15,15 +15,14 @@ ~ See the License for the specific language governing permissions and ~ limitations under the License. --> - + 4.0.0 - dolphinscheduler-alert-plugins org.apache.dolphinscheduler + dolphinscheduler-alert-plugins dev-SNAPSHOT - 4.0.0 dolphinscheduler-alert-feishu jar diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-feishu/src/main/java/org/apache/dolphinscheduler/plugin/alert/feishu/FeiShuAlertChannel.java b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-feishu/src/main/java/org/apache/dolphinscheduler/plugin/alert/feishu/FeiShuAlertChannel.java index 509eebe6157c..8959c8aaec13 100644 --- a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-feishu/src/main/java/org/apache/dolphinscheduler/plugin/alert/feishu/FeiShuAlertChannel.java +++ b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-feishu/src/main/java/org/apache/dolphinscheduler/plugin/alert/feishu/FeiShuAlertChannel.java @@ -25,6 +25,7 @@ import java.util.Map; public final class FeiShuAlertChannel implements AlertChannel { + @Override public AlertResult process(AlertInfo alertInfo) { AlertData alertData = alertInfo.getAlertData(); diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-feishu/src/main/java/org/apache/dolphinscheduler/plugin/alert/feishu/FeiShuAlertChannelFactory.java b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-feishu/src/main/java/org/apache/dolphinscheduler/plugin/alert/feishu/FeiShuAlertChannelFactory.java index da78b9f39b5f..7f6e65ca93f9 100644 --- a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-feishu/src/main/java/org/apache/dolphinscheduler/plugin/alert/feishu/FeiShuAlertChannelFactory.java +++ b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-feishu/src/main/java/org/apache/dolphinscheduler/plugin/alert/feishu/FeiShuAlertChannelFactory.java @@ -38,6 +38,7 @@ @AutoService(AlertChannelFactory.class) public final class FeiShuAlertChannelFactory implements AlertChannelFactory { + @Override public String name() { return "Feishu"; @@ -45,39 +46,44 @@ public String name() { @Override public List params() { - InputParam webHookParam = InputParam.newBuilder(FeiShuParamsConstants.NAME_WEB_HOOK, FeiShuParamsConstants.WEB_HOOK) - .addValidate(Validate.newBuilder() - .setRequired(true) - .build()) - .build(); + InputParam webHookParam = + InputParam.newBuilder(FeiShuParamsConstants.NAME_WEB_HOOK, FeiShuParamsConstants.WEB_HOOK) + .addValidate(Validate.newBuilder() + .setRequired(true) + .build()) + .build(); RadioParam isEnableProxy = - RadioParam.newBuilder(FeiShuParamsConstants.NAME_FEI_SHU_PROXY_ENABLE, FeiShuParamsConstants.FEI_SHU_PROXY_ENABLE) - .addParamsOptions(new ParamsOptions(STRING_YES, STRING_TRUE, false)) - .addParamsOptions(new ParamsOptions(STRING_NO, STRING_FALSE, false)) - .setValue(STRING_TRUE) - .addValidate(Validate.newBuilder() - .setRequired(false) - .build()) - .build(); + RadioParam + .newBuilder(FeiShuParamsConstants.NAME_FEI_SHU_PROXY_ENABLE, + FeiShuParamsConstants.FEI_SHU_PROXY_ENABLE) + .addParamsOptions(new ParamsOptions(STRING_YES, STRING_TRUE, false)) + .addParamsOptions(new ParamsOptions(STRING_NO, STRING_FALSE, false)) + .setValue(STRING_TRUE) + .addValidate(Validate.newBuilder() + .setRequired(false) + .build()) + .build(); InputParam proxyParam = - InputParam.newBuilder(FeiShuParamsConstants.NAME_FEI_SHU_PROXY, FeiShuParamsConstants.FEI_SHU_PROXY) - .addValidate(Validate.newBuilder() - .setRequired(false).build()) - .build(); + InputParam.newBuilder(FeiShuParamsConstants.NAME_FEI_SHU_PROXY, FeiShuParamsConstants.FEI_SHU_PROXY) + .addValidate(Validate.newBuilder() + .setRequired(false).build()) + .build(); - InputParam portParam = InputParam.newBuilder(FeiShuParamsConstants.NAME_FEI_SHU_PORT, FeiShuParamsConstants.FEI_SHU_PORT) - .addValidate(Validate.newBuilder() - .setRequired(false).build()) - .build(); + InputParam portParam = + InputParam.newBuilder(FeiShuParamsConstants.NAME_FEI_SHU_PORT, FeiShuParamsConstants.FEI_SHU_PORT) + .addValidate(Validate.newBuilder() + .setRequired(false).build()) + .build(); InputParam userParam = - InputParam.newBuilder(FeiShuParamsConstants.NAME_FEI_SHU_USER, FeiShuParamsConstants.FEI_SHU_USER) - .addValidate(Validate.newBuilder() - .setRequired(false).build()) - .build(); - PasswordParam passwordParam = PasswordParam.newBuilder(FeiShuParamsConstants.NAME_FEI_SHU_PASSWORD, FeiShuParamsConstants.FEI_SHU_PASSWORD) - .setPlaceholder("if enable use authentication, you need input password") - .build(); + InputParam.newBuilder(FeiShuParamsConstants.NAME_FEI_SHU_USER, FeiShuParamsConstants.FEI_SHU_USER) + .addValidate(Validate.newBuilder() + .setRequired(false).build()) + .build(); + PasswordParam passwordParam = PasswordParam + .newBuilder(FeiShuParamsConstants.NAME_FEI_SHU_PASSWORD, FeiShuParamsConstants.FEI_SHU_PASSWORD) + .setPlaceholder("if enable use authentication, you need input password") + .build(); return Arrays.asList(webHookParam, isEnableProxy, proxyParam, portParam, userParam, passwordParam); diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-feishu/src/main/java/org/apache/dolphinscheduler/plugin/alert/feishu/FeiShuParamsConstants.java b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-feishu/src/main/java/org/apache/dolphinscheduler/plugin/alert/feishu/FeiShuParamsConstants.java index aa743903e956..0d2c3912f88d 100644 --- a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-feishu/src/main/java/org/apache/dolphinscheduler/plugin/alert/feishu/FeiShuParamsConstants.java +++ b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-feishu/src/main/java/org/apache/dolphinscheduler/plugin/alert/feishu/FeiShuParamsConstants.java @@ -18,6 +18,7 @@ package org.apache.dolphinscheduler.plugin.alert.feishu; public final class FeiShuParamsConstants { + public static final String FEI_SHU_PROXY_ENABLE = "$t('isEnableProxy')"; static final String WEB_HOOK = "$t('webhook')"; static final String NAME_WEB_HOOK = "WebHook"; diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-feishu/src/main/java/org/apache/dolphinscheduler/plugin/alert/feishu/FeiShuSender.java b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-feishu/src/main/java/org/apache/dolphinscheduler/plugin/alert/feishu/FeiShuSender.java index a5d11c368497..ba245c3ae26e 100644 --- a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-feishu/src/main/java/org/apache/dolphinscheduler/plugin/alert/feishu/FeiShuSender.java +++ b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-feishu/src/main/java/org/apache/dolphinscheduler/plugin/alert/feishu/FeiShuSender.java @@ -41,6 +41,7 @@ import com.fasterxml.jackson.annotation.JsonProperty; public final class FeiShuSender { + private static final Logger logger = LoggerFactory.getLogger(FeiShuSender.class); private final String url; private final Boolean enableProxy; @@ -98,7 +99,8 @@ public static AlertResult checkSendFeiShuSendMsgResult(String result) { return alertResult; } alertResult.setMessage(String.format("alert send fei shu msg error : %s", sendMsgResponse.getStatusMessage())); - logger.info("alert send fei shu msg error : {} ,Extra : {} ", sendMsgResponse.getStatusMessage(), sendMsgResponse.getExtra()); + logger.info("alert send fei shu msg error : {} ,Extra : {} ", sendMsgResponse.getStatusMessage(), + sendMsgResponse.getExtra()); return alertResult; } @@ -164,7 +166,8 @@ private String sendMsg(AlertData alertData) throws IOException { } finally { response.close(); } - logger.info("Fei Shu send title :{} ,content :{}, resp: {}", alertData.getTitle(), alertData.getContent(), resp); + logger.info("Fei Shu send title :{} ,content :{}, resp: {}", alertData.getTitle(), alertData.getContent(), + resp); return resp; } finally { httpClient.close(); @@ -172,6 +175,7 @@ private String sendMsg(AlertData alertData) throws IOException { } static final class FeiShuSendMsgResponse { + @JsonProperty("Extra") private String extra; @JsonProperty("StatusCode") @@ -229,7 +233,8 @@ public boolean equals(final Object o) { } final Object this$statusMessage = this.getStatusMessage(); final Object other$statusMessage = other.getStatusMessage(); - if (this$statusMessage == null ? other$statusMessage != null : !this$statusMessage.equals(other$statusMessage)) { + if (this$statusMessage == null ? other$statusMessage != null + : !this$statusMessage.equals(other$statusMessage)) { return false; } return true; @@ -248,7 +253,8 @@ public int hashCode() { } public String toString() { - return "FeiShuSender.FeiShuSendMsgResponse(extra=" + this.getExtra() + ", statusCode=" + this.getStatusCode() + ", statusMessage=" + this.getStatusMessage() + ")"; + return "FeiShuSender.FeiShuSendMsgResponse(extra=" + this.getExtra() + ", statusCode=" + + this.getStatusCode() + ", statusMessage=" + this.getStatusMessage() + ")"; } } } diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-feishu/src/main/java/org/apache/dolphinscheduler/plugin/alert/feishu/HttpRequestUtil.java b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-feishu/src/main/java/org/apache/dolphinscheduler/plugin/alert/feishu/HttpRequestUtil.java index eca9b07d15ba..2ba373df0871 100644 --- a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-feishu/src/main/java/org/apache/dolphinscheduler/plugin/alert/feishu/HttpRequestUtil.java +++ b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-feishu/src/main/java/org/apache/dolphinscheduler/plugin/alert/feishu/HttpRequestUtil.java @@ -29,11 +29,13 @@ import org.apache.http.impl.client.HttpClients; public final class HttpRequestUtil { + private HttpRequestUtil() { throw new UnsupportedOperationException("This is a utility class and cannot be instantiated"); } - public static CloseableHttpClient getHttpClient(boolean enableProxy, String proxy, Integer port, String user, String password) { + public static CloseableHttpClient getHttpClient(boolean enableProxy, String proxy, Integer port, String user, + String password) { if (enableProxy) { HttpHost httpProxy = new HttpHost(proxy, port); CredentialsProvider provider = new BasicCredentialsProvider(); diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-http/pom.xml b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-http/pom.xml index 007c3cd83511..c83356af4dc2 100644 --- a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-http/pom.xml +++ b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-http/pom.xml @@ -15,15 +15,14 @@ ~ See the License for the specific language governing permissions and ~ limitations under the License. --> - + 4.0.0 - dolphinscheduler-alert-plugins org.apache.dolphinscheduler + dolphinscheduler-alert-plugins dev-SNAPSHOT - 4.0.0 dolphinscheduler-alert-http jar diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-http/src/main/java/org/apache/dolphinscheduler/plugin/alert/http/HttpAlertChannel.java b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-http/src/main/java/org/apache/dolphinscheduler/plugin/alert/http/HttpAlertChannel.java index 14b416785dde..944762f13f22 100644 --- a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-http/src/main/java/org/apache/dolphinscheduler/plugin/alert/http/HttpAlertChannel.java +++ b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-http/src/main/java/org/apache/dolphinscheduler/plugin/alert/http/HttpAlertChannel.java @@ -25,6 +25,7 @@ import java.util.Map; public final class HttpAlertChannel implements AlertChannel { + @Override public AlertResult process(AlertInfo alertInfo) { AlertData alertData = alertInfo.getAlertData(); diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-http/src/main/java/org/apache/dolphinscheduler/plugin/alert/http/HttpAlertChannelFactory.java b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-http/src/main/java/org/apache/dolphinscheduler/plugin/alert/http/HttpAlertChannelFactory.java index c18ebe4454dc..0a54b4235727 100644 --- a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-http/src/main/java/org/apache/dolphinscheduler/plugin/alert/http/HttpAlertChannelFactory.java +++ b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-http/src/main/java/org/apache/dolphinscheduler/plugin/alert/http/HttpAlertChannelFactory.java @@ -30,6 +30,7 @@ @AutoService(AlertChannelFactory.class) public final class HttpAlertChannelFactory implements AlertChannelFactory { + @Override public String name() { return "Http"; @@ -39,39 +40,43 @@ public String name() { public List params() { InputParam url = InputParam.newBuilder(HttpAlertConstants.NAME_URL, HttpAlertConstants.URL) - .setPlaceholder("input request URL") - .addValidate(Validate.newBuilder() - .setRequired(true) - .build()) - .build(); + .setPlaceholder("input request URL") + .addValidate(Validate.newBuilder() + .setRequired(true) + .build()) + .build(); - InputParam headerParams = InputParam.newBuilder(HttpAlertConstants.NAME_HEADER_PARAMS, HttpAlertConstants.HEADER_PARAMS) - .setPlaceholder("input request headers as JSON format ") - .addValidate(Validate.newBuilder() - .setRequired(true) - .build()) - .build(); + InputParam headerParams = + InputParam.newBuilder(HttpAlertConstants.NAME_HEADER_PARAMS, HttpAlertConstants.HEADER_PARAMS) + .setPlaceholder("input request headers as JSON format ") + .addValidate(Validate.newBuilder() + .setRequired(true) + .build()) + .build(); - InputParam bodyParams = InputParam.newBuilder(HttpAlertConstants.NAME_BODY_PARAMS, HttpAlertConstants.BODY_PARAMS) - .setPlaceholder("input request body as JSON format ") - .addValidate(Validate.newBuilder() - .setRequired(false) - .build()) - .build(); + InputParam bodyParams = + InputParam.newBuilder(HttpAlertConstants.NAME_BODY_PARAMS, HttpAlertConstants.BODY_PARAMS) + .setPlaceholder("input request body as JSON format ") + .addValidate(Validate.newBuilder() + .setRequired(false) + .build()) + .build(); - InputParam contentField = InputParam.newBuilder(HttpAlertConstants.NAME_CONTENT_FIELD, HttpAlertConstants.CONTENT_FIELD) - .setPlaceholder("input alert msg field name") - .addValidate(Validate.newBuilder() - .setRequired(true) - .build()) - .build(); + InputParam contentField = + InputParam.newBuilder(HttpAlertConstants.NAME_CONTENT_FIELD, HttpAlertConstants.CONTENT_FIELD) + .setPlaceholder("input alert msg field name") + .addValidate(Validate.newBuilder() + .setRequired(true) + .build()) + .build(); - InputParam requestType = InputParam.newBuilder(HttpAlertConstants.NAME_REQUEST_TYPE, HttpAlertConstants.REQUEST_TYPE) - .setPlaceholder("input request type POST or GET") - .addValidate(Validate.newBuilder() - .setRequired(true) - .build()) - .build(); + InputParam requestType = + InputParam.newBuilder(HttpAlertConstants.NAME_REQUEST_TYPE, HttpAlertConstants.REQUEST_TYPE) + .setPlaceholder("input request type POST or GET") + .addValidate(Validate.newBuilder() + .setRequired(true) + .build()) + .build(); return Arrays.asList(url, requestType, headerParams, bodyParams, contentField); } diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-http/src/main/java/org/apache/dolphinscheduler/plugin/alert/http/HttpAlertConstants.java b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-http/src/main/java/org/apache/dolphinscheduler/plugin/alert/http/HttpAlertConstants.java index 6bbf0bfbe0b0..22db292d5fcb 100644 --- a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-http/src/main/java/org/apache/dolphinscheduler/plugin/alert/http/HttpAlertConstants.java +++ b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-http/src/main/java/org/apache/dolphinscheduler/plugin/alert/http/HttpAlertConstants.java @@ -18,6 +18,7 @@ package org.apache.dolphinscheduler.plugin.alert.http; public final class HttpAlertConstants { + public static final String URL = "$t('url')"; public static final String NAME_URL = "url"; diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-pagerduty/pom.xml b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-pagerduty/pom.xml index 08a664d531e5..671bff25ea17 100644 --- a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-pagerduty/pom.xml +++ b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-pagerduty/pom.xml @@ -15,15 +15,14 @@ ~ See the License for the specific language governing permissions and ~ limitations under the License. --> - + 4.0.0 - dolphinscheduler-alert-plugins org.apache.dolphinscheduler + dolphinscheduler-alert-plugins dev-SNAPSHOT - 4.0.0 dolphinscheduler-alert-pagerduty jar diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-pagerduty/src/main/java/org/apache/dolphinscheduler/plugin/alert/pagerduty/PagerDutyAlertChannel.java b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-pagerduty/src/main/java/org/apache/dolphinscheduler/plugin/alert/pagerduty/PagerDutyAlertChannel.java index 6ad4c296c181..b03313952072 100644 --- a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-pagerduty/src/main/java/org/apache/dolphinscheduler/plugin/alert/pagerduty/PagerDutyAlertChannel.java +++ b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-pagerduty/src/main/java/org/apache/dolphinscheduler/plugin/alert/pagerduty/PagerDutyAlertChannel.java @@ -25,6 +25,7 @@ import java.util.Map; public final class PagerDutyAlertChannel implements AlertChannel { + @Override public AlertResult process(AlertInfo alertInfo) { AlertData alertData = alertInfo.getAlertData(); diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-pagerduty/src/main/java/org/apache/dolphinscheduler/plugin/alert/pagerduty/PagerDutyAlertChannelFactory.java b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-pagerduty/src/main/java/org/apache/dolphinscheduler/plugin/alert/pagerduty/PagerDutyAlertChannelFactory.java index 24840f730e64..92c42b9d041d 100644 --- a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-pagerduty/src/main/java/org/apache/dolphinscheduler/plugin/alert/pagerduty/PagerDutyAlertChannelFactory.java +++ b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-pagerduty/src/main/java/org/apache/dolphinscheduler/plugin/alert/pagerduty/PagerDutyAlertChannelFactory.java @@ -17,17 +17,20 @@ package org.apache.dolphinscheduler.plugin.alert.pagerduty; -import com.google.auto.service.AutoService; import org.apache.dolphinscheduler.alert.api.AlertChannel; import org.apache.dolphinscheduler.alert.api.AlertChannelFactory; import org.apache.dolphinscheduler.spi.params.base.PluginParams; import org.apache.dolphinscheduler.spi.params.base.Validate; import org.apache.dolphinscheduler.spi.params.input.InputParam; + import java.util.Collections; import java.util.List; +import com.google.auto.service.AutoService; + @AutoService(AlertChannelFactory.class) public final class PagerDutyAlertChannelFactory implements AlertChannelFactory { + @Override public String name() { return "PagerDuty"; @@ -35,7 +38,9 @@ public String name() { @Override public List params() { - InputParam integrationKey = InputParam.newBuilder(PagerDutyParamsConstants.NAME_PAGER_DUTY_INTEGRATION_KEY_NAME, PagerDutyParamsConstants.PAGER_DUTY_INTEGRATION_KEY) + InputParam integrationKey = InputParam + .newBuilder(PagerDutyParamsConstants.NAME_PAGER_DUTY_INTEGRATION_KEY_NAME, + PagerDutyParamsConstants.PAGER_DUTY_INTEGRATION_KEY) .addValidate(Validate.newBuilder() .setRequired(true) .build()) diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-pagerduty/src/main/java/org/apache/dolphinscheduler/plugin/alert/pagerduty/PagerDutyParamsConstants.java b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-pagerduty/src/main/java/org/apache/dolphinscheduler/plugin/alert/pagerduty/PagerDutyParamsConstants.java index a90d30250479..bb9ccf6a3e7d 100644 --- a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-pagerduty/src/main/java/org/apache/dolphinscheduler/plugin/alert/pagerduty/PagerDutyParamsConstants.java +++ b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-pagerduty/src/main/java/org/apache/dolphinscheduler/plugin/alert/pagerduty/PagerDutyParamsConstants.java @@ -18,6 +18,7 @@ package org.apache.dolphinscheduler.plugin.alert.pagerduty; public final class PagerDutyParamsConstants { + public static final String NAME_PAGER_DUTY_INTEGRATION_KEY_NAME = "IntegrationKey"; public static final String PAGER_DUTY_INTEGRATION_KEY = "integrationKey"; public static final String PAGER_DUTY_EVENT_ACTION_TRIGGER = "trigger"; diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-pagerduty/src/main/java/org/apache/dolphinscheduler/plugin/alert/pagerduty/PagerDutySender.java b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-pagerduty/src/main/java/org/apache/dolphinscheduler/plugin/alert/pagerduty/PagerDutySender.java index 3dd0b840c063..5fe11e00a77a 100644 --- a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-pagerduty/src/main/java/org/apache/dolphinscheduler/plugin/alert/pagerduty/PagerDutySender.java +++ b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-pagerduty/src/main/java/org/apache/dolphinscheduler/plugin/alert/pagerduty/PagerDutySender.java @@ -17,21 +17,29 @@ package org.apache.dolphinscheduler.plugin.alert.pagerduty; -import com.google.common.base.Preconditions; import org.apache.dolphinscheduler.alert.api.AlertResult; import org.apache.dolphinscheduler.spi.utils.JSONUtils; + import org.apache.http.HttpStatus; import org.apache.http.client.methods.CloseableHttpResponse; import org.apache.http.client.methods.HttpPost; import org.apache.http.entity.StringEntity; import org.apache.http.impl.client.CloseableHttpClient; import org.apache.http.impl.client.HttpClients; -import org.slf4j.Logger; + import java.io.IOException; import java.nio.charset.StandardCharsets; -import java.util.*; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; + +import org.slf4j.Logger; + +import com.google.common.base.Preconditions; public final class PagerDutySender { + private static final Logger log = org.slf4j.LoggerFactory.getLogger(PagerDutySender.class); private final String integrationKey; @@ -41,7 +49,7 @@ public PagerDutySender(Map config) { Preconditions.checkArgument(!Objects.isNull(integrationKey), "PagerDuty integration key can not be null"); } - public AlertResult sendPagerDutyAlter(String title, String content){ + public AlertResult sendPagerDutyAlter(String title, String content) { AlertResult alertResult = new AlertResult(); alertResult.setStatus("false"); alertResult.setMessage("send pager duty alert fail."); @@ -72,10 +80,10 @@ private AlertResult send(AlertResult alertResult, String url, String requestBody if (statusCode == HttpStatus.SC_OK || statusCode == HttpStatus.SC_ACCEPTED) { alertResult.setStatus("true"); alertResult.setMessage("send pager duty alert success"); - }else { + } else { log.info("send pager duty alert fail, statusCode : {}", statusCode); } - }finally { + } finally { response.close(); } } catch (IOException e) { diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-script/pom.xml b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-script/pom.xml index 96038b970cd2..e5b168d5436e 100644 --- a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-script/pom.xml +++ b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-script/pom.xml @@ -15,15 +15,14 @@ ~ See the License for the specific language governing permissions and ~ limitations under the License. --> - + 4.0.0 - dolphinscheduler-alert-plugins org.apache.dolphinscheduler + dolphinscheduler-alert-plugins dev-SNAPSHOT - 4.0.0 dolphinscheduler-alert-script jar diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-script/src/main/java/org/apache/dolphinscheduler/plugin/alert/script/OSUtils.java b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-script/src/main/java/org/apache/dolphinscheduler/plugin/alert/script/OSUtils.java index 51e567b47e74..ef5fcf67d2f8 100644 --- a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-script/src/main/java/org/apache/dolphinscheduler/plugin/alert/script/OSUtils.java +++ b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-script/src/main/java/org/apache/dolphinscheduler/plugin/alert/script/OSUtils.java @@ -18,6 +18,7 @@ package org.apache.dolphinscheduler.plugin.alert.script; public final class OSUtils { + private OSUtils() { throw new UnsupportedOperationException("This is a utility class and cannot be instantiated"); } diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-script/src/main/java/org/apache/dolphinscheduler/plugin/alert/script/ProcessUtils.java b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-script/src/main/java/org/apache/dolphinscheduler/plugin/alert/script/ProcessUtils.java index 0bc0a121d678..6ea9a8db9299 100644 --- a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-script/src/main/java/org/apache/dolphinscheduler/plugin/alert/script/ProcessUtils.java +++ b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-script/src/main/java/org/apache/dolphinscheduler/plugin/alert/script/ProcessUtils.java @@ -23,6 +23,7 @@ import org.slf4j.LoggerFactory; public final class ProcessUtils { + private static final Logger logger = LoggerFactory.getLogger(ProcessUtils.class); private ProcessUtils() { diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-script/src/main/java/org/apache/dolphinscheduler/plugin/alert/script/ScriptAlertChannel.java b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-script/src/main/java/org/apache/dolphinscheduler/plugin/alert/script/ScriptAlertChannel.java index bd52955b747b..3d865c6dc063 100644 --- a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-script/src/main/java/org/apache/dolphinscheduler/plugin/alert/script/ScriptAlertChannel.java +++ b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-script/src/main/java/org/apache/dolphinscheduler/plugin/alert/script/ScriptAlertChannel.java @@ -25,6 +25,7 @@ import java.util.Map; public final class ScriptAlertChannel implements AlertChannel { + @Override public AlertResult process(AlertInfo alertinfo) { AlertData alertData = alertinfo.getAlertData(); diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-script/src/main/java/org/apache/dolphinscheduler/plugin/alert/script/ScriptAlertChannelFactory.java b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-script/src/main/java/org/apache/dolphinscheduler/plugin/alert/script/ScriptAlertChannelFactory.java index aacde95ba113..294bca04fd61 100644 --- a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-script/src/main/java/org/apache/dolphinscheduler/plugin/alert/script/ScriptAlertChannelFactory.java +++ b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-script/src/main/java/org/apache/dolphinscheduler/plugin/alert/script/ScriptAlertChannelFactory.java @@ -32,6 +32,7 @@ @AutoService(AlertChannelFactory.class) public final class ScriptAlertChannelFactory implements AlertChannelFactory { + @Override public String name() { return "Script"; @@ -40,26 +41,30 @@ public String name() { @Override public List params() { - InputParam scriptUserParam = InputParam.newBuilder(ScriptParamsConstants.NAME_SCRIPT_USER_PARAMS, ScriptParamsConstants.SCRIPT_USER_PARAMS) - .addValidate(Validate.newBuilder() - .setRequired(false) - .build()) - .setPlaceholder("please enter your custom parameters, which will be passed to you when calling your script") - .build(); + InputParam scriptUserParam = InputParam + .newBuilder(ScriptParamsConstants.NAME_SCRIPT_USER_PARAMS, ScriptParamsConstants.SCRIPT_USER_PARAMS) + .addValidate(Validate.newBuilder() + .setRequired(false) + .build()) + .setPlaceholder( + "please enter your custom parameters, which will be passed to you when calling your script") + .build(); // need check file type and file exist - InputParam scriptPathParam = InputParam.newBuilder(ScriptParamsConstants.NAME_SCRIPT_PATH, ScriptParamsConstants.SCRIPT_PATH) - .addValidate(Validate.newBuilder() - .setRequired(true) - .build()) - .setPlaceholder("please upload the file to the disk directory of the alert server," - + " and ensure that the path is absolute and has the corresponding access rights") - .build(); + InputParam scriptPathParam = + InputParam.newBuilder(ScriptParamsConstants.NAME_SCRIPT_PATH, ScriptParamsConstants.SCRIPT_PATH) + .addValidate(Validate.newBuilder() + .setRequired(true) + .build()) + .setPlaceholder("please upload the file to the disk directory of the alert server," + + " and ensure that the path is absolute and has the corresponding access rights") + .build(); - RadioParam scriptTypeParams = RadioParam.newBuilder(ScriptParamsConstants.NAME_SCRIPT_TYPE, ScriptParamsConstants.SCRIPT_TYPE) - .addParamsOptions(new ParamsOptions(ScriptType.SHELL.getDescp(), ScriptType.SHELL.getDescp(), false)) - .setValue(ScriptType.SHELL.getDescp()) - .addValidate(Validate.newBuilder().setRequired(true).build()) - .build(); + RadioParam scriptTypeParams = RadioParam + .newBuilder(ScriptParamsConstants.NAME_SCRIPT_TYPE, ScriptParamsConstants.SCRIPT_TYPE) + .addParamsOptions(new ParamsOptions(ScriptType.SHELL.getDescp(), ScriptType.SHELL.getDescp(), false)) + .setValue(ScriptType.SHELL.getDescp()) + .addValidate(Validate.newBuilder().setRequired(true).build()) + .build(); return Arrays.asList(scriptUserParam, scriptPathParam, scriptTypeParams); } diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-script/src/main/java/org/apache/dolphinscheduler/plugin/alert/script/ScriptParamsConstants.java b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-script/src/main/java/org/apache/dolphinscheduler/plugin/alert/script/ScriptParamsConstants.java index e680903fb2f3..1c759525b399 100644 --- a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-script/src/main/java/org/apache/dolphinscheduler/plugin/alert/script/ScriptParamsConstants.java +++ b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-script/src/main/java/org/apache/dolphinscheduler/plugin/alert/script/ScriptParamsConstants.java @@ -18,6 +18,7 @@ package org.apache.dolphinscheduler.plugin.alert.script; public final class ScriptParamsConstants { + static final String SCRIPT_TYPE = "$t('scriptType')"; static final String NAME_SCRIPT_TYPE = "type"; diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-script/src/main/java/org/apache/dolphinscheduler/plugin/alert/script/ScriptType.java b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-script/src/main/java/org/apache/dolphinscheduler/plugin/alert/script/ScriptType.java index cbb3e11f28a6..b31e10f575d0 100644 --- a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-script/src/main/java/org/apache/dolphinscheduler/plugin/alert/script/ScriptType.java +++ b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-script/src/main/java/org/apache/dolphinscheduler/plugin/alert/script/ScriptType.java @@ -18,6 +18,7 @@ package org.apache.dolphinscheduler.plugin.alert.script; public enum ScriptType { + SHELL(0, "SHELL"), ; diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-script/src/main/java/org/apache/dolphinscheduler/plugin/alert/script/StreamGobbler.java b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-script/src/main/java/org/apache/dolphinscheduler/plugin/alert/script/StreamGobbler.java index 208791b6f242..e2765a8a40ed 100644 --- a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-script/src/main/java/org/apache/dolphinscheduler/plugin/alert/script/StreamGobbler.java +++ b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-script/src/main/java/org/apache/dolphinscheduler/plugin/alert/script/StreamGobbler.java @@ -26,6 +26,7 @@ import org.slf4j.LoggerFactory; public final class StreamGobbler extends Thread { + private static final Logger logger = LoggerFactory.getLogger(StreamGobbler.class); private final InputStream inputStream; diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-slack/pom.xml b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-slack/pom.xml index 256b1ebd678e..e92ddc6a42b1 100644 --- a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-slack/pom.xml +++ b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-slack/pom.xml @@ -15,15 +15,14 @@ ~ See the License for the specific language governing permissions and ~ limitations under the License. --> - + 4.0.0 - dolphinscheduler-alert-plugins org.apache.dolphinscheduler + dolphinscheduler-alert-plugins dev-SNAPSHOT - 4.0.0 dolphinscheduler-alert-slack jar diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-slack/src/main/java/org/apache/dolphinscheduler/plugin/alert/slack/SlackAlertChannel.java b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-slack/src/main/java/org/apache/dolphinscheduler/plugin/alert/slack/SlackAlertChannel.java index b63fed05740f..c8cb36a78b39 100644 --- a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-slack/src/main/java/org/apache/dolphinscheduler/plugin/alert/slack/SlackAlertChannel.java +++ b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-slack/src/main/java/org/apache/dolphinscheduler/plugin/alert/slack/SlackAlertChannel.java @@ -25,6 +25,7 @@ import java.util.Map; public final class SlackAlertChannel implements AlertChannel { + @Override public AlertResult process(AlertInfo alertInfo) { AlertData alertData = alertInfo.getAlertData(); diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-slack/src/main/java/org/apache/dolphinscheduler/plugin/alert/slack/SlackAlertChannelFactory.java b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-slack/src/main/java/org/apache/dolphinscheduler/plugin/alert/slack/SlackAlertChannelFactory.java index f86e32933f7f..e08bc4618e50 100644 --- a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-slack/src/main/java/org/apache/dolphinscheduler/plugin/alert/slack/SlackAlertChannelFactory.java +++ b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-slack/src/main/java/org/apache/dolphinscheduler/plugin/alert/slack/SlackAlertChannelFactory.java @@ -30,6 +30,7 @@ @AutoService(AlertChannelFactory.class) public final class SlackAlertChannelFactory implements AlertChannelFactory { + @Override public String name() { return "Slack"; @@ -39,19 +40,20 @@ public String name() { public List params() { List paramsList = new LinkedList<>(); - InputParam webHookParam = InputParam.newBuilder(SlackParamsConstants.SLACK_WEB_HOOK_URL_NAME, SlackParamsConstants.SLACK_WEB_HOOK_URL) - .addValidate(Validate.newBuilder() - .setRequired(true) - .build()) - .setPlaceholder("Input WebHook Url") - .build(); + InputParam webHookParam = InputParam + .newBuilder(SlackParamsConstants.SLACK_WEB_HOOK_URL_NAME, SlackParamsConstants.SLACK_WEB_HOOK_URL) + .addValidate(Validate.newBuilder() + .setRequired(true) + .build()) + .setPlaceholder("Input WebHook Url") + .build(); InputParam botName = InputParam.newBuilder(SlackParamsConstants.SLACK_BOT_NAME, SlackParamsConstants.SLACK_BOT) - .addValidate(Validate.newBuilder() - .setRequired(true) - .build()) - .setPlaceholder("Input the bot username") - .build(); + .addValidate(Validate.newBuilder() + .setRequired(true) + .build()) + .setPlaceholder("Input the bot username") + .build(); paramsList.add(webHookParam); paramsList.add(botName); diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-slack/src/main/java/org/apache/dolphinscheduler/plugin/alert/slack/SlackParamsConstants.java b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-slack/src/main/java/org/apache/dolphinscheduler/plugin/alert/slack/SlackParamsConstants.java index a3c9f19ac520..c92f3031550e 100644 --- a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-slack/src/main/java/org/apache/dolphinscheduler/plugin/alert/slack/SlackParamsConstants.java +++ b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-slack/src/main/java/org/apache/dolphinscheduler/plugin/alert/slack/SlackParamsConstants.java @@ -18,6 +18,7 @@ package org.apache.dolphinscheduler.plugin.alert.slack; public final class SlackParamsConstants { + public static final String SLACK_WEB_HOOK_URL = "$t('webhook')"; public static final String SLACK_WEB_HOOK_URL_NAME = "webHook"; public static final String SLACK_BOT = "$t('Username')"; diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-slack/src/main/java/org/apache/dolphinscheduler/plugin/alert/slack/SlackSender.java b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-slack/src/main/java/org/apache/dolphinscheduler/plugin/alert/slack/SlackSender.java index 1017c839fd28..e571642f3845 100644 --- a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-slack/src/main/java/org/apache/dolphinscheduler/plugin/alert/slack/SlackSender.java +++ b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-slack/src/main/java/org/apache/dolphinscheduler/plugin/alert/slack/SlackSender.java @@ -44,6 +44,7 @@ import com.google.common.base.Preconditions; public final class SlackSender { + private static final Logger logger = LoggerFactory.getLogger(SlackSender.class); private final String webHookUrl; @@ -53,7 +54,8 @@ public SlackSender(Map slackAlertParam) { webHookUrl = slackAlertParam.get(SlackParamsConstants.SLACK_WEB_HOOK_URL_NAME); botName = slackAlertParam.get(SlackParamsConstants.SLACK_BOT_NAME); Preconditions.checkArgument(!Objects.isNull(webHookUrl), "SlackWebHookURL can not be null"); - Preconditions.checkArgument(webHookUrl.startsWith("https://hooks.slack.com/services/"), "SlackWebHookURL invalidate"); + Preconditions.checkArgument(webHookUrl.startsWith("https://hooks.slack.com/services/"), + "SlackWebHookURL invalidate"); Preconditions.checkArgument(!Objects.isNull(botName), "slack bot name can not be null"); } @@ -120,17 +122,17 @@ private String generateMarkDownTable(String content) { final int elementLen = maxLen; StringBuilder stringBuilder = new StringBuilder(200); stringBuilder.append(headers.stream() - .map(header -> generateString(header, elementLen, " ")) - .collect(Collectors.joining("|"))); + .map(header -> generateString(header, elementLen, " ")) + .collect(Collectors.joining("|"))); stringBuilder.append("\n"); for (List element : elements) { stringBuilder.append(element.stream() - .map(lement -> generateString("", elementLen, "-")) - .collect(Collectors.joining("|"))); + .map(lement -> generateString("", elementLen, "-")) + .collect(Collectors.joining("|"))); stringBuilder.append("\n"); stringBuilder.append(element.stream() - .map(e -> generateString(e, elementLen, " ")) - .collect(Collectors.joining("|"))); + .map(e -> generateString(e, elementLen, " ")) + .collect(Collectors.joining("|"))); stringBuilder.append("\n"); } return String.format("```%s```", stringBuilder); diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-telegram/pom.xml b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-telegram/pom.xml index ed761d446ae3..e24045c8aa32 100644 --- a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-telegram/pom.xml +++ b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-telegram/pom.xml @@ -15,15 +15,14 @@ ~ See the License for the specific language governing permissions and ~ limitations under the License. --> - + 4.0.0 - dolphinscheduler-alert-plugins org.apache.dolphinscheduler + dolphinscheduler-alert-plugins dev-SNAPSHOT - 4.0.0 dolphinscheduler-alert-telegram jar diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-telegram/src/main/java/org/apache/dolphinscheduler/plugin/alert/telegram/TelegramAlertChannel.java b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-telegram/src/main/java/org/apache/dolphinscheduler/plugin/alert/telegram/TelegramAlertChannel.java index 26db453dd4c3..efc8912d1a0a 100644 --- a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-telegram/src/main/java/org/apache/dolphinscheduler/plugin/alert/telegram/TelegramAlertChannel.java +++ b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-telegram/src/main/java/org/apache/dolphinscheduler/plugin/alert/telegram/TelegramAlertChannel.java @@ -25,6 +25,7 @@ import java.util.Map; public final class TelegramAlertChannel implements AlertChannel { + @Override public AlertResult process(AlertInfo info) { Map alertParams = info.getAlertParams(); diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-telegram/src/main/java/org/apache/dolphinscheduler/plugin/alert/telegram/TelegramAlertChannelFactory.java b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-telegram/src/main/java/org/apache/dolphinscheduler/plugin/alert/telegram/TelegramAlertChannelFactory.java index f282cb515315..240578cfb73c 100644 --- a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-telegram/src/main/java/org/apache/dolphinscheduler/plugin/alert/telegram/TelegramAlertChannelFactory.java +++ b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-telegram/src/main/java/org/apache/dolphinscheduler/plugin/alert/telegram/TelegramAlertChannelFactory.java @@ -17,7 +17,11 @@ package org.apache.dolphinscheduler.plugin.alert.telegram; -import com.google.auto.service.AutoService; +import static org.apache.dolphinscheduler.spi.utils.Constants.STRING_FALSE; +import static org.apache.dolphinscheduler.spi.utils.Constants.STRING_NO; +import static org.apache.dolphinscheduler.spi.utils.Constants.STRING_TRUE; +import static org.apache.dolphinscheduler.spi.utils.Constants.STRING_YES; + import org.apache.dolphinscheduler.alert.api.AlertChannel; import org.apache.dolphinscheduler.alert.api.AlertChannelFactory; import org.apache.dolphinscheduler.spi.params.PasswordParam; @@ -31,7 +35,7 @@ import java.util.Arrays; import java.util.List; -import static org.apache.dolphinscheduler.spi.utils.Constants.*; +import com.google.auto.service.AutoService; @AutoService(AlertChannelFactory.class) public final class TelegramAlertChannelFactory implements AlertChannelFactory { @@ -49,40 +53,50 @@ public AlertChannel create() { @Override public List params() { - - InputParam webHookParam = InputParam.newBuilder(TelegramParamsConstants.NAME_TELEGRAM_WEB_HOOK, TelegramParamsConstants.TELEGRAM_WEB_HOOK) + InputParam webHookParam = InputParam + .newBuilder(TelegramParamsConstants.NAME_TELEGRAM_WEB_HOOK, TelegramParamsConstants.TELEGRAM_WEB_HOOK) .addValidate(Validate.newBuilder() .setRequired(true) .build()) .setPlaceholder("telegram web hook") .build(); - InputParam botTokenParam = InputParam.newBuilder(TelegramParamsConstants.NAME_TELEGRAM_BOT_TOKEN, TelegramParamsConstants.TELEGRAM_BOT_TOKEN) + InputParam botTokenParam = InputParam + .newBuilder(TelegramParamsConstants.NAME_TELEGRAM_BOT_TOKEN, TelegramParamsConstants.TELEGRAM_BOT_TOKEN) .addValidate(Validate.newBuilder() .setRequired(true) .build()) .setPlaceholder("telegram bot token") .build(); - InputParam chatIdParam = InputParam.newBuilder(TelegramParamsConstants.NAME_TELEGRAM_CHAT_ID, TelegramParamsConstants.TELEGRAM_CHAT_ID) + InputParam chatIdParam = InputParam + .newBuilder(TelegramParamsConstants.NAME_TELEGRAM_CHAT_ID, TelegramParamsConstants.TELEGRAM_CHAT_ID) .addValidate(Validate.newBuilder() .setRequired(true) .build()) .setPlaceholder("telegram channel chat id") .build(); - SelectParam parseMode = SelectParam.newBuilder(TelegramParamsConstants.NAME_TELEGRAM_PARSE_MODE, TelegramParamsConstants.TELEGRAM_PARSE_MODE) - .addOptions(new ParamsOptions(TelegramAlertConstants.PARSE_MODE_TXT, TelegramAlertConstants.PARSE_MODE_TXT, false)) - .addOptions(new ParamsOptions(TelegramAlertConstants.PARSE_MODE_MARKDOWN, TelegramAlertConstants.PARSE_MODE_MARKDOWN, false)) - .addOptions(new ParamsOptions(TelegramAlertConstants.PARSE_MODE_MARKDOWN_V2, TelegramAlertConstants.PARSE_MODE_MARKDOWN_V2, false)) - .addOptions(new ParamsOptions(TelegramAlertConstants.PARSE_MODE_HTML, TelegramAlertConstants.PARSE_MODE_HTML, false)) + SelectParam parseMode = SelectParam + .newBuilder(TelegramParamsConstants.NAME_TELEGRAM_PARSE_MODE, + TelegramParamsConstants.TELEGRAM_PARSE_MODE) + .addOptions(new ParamsOptions(TelegramAlertConstants.PARSE_MODE_TXT, + TelegramAlertConstants.PARSE_MODE_TXT, false)) + .addOptions(new ParamsOptions(TelegramAlertConstants.PARSE_MODE_MARKDOWN, + TelegramAlertConstants.PARSE_MODE_MARKDOWN, false)) + .addOptions(new ParamsOptions(TelegramAlertConstants.PARSE_MODE_MARKDOWN_V2, + TelegramAlertConstants.PARSE_MODE_MARKDOWN_V2, false)) + .addOptions(new ParamsOptions(TelegramAlertConstants.PARSE_MODE_HTML, + TelegramAlertConstants.PARSE_MODE_HTML, false)) .setValue(TelegramAlertConstants.PARSE_MODE_TXT) .addValidate(Validate.newBuilder() .setRequired(true) .build()) .build(); - RadioParam isEnableProxy = RadioParam.newBuilder(TelegramParamsConstants.NAME_TELEGRAM_PROXY_ENABLE, TelegramParamsConstants.TELEGRAM_PROXY_ENABLE) + RadioParam isEnableProxy = RadioParam + .newBuilder(TelegramParamsConstants.NAME_TELEGRAM_PROXY_ENABLE, + TelegramParamsConstants.TELEGRAM_PROXY_ENABLE) .addParamsOptions(new ParamsOptions(STRING_YES, STRING_TRUE, false)) .addParamsOptions(new ParamsOptions(STRING_NO, STRING_FALSE, false)) .setValue(STRING_FALSE) @@ -91,33 +105,37 @@ public List params() { .build()) .build(); - InputParam proxyParam = InputParam.newBuilder(TelegramParamsConstants.NAME_TELEGRAM_PROXY, TelegramParamsConstants.TELEGRAM_PROXY) + InputParam proxyParam = InputParam + .newBuilder(TelegramParamsConstants.NAME_TELEGRAM_PROXY, TelegramParamsConstants.TELEGRAM_PROXY) .addValidate(Validate.newBuilder() .setRequired(false) .build()) .build(); - InputParam portParam = InputParam.newBuilder(TelegramParamsConstants.NAME_TELEGRAM_PORT, TelegramParamsConstants.TELEGRAM_PORT) - .addValidate(Validate.newBuilder() - .setRequired(false) - .build()) - .build(); - - InputParam userParam = InputParam.newBuilder(TelegramParamsConstants.NAME_TELEGRAM_USER, TelegramParamsConstants.TELEGRAM_USER) - .addValidate(Validate.newBuilder() - .setRequired(false) - .build()) - .build(); - - PasswordParam passwordParam = PasswordParam.newBuilder(TelegramParamsConstants.NAME_TELEGRAM_PASSWORD, TelegramParamsConstants.TELEGRAM_PASSWORD) + InputParam portParam = + InputParam.newBuilder(TelegramParamsConstants.NAME_TELEGRAM_PORT, TelegramParamsConstants.TELEGRAM_PORT) + .addValidate(Validate.newBuilder() + .setRequired(false) + .build()) + .build(); + + InputParam userParam = + InputParam.newBuilder(TelegramParamsConstants.NAME_TELEGRAM_USER, TelegramParamsConstants.TELEGRAM_USER) + .addValidate(Validate.newBuilder() + .setRequired(false) + .build()) + .build(); + + PasswordParam passwordParam = PasswordParam + .newBuilder(TelegramParamsConstants.NAME_TELEGRAM_PASSWORD, TelegramParamsConstants.TELEGRAM_PASSWORD) .addValidate(Validate.newBuilder() .setRequired(false) .build()) .setPlaceholder("if enable use authentication, you need input password") .build(); - return Arrays.asList(webHookParam, botTokenParam, chatIdParam, parseMode, isEnableProxy, proxyParam, portParam, userParam, passwordParam); + return Arrays.asList(webHookParam, botTokenParam, chatIdParam, parseMode, isEnableProxy, proxyParam, portParam, + userParam, passwordParam); } - } diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-telegram/src/main/java/org/apache/dolphinscheduler/plugin/alert/telegram/TelegramParamsConstants.java b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-telegram/src/main/java/org/apache/dolphinscheduler/plugin/alert/telegram/TelegramParamsConstants.java index d9fbb3f218e6..4ae48506c410 100644 --- a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-telegram/src/main/java/org/apache/dolphinscheduler/plugin/alert/telegram/TelegramParamsConstants.java +++ b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-telegram/src/main/java/org/apache/dolphinscheduler/plugin/alert/telegram/TelegramParamsConstants.java @@ -17,7 +17,6 @@ package org.apache.dolphinscheduler.plugin.alert.telegram; - public final class TelegramParamsConstants { static final String TELEGRAM_WEB_HOOK = "$t('webHook')"; diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-telegram/src/main/java/org/apache/dolphinscheduler/plugin/alert/telegram/TelegramSender.java b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-telegram/src/main/java/org/apache/dolphinscheduler/plugin/alert/telegram/TelegramSender.java index 2472e81a408f..eb7d7bfa3c48 100644 --- a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-telegram/src/main/java/org/apache/dolphinscheduler/plugin/alert/telegram/TelegramSender.java +++ b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-telegram/src/main/java/org/apache/dolphinscheduler/plugin/alert/telegram/TelegramSender.java @@ -17,11 +17,11 @@ package org.apache.dolphinscheduler.plugin.alert.telegram; -import com.fasterxml.jackson.annotation.JsonProperty; import org.apache.dolphinscheduler.alert.api.AlertData; import org.apache.dolphinscheduler.alert.api.AlertResult; import org.apache.dolphinscheduler.spi.utils.JSONUtils; import org.apache.dolphinscheduler.spi.utils.StringUtils; + import org.apache.http.HttpEntity; import org.apache.http.HttpHeaders; import org.apache.http.HttpHost; @@ -37,14 +37,17 @@ import org.apache.http.impl.client.CloseableHttpClient; import org.apache.http.impl.client.HttpClients; import org.apache.http.util.EntityUtils; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import java.io.IOException; import java.nio.charset.StandardCharsets; import java.util.HashMap; import java.util.Map; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.fasterxml.jackson.annotation.JsonProperty; + public final class TelegramSender { private static final Logger logger = LoggerFactory.getLogger(TelegramSender.class); @@ -69,7 +72,6 @@ public final class TelegramSender { private String password; - TelegramSender(Map config) { url = config.get(TelegramParamsConstants.NAME_TELEGRAM_WEB_HOOK); botToken = config.get(TelegramParamsConstants.NAME_TELEGRAM_BOT_TOKEN); @@ -132,14 +134,13 @@ private AlertResult parseRespToResult(String resp) { return result; } - private String sendInvoke(String title, String content) throws IOException { HttpPost httpPost = buildHttpPost(url, buildMsgJsonStr(content)); CloseableHttpClient httpClient; if (Boolean.TRUE.equals(enableProxy)) { if (StringUtils.isNotEmpty(user) && StringUtils.isNotEmpty(password)) { httpClient = getProxyClient(proxy, port, user, password); - }else { + } else { httpClient = getDefaultClient(); } RequestConfig rcf = getProxyConfig(proxy, port); @@ -180,6 +181,7 @@ private boolean isTextParseMode() { } static class TelegramSendMsgResponse { + @JsonProperty("ok") private Boolean ok; @JsonProperty("error_code") diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-webexteams/pom.xml b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-webexteams/pom.xml index 5668088eb7c1..fc1c373bee6a 100644 --- a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-webexteams/pom.xml +++ b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-webexteams/pom.xml @@ -15,15 +15,14 @@ ~ See the License for the specific language governing permissions and ~ limitations under the License. --> - + 4.0.0 - dolphinscheduler-alert-plugins org.apache.dolphinscheduler + dolphinscheduler-alert-plugins dev-SNAPSHOT - 4.0.0 dolphinscheduler-alert-webexteams jar diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-webexteams/src/main/java/org/apache/dolphinscheduler/plugin/alert/webexteams/WebexMessage.java b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-webexteams/src/main/java/org/apache/dolphinscheduler/plugin/alert/webexteams/WebexMessage.java index f1b908ff6c18..4a417909e45a 100644 --- a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-webexteams/src/main/java/org/apache/dolphinscheduler/plugin/alert/webexteams/WebexMessage.java +++ b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-webexteams/src/main/java/org/apache/dolphinscheduler/plugin/alert/webexteams/WebexMessage.java @@ -18,6 +18,7 @@ package org.apache.dolphinscheduler.plugin.alert.webexteams; public class WebexMessage { + private String roomId; private String toPersonId; private String toPersonEmail; diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-webexteams/src/main/java/org/apache/dolphinscheduler/plugin/alert/webexteams/WebexTeamsAlertChannel.java b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-webexteams/src/main/java/org/apache/dolphinscheduler/plugin/alert/webexteams/WebexTeamsAlertChannel.java index 08963cab6ae8..1a799ae50c58 100644 --- a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-webexteams/src/main/java/org/apache/dolphinscheduler/plugin/alert/webexteams/WebexTeamsAlertChannel.java +++ b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-webexteams/src/main/java/org/apache/dolphinscheduler/plugin/alert/webexteams/WebexTeamsAlertChannel.java @@ -25,6 +25,7 @@ import java.util.Map; public final class WebexTeamsAlertChannel implements AlertChannel { + @Override public AlertResult process(AlertInfo alertInfo) { AlertData alertData = alertInfo.getAlertData(); diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-webexteams/src/main/java/org/apache/dolphinscheduler/plugin/alert/webexteams/WebexTeamsAlertChannelFactory.java b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-webexteams/src/main/java/org/apache/dolphinscheduler/plugin/alert/webexteams/WebexTeamsAlertChannelFactory.java index f17c3cde8f98..e11078a31c10 100644 --- a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-webexteams/src/main/java/org/apache/dolphinscheduler/plugin/alert/webexteams/WebexTeamsAlertChannelFactory.java +++ b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-webexteams/src/main/java/org/apache/dolphinscheduler/plugin/alert/webexteams/WebexTeamsAlertChannelFactory.java @@ -32,6 +32,7 @@ @AutoService(AlertChannelFactory.class) public final class WebexTeamsAlertChannelFactory implements AlertChannelFactory { + @Override public String name() { return "WebexTeams"; @@ -40,48 +41,63 @@ public String name() { @Override public List params() { List paramsList = new ArrayList<>(); - InputParam botAccessToken = InputParam.newBuilder(WebexTeamsParamsConstants.NAME_WEBEX_TEAMS_BOT_ACCESS_TOKEN, WebexTeamsParamsConstants.WEBEX_TEAMS_BOT_ACCESS_TOKEN) - .setPlaceholder("Please enter the robot's access token you were given") - .addValidate(Validate.newBuilder() - .setRequired(true) - .build()) - .build(); + InputParam botAccessToken = InputParam + .newBuilder(WebexTeamsParamsConstants.NAME_WEBEX_TEAMS_BOT_ACCESS_TOKEN, + WebexTeamsParamsConstants.WEBEX_TEAMS_BOT_ACCESS_TOKEN) + .setPlaceholder("Please enter the robot's access token you were given") + .addValidate(Validate.newBuilder() + .setRequired(true) + .build()) + .build(); - InputParam roomId = InputParam.newBuilder(WebexTeamsParamsConstants.NAME_WEBEX_TEAMS_ROOM_ID, WebexTeamsParamsConstants.WEBEX_TEAMS_ROOM_ID) - .setPlaceholder("The room ID of the message") - .addValidate(Validate.newBuilder() - .setRequired(false) - .build()) - .build(); + InputParam roomId = InputParam + .newBuilder(WebexTeamsParamsConstants.NAME_WEBEX_TEAMS_ROOM_ID, + WebexTeamsParamsConstants.WEBEX_TEAMS_ROOM_ID) + .setPlaceholder("The room ID of the message") + .addValidate(Validate.newBuilder() + .setRequired(false) + .build()) + .build(); - InputParam toPersonId = InputParam.newBuilder(WebexTeamsParamsConstants.NAME_WEBEX_TEAMS_TO_PERSON_ID, WebexTeamsParamsConstants.WEBEX_TEAMS_TO_PERSON_ID) - .setPlaceholder("The person ID of the message recipient") - .addValidate(Validate.newBuilder() - .setRequired(false) - .build()) - .build(); + InputParam toPersonId = InputParam + .newBuilder(WebexTeamsParamsConstants.NAME_WEBEX_TEAMS_TO_PERSON_ID, + WebexTeamsParamsConstants.WEBEX_TEAMS_TO_PERSON_ID) + .setPlaceholder("The person ID of the message recipient") + .addValidate(Validate.newBuilder() + .setRequired(false) + .build()) + .build(); - InputParam toPersonEmail = InputParam.newBuilder(WebexTeamsParamsConstants.NAME_WEBEX_TEAMS_TO_PERSON_EMAIL, WebexTeamsParamsConstants.WEBEX_TEAMS_TO_PERSON_EMAIL) - .setPlaceholder("The email address of the message recipient") - .addValidate(Validate.newBuilder() - .setRequired(false) - .build()) - .build(); + InputParam toPersonEmail = InputParam + .newBuilder(WebexTeamsParamsConstants.NAME_WEBEX_TEAMS_TO_PERSON_EMAIL, + WebexTeamsParamsConstants.WEBEX_TEAMS_TO_PERSON_EMAIL) + .setPlaceholder("The email address of the message recipient") + .addValidate(Validate.newBuilder() + .setRequired(false) + .build()) + .build(); - InputParam atSomeoneInRoom = InputParam.newBuilder(WebexTeamsParamsConstants.NAME_WEBEX_TEAMS_AT_SOMEONE_IN_ROOM, WebexTeamsParamsConstants.WEBEX_TEAMS_AT_SOMEONE_IN_ROOM) - .setPlaceholder("use ,(eng commas) to separate multiple emails") - .addValidate(Validate.newBuilder() - .setRequired(false) - .build()) - .build(); + InputParam atSomeoneInRoom = InputParam + .newBuilder(WebexTeamsParamsConstants.NAME_WEBEX_TEAMS_AT_SOMEONE_IN_ROOM, + WebexTeamsParamsConstants.WEBEX_TEAMS_AT_SOMEONE_IN_ROOM) + .setPlaceholder("use ,(eng commas) to separate multiple emails") + .addValidate(Validate.newBuilder() + .setRequired(false) + .build()) + .build(); - RadioParam destination = RadioParam.newBuilder(WebexTeamsParamsConstants.NAME_WEBEX_TEAMS_DESTINATION, WebexTeamsParamsConstants.WEBEX_TEAMS_DESTINATION) - .addParamsOptions(new ParamsOptions(WebexTeamsDestination.ROOM_ID.getDescp(), WebexTeamsDestination.ROOM_ID.getDescp(), false)) - .addParamsOptions(new ParamsOptions(WebexTeamsDestination.PERSON_EMAIL.getDescp(), WebexTeamsDestination.PERSON_EMAIL.getDescp(), false)) - .addParamsOptions(new ParamsOptions(WebexTeamsDestination.PERSON_ID.getDescp(), WebexTeamsDestination.PERSON_ID.getDescp(), false)) - .setValue(WebexTeamsDestination.ROOM_ID.getDescp()) - .addValidate(Validate.newBuilder().setRequired(true).build()) - .build(); + RadioParam destination = RadioParam + .newBuilder(WebexTeamsParamsConstants.NAME_WEBEX_TEAMS_DESTINATION, + WebexTeamsParamsConstants.WEBEX_TEAMS_DESTINATION) + .addParamsOptions(new ParamsOptions(WebexTeamsDestination.ROOM_ID.getDescp(), + WebexTeamsDestination.ROOM_ID.getDescp(), false)) + .addParamsOptions(new ParamsOptions(WebexTeamsDestination.PERSON_EMAIL.getDescp(), + WebexTeamsDestination.PERSON_EMAIL.getDescp(), false)) + .addParamsOptions(new ParamsOptions(WebexTeamsDestination.PERSON_ID.getDescp(), + WebexTeamsDestination.PERSON_ID.getDescp(), false)) + .setValue(WebexTeamsDestination.ROOM_ID.getDescp()) + .addValidate(Validate.newBuilder().setRequired(true).build()) + .build(); paramsList.add(botAccessToken); paramsList.add(roomId); diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-webexteams/src/main/java/org/apache/dolphinscheduler/plugin/alert/webexteams/WebexTeamsDestination.java b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-webexteams/src/main/java/org/apache/dolphinscheduler/plugin/alert/webexteams/WebexTeamsDestination.java index 1d23aed5910c..694fd3fdeaff 100644 --- a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-webexteams/src/main/java/org/apache/dolphinscheduler/plugin/alert/webexteams/WebexTeamsDestination.java +++ b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-webexteams/src/main/java/org/apache/dolphinscheduler/plugin/alert/webexteams/WebexTeamsDestination.java @@ -25,6 +25,7 @@ import com.google.common.base.Functions; public enum WebexTeamsDestination { + /** * 0 ROOM_ID; * 1 PERSON_EMAIL; @@ -51,7 +52,8 @@ public String getDescp() { } private static final Map WEBEX_TEAMS_DESTINATION_MAP = - Arrays.stream(WebexTeamsDestination.values()).collect(toMap(WebexTeamsDestination::getDescp, Functions.identity())); + Arrays.stream(WebexTeamsDestination.values()) + .collect(toMap(WebexTeamsDestination::getDescp, Functions.identity())); public static WebexTeamsDestination of(String descp) { if (WEBEX_TEAMS_DESTINATION_MAP.containsKey(descp)) { diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-webexteams/src/main/java/org/apache/dolphinscheduler/plugin/alert/webexteams/WebexTeamsSender.java b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-webexteams/src/main/java/org/apache/dolphinscheduler/plugin/alert/webexteams/WebexTeamsSender.java index 7b61696c8f64..adbbbe9eb163 100644 --- a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-webexteams/src/main/java/org/apache/dolphinscheduler/plugin/alert/webexteams/WebexTeamsSender.java +++ b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-webexteams/src/main/java/org/apache/dolphinscheduler/plugin/alert/webexteams/WebexTeamsSender.java @@ -39,6 +39,7 @@ import com.google.common.base.Preconditions; public final class WebexTeamsSender { + private static final Logger log = org.slf4j.LoggerFactory.getLogger(WebexTeamsSender.class); private final String botAccessToken; @@ -57,7 +58,9 @@ public WebexTeamsSender(Map config) { destination = WebexTeamsDestination.of(config.get(WebexTeamsParamsConstants.NAME_WEBEX_TEAMS_DESTINATION)); Preconditions.checkArgument(!Objects.isNull(botAccessToken), "WebexTeams bot access token can not be null"); Preconditions.checkArgument(!Objects.isNull(destination), "WebexTeams message destination can not be null"); - Preconditions.checkArgument((!Objects.isNull(roomId) || !Objects.isNull(toPersonId) || !Objects.isNull(toPersonEmail)),"WebexTeams message destination could not be determined. Provide only one destination in the roomId, toPersonEmail, or toPersonId field"); + Preconditions.checkArgument( + (!Objects.isNull(roomId) || !Objects.isNull(toPersonId) || !Objects.isNull(toPersonEmail)), + "WebexTeams message destination could not be determined. Provide only one destination in the roomId, toPersonEmail, or toPersonId field"); } public AlertResult sendWebexTeamsAlter(AlertData alertData) { @@ -137,11 +140,11 @@ private static HttpPost constructHttpPost(WebexMessage message, String botAccess public static String addAtPersonEmailInRoom(String formatContent, String atPersonEmailInRoom) { String[] emailArr = atPersonEmailInRoom.split(","); StringBuilder formatContentBuilder = new StringBuilder(formatContent); - for (String email: emailArr) { + for (String email : emailArr) { formatContentBuilder.append(" <@personEmail:").append(email).append(">"); } - return formatContentBuilder.toString(); + return formatContentBuilder.toString(); } public static String formatContent(AlertData alertData) { diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-wechat/pom.xml b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-wechat/pom.xml index b293ac6c1e9d..3576f0c5e236 100644 --- a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-wechat/pom.xml +++ b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-wechat/pom.xml @@ -15,15 +15,14 @@ ~ See the License for the specific language governing permissions and ~ limitations under the License. --> - + 4.0.0 - dolphinscheduler-alert-plugins org.apache.dolphinscheduler + dolphinscheduler-alert-plugins dev-SNAPSHOT - 4.0.0 dolphinscheduler-alert-wechat jar diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-wechat/src/main/java/org/apache/dolphinscheduler/plugin/alert/wechat/WeChatAlertChannel.java b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-wechat/src/main/java/org/apache/dolphinscheduler/plugin/alert/wechat/WeChatAlertChannel.java index 94b43ea3006d..786cdb159f07 100644 --- a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-wechat/src/main/java/org/apache/dolphinscheduler/plugin/alert/wechat/WeChatAlertChannel.java +++ b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-wechat/src/main/java/org/apache/dolphinscheduler/plugin/alert/wechat/WeChatAlertChannel.java @@ -25,6 +25,7 @@ import java.util.Map; public final class WeChatAlertChannel implements AlertChannel { + @Override public AlertResult process(AlertInfo info) { AlertData alertData = info.getAlertData(); diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-wechat/src/main/java/org/apache/dolphinscheduler/plugin/alert/wechat/WeChatAlertChannelFactory.java b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-wechat/src/main/java/org/apache/dolphinscheduler/plugin/alert/wechat/WeChatAlertChannelFactory.java index 39a4a8159fb2..a35775609027 100644 --- a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-wechat/src/main/java/org/apache/dolphinscheduler/plugin/alert/wechat/WeChatAlertChannelFactory.java +++ b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-wechat/src/main/java/org/apache/dolphinscheduler/plugin/alert/wechat/WeChatAlertChannelFactory.java @@ -34,6 +34,7 @@ @AutoService(AlertChannelFactory.class) public final class WeChatAlertChannelFactory implements AlertChannelFactory { + @Override public String name() { return "WeChat"; @@ -41,47 +42,58 @@ public String name() { @Override public List params() { - InputParam corpIdParam = InputParam.newBuilder(WeChatAlertParamsConstants.NAME_ENTERPRISE_WE_CHAT_CORP_ID, WeChatAlertParamsConstants.ENTERPRISE_WE_CHAT_CORP_ID) - .setPlaceholder("please input corp id ") - .addValidate(Validate.newBuilder() - .setRequired(true) - .build()) - .build(); + InputParam corpIdParam = InputParam + .newBuilder(WeChatAlertParamsConstants.NAME_ENTERPRISE_WE_CHAT_CORP_ID, + WeChatAlertParamsConstants.ENTERPRISE_WE_CHAT_CORP_ID) + .setPlaceholder("please input corp id ") + .addValidate(Validate.newBuilder() + .setRequired(true) + .build()) + .build(); - InputParam secretParam = InputParam.newBuilder(WeChatAlertParamsConstants.NAME_ENTERPRISE_WE_CHAT_SECRET, WeChatAlertParamsConstants.ENTERPRISE_WE_CHAT_SECRET) - .setPlaceholder("please input secret ") - .addValidate(Validate.newBuilder() - .setRequired(true) - .build()) - .build(); + InputParam secretParam = InputParam + .newBuilder(WeChatAlertParamsConstants.NAME_ENTERPRISE_WE_CHAT_SECRET, + WeChatAlertParamsConstants.ENTERPRISE_WE_CHAT_SECRET) + .setPlaceholder("please input secret ") + .addValidate(Validate.newBuilder() + .setRequired(true) + .build()) + .build(); - InputParam usersParam = InputParam.newBuilder(WeChatAlertParamsConstants.NAME_ENTERPRISE_WE_CHAT_USERS, WeChatAlertParamsConstants.ENTERPRISE_WE_CHAT_USERS) - .setPlaceholder("use `|` to separate userIds and `@all` to everyone ") - .addValidate(Validate.newBuilder() - .setRequired(false) - .build()) - .build(); + InputParam usersParam = InputParam + .newBuilder(WeChatAlertParamsConstants.NAME_ENTERPRISE_WE_CHAT_USERS, + WeChatAlertParamsConstants.ENTERPRISE_WE_CHAT_USERS) + .setPlaceholder("use `|` to separate userIds and `@all` to everyone ") + .addValidate(Validate.newBuilder() + .setRequired(false) + .build()) + .build(); - InputParam agentIdParam = InputParam.newBuilder(WeChatAlertParamsConstants.NAME_ENTERPRISE_WE_CHAT_AGENT_ID, WeChatAlertParamsConstants.ENTERPRISE_WE_CHAT_AGENT_ID) - .setPlaceholder("please input agent id or chat id ") - .addValidate(Validate.newBuilder() - .setRequired(true) - .build()) - .build(); + InputParam agentIdParam = InputParam + .newBuilder(WeChatAlertParamsConstants.NAME_ENTERPRISE_WE_CHAT_AGENT_ID, + WeChatAlertParamsConstants.ENTERPRISE_WE_CHAT_AGENT_ID) + .setPlaceholder("please input agent id or chat id ") + .addValidate(Validate.newBuilder() + .setRequired(true) + .build()) + .build(); - RadioParam sendType = RadioParam.newBuilder(WeChatAlertParamsConstants.NAME_ENTERPRISE_WE_CHAT_SEND_TYPE, WeChatAlertParamsConstants.ENTERPRISE_WE_CHAT_SEND_TYPE) + RadioParam sendType = RadioParam + .newBuilder(WeChatAlertParamsConstants.NAME_ENTERPRISE_WE_CHAT_SEND_TYPE, + WeChatAlertParamsConstants.ENTERPRISE_WE_CHAT_SEND_TYPE) .addParamsOptions(new ParamsOptions(WeChatType.APP.getDescp(), WeChatType.APP.getDescp(), false)) - .addParamsOptions(new ParamsOptions(WeChatType.APPCHAT.getDescp(), WeChatType.APPCHAT.getDescp(), false)) + .addParamsOptions( + new ParamsOptions(WeChatType.APPCHAT.getDescp(), WeChatType.APPCHAT.getDescp(), false)) .setValue(WeChatType.APP.getDescp()) .addValidate(Validate.newBuilder().setRequired(true).build()) .build(); RadioParam showType = RadioParam.newBuilder(AlertConstants.NAME_SHOW_TYPE, AlertConstants.SHOW_TYPE) - .addParamsOptions(new ParamsOptions(ShowType.MARKDOWN.getDescp(), ShowType.MARKDOWN.getDescp(), false)) - .addParamsOptions(new ParamsOptions(ShowType.TEXT.getDescp(), ShowType.TEXT.getDescp(), false)) - .setValue(ShowType.MARKDOWN.getDescp()) - .addValidate(Validate.newBuilder().setRequired(true).build()) - .build(); + .addParamsOptions(new ParamsOptions(ShowType.MARKDOWN.getDescp(), ShowType.MARKDOWN.getDescp(), false)) + .addParamsOptions(new ParamsOptions(ShowType.TEXT.getDescp(), ShowType.TEXT.getDescp(), false)) + .setValue(ShowType.MARKDOWN.getDescp()) + .addValidate(Validate.newBuilder().setRequired(true).build()) + .build(); return Arrays.asList(corpIdParam, secretParam, usersParam, agentIdParam, sendType, showType); } diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-wechat/src/main/java/org/apache/dolphinscheduler/plugin/alert/wechat/WeChatAlertConstants.java b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-wechat/src/main/java/org/apache/dolphinscheduler/plugin/alert/wechat/WeChatAlertConstants.java index 36ebc1c8d4d2..7f5eaef4f968 100644 --- a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-wechat/src/main/java/org/apache/dolphinscheduler/plugin/alert/wechat/WeChatAlertConstants.java +++ b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-wechat/src/main/java/org/apache/dolphinscheduler/plugin/alert/wechat/WeChatAlertConstants.java @@ -18,6 +18,7 @@ package org.apache.dolphinscheduler.plugin.alert.wechat; public final class WeChatAlertConstants { + static final String MARKDOWN_QUOTE = ">"; static final String MARKDOWN_ENTER = "\n"; @@ -29,7 +30,8 @@ public final class WeChatAlertConstants { static final String WE_CHAT_APP_CHAT_PUSH_URL = "https://qyapi.weixin.qq.com/cgi-bin/appchat/send?access_token" + "={token}"; - static final String WE_CHAT_TOKEN_URL = "https://qyapi.weixin.qq.com/cgi-bin/gettoken?corpid={corpId}&corpsecret={secret}"; + static final String WE_CHAT_TOKEN_URL = + "https://qyapi.weixin.qq.com/cgi-bin/gettoken?corpid={corpId}&corpsecret={secret}"; static final String WE_CHAT_CONTENT_KEY = "content"; diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-wechat/src/main/java/org/apache/dolphinscheduler/plugin/alert/wechat/WeChatAlertParamsConstants.java b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-wechat/src/main/java/org/apache/dolphinscheduler/plugin/alert/wechat/WeChatAlertParamsConstants.java index d8085b901fc3..9511b04f88a5 100644 --- a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-wechat/src/main/java/org/apache/dolphinscheduler/plugin/alert/wechat/WeChatAlertParamsConstants.java +++ b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-wechat/src/main/java/org/apache/dolphinscheduler/plugin/alert/wechat/WeChatAlertParamsConstants.java @@ -18,6 +18,7 @@ package org.apache.dolphinscheduler.plugin.alert.wechat; public final class WeChatAlertParamsConstants { + static final String ENTERPRISE_WE_CHAT_CORP_ID = "$t('corpId')"; static final String NAME_ENTERPRISE_WE_CHAT_CORP_ID = "corpId"; static final String ENTERPRISE_WE_CHAT_SECRET = "$t('secret')"; diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-wechat/src/main/java/org/apache/dolphinscheduler/plugin/alert/wechat/WeChatSender.java b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-wechat/src/main/java/org/apache/dolphinscheduler/plugin/alert/wechat/WeChatSender.java index b8ed73d58700..78408c5ba722 100644 --- a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-wechat/src/main/java/org/apache/dolphinscheduler/plugin/alert/wechat/WeChatSender.java +++ b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-wechat/src/main/java/org/apache/dolphinscheduler/plugin/alert/wechat/WeChatSender.java @@ -17,10 +17,16 @@ package org.apache.dolphinscheduler.plugin.alert.wechat; +import static java.util.Objects.requireNonNull; +import static org.apache.dolphinscheduler.plugin.alert.wechat.WeChatAlertConstants.WE_CHAT_DUPLICATE_CHECK_INTERVAL_ZERO; +import static org.apache.dolphinscheduler.plugin.alert.wechat.WeChatAlertConstants.WE_CHAT_ENABLE_ID_TRANS; +import static org.apache.dolphinscheduler.plugin.alert.wechat.WeChatAlertConstants.WE_CHAT_MESSAGE_SAFE_PUBLICITY; + import org.apache.dolphinscheduler.alert.api.AlertConstants; import org.apache.dolphinscheduler.alert.api.AlertResult; import org.apache.dolphinscheduler.spi.utils.JSONUtils; import org.apache.dolphinscheduler.spi.utils.StringUtils; + import org.apache.http.HttpEntity; import org.apache.http.client.methods.CloseableHttpResponse; import org.apache.http.client.methods.HttpGet; @@ -29,8 +35,6 @@ import org.apache.http.impl.client.CloseableHttpClient; import org.apache.http.impl.client.HttpClients; import org.apache.http.util.EntityUtils; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import java.io.IOException; import java.util.HashMap; @@ -40,11 +44,11 @@ import java.util.Map.Entry; import java.util.Set; - -import static java.util.Objects.requireNonNull; -import static org.apache.dolphinscheduler.plugin.alert.wechat.WeChatAlertConstants.*; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; public final class WeChatSender { + private static final Logger logger = LoggerFactory.getLogger(WeChatSender.class); private static final String MUST_NOT_NULL = " must not null"; private static final String ALERT_STATUS = "false"; @@ -210,11 +214,15 @@ public AlertResult sendEnterpriseWeChat(String title, String content) { String msgJson = ""; if (sendType.equals(WeChatType.APP.getDescp())) { enterpriseWeChatPushUrlReplace = WeChatAlertConstants.WE_CHAT_PUSH_URL.replace(TOKEN_REGEX, weChatToken); - WechatAppMessage wechatAppMessage = new WechatAppMessage(weChatUsers, showType, Integer.valueOf(weChatAgentIdChatId), contentMap, WE_CHAT_MESSAGE_SAFE_PUBLICITY, WE_CHAT_ENABLE_ID_TRANS, WE_CHAT_DUPLICATE_CHECK_INTERVAL_ZERO); + WechatAppMessage wechatAppMessage = new WechatAppMessage(weChatUsers, showType, + Integer.valueOf(weChatAgentIdChatId), contentMap, WE_CHAT_MESSAGE_SAFE_PUBLICITY, + WE_CHAT_ENABLE_ID_TRANS, WE_CHAT_DUPLICATE_CHECK_INTERVAL_ZERO); msgJson = JSONUtils.toJsonString(wechatAppMessage); } else if (sendType.equals(WeChatType.APPCHAT.getDescp())) { - enterpriseWeChatPushUrlReplace = WeChatAlertConstants.WE_CHAT_APP_CHAT_PUSH_URL.replace(TOKEN_REGEX, weChatToken); - WechatAppChatMessage wechatAppChatMessage = new WechatAppChatMessage(weChatAgentIdChatId, showType, contentMap, WE_CHAT_MESSAGE_SAFE_PUBLICITY); + enterpriseWeChatPushUrlReplace = + WeChatAlertConstants.WE_CHAT_APP_CHAT_PUSH_URL.replace(TOKEN_REGEX, weChatToken); + WechatAppChatMessage wechatAppChatMessage = + new WechatAppChatMessage(weChatAgentIdChatId, showType, contentMap, WE_CHAT_MESSAGE_SAFE_PUBLICITY); msgJson = JSONUtils.toJsonString(wechatAppChatMessage); } @@ -248,6 +256,7 @@ private String getToken() { } static final class WeChatSendMsgResponse { + private Integer errcode; private String errmsg; @@ -302,7 +311,8 @@ public int hashCode() { } public String toString() { - return "WeChatSender.WeChatSendMsgResponse(errcode=" + this.getErrcode() + ", errmsg=" + this.getErrmsg() + ")"; + return "WeChatSender.WeChatSendMsgResponse(errcode=" + this.getErrcode() + ", errmsg=" + this.getErrmsg() + + ")"; } } } diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-wechat/src/main/java/org/apache/dolphinscheduler/plugin/alert/wechat/WeChatType.java b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-wechat/src/main/java/org/apache/dolphinscheduler/plugin/alert/wechat/WeChatType.java index 9060b530898c..109da599441e 100644 --- a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-wechat/src/main/java/org/apache/dolphinscheduler/plugin/alert/wechat/WeChatType.java +++ b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-wechat/src/main/java/org/apache/dolphinscheduler/plugin/alert/wechat/WeChatType.java @@ -15,10 +15,10 @@ * limitations under the License. */ - package org.apache.dolphinscheduler.plugin.alert.wechat; public enum WeChatType { + APP(1, "APP/应用"), APPCHAT(2, "GROUP CHAT/群聊"), ; diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-wechat/src/main/java/org/apache/dolphinscheduler/plugin/alert/wechat/WechatAppChatMessage.java b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-wechat/src/main/java/org/apache/dolphinscheduler/plugin/alert/wechat/WechatAppChatMessage.java index e04f7335b83d..9c16f759fa03 100644 --- a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-wechat/src/main/java/org/apache/dolphinscheduler/plugin/alert/wechat/WechatAppChatMessage.java +++ b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-wechat/src/main/java/org/apache/dolphinscheduler/plugin/alert/wechat/WechatAppChatMessage.java @@ -25,8 +25,8 @@ public class WechatAppChatMessage { private String chatid; private String msgtype; - private Map text; - private Map markdown; + private Map text; + private Map markdown; private Integer safe; public String getChatid() { diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-wechat/src/main/java/org/apache/dolphinscheduler/plugin/alert/wechat/WechatAppMessage.java b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-wechat/src/main/java/org/apache/dolphinscheduler/plugin/alert/wechat/WechatAppMessage.java index dbb74a6a93d7..f5ec7183bade 100644 --- a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-wechat/src/main/java/org/apache/dolphinscheduler/plugin/alert/wechat/WechatAppMessage.java +++ b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-wechat/src/main/java/org/apache/dolphinscheduler/plugin/alert/wechat/WechatAppMessage.java @@ -99,7 +99,8 @@ public void setMarkdown(Map markdown) { public WechatAppMessage() { } - public WechatAppMessage(String touser, String msgtype, Integer agentid, Map contentMap, Integer safe, Integer enableIdTrans, Integer enableDuplicateCheck) { + public WechatAppMessage(String touser, String msgtype, Integer agentid, Map contentMap, + Integer safe, Integer enableIdTrans, Integer enableDuplicateCheck) { this.touser = touser; this.msgtype = msgtype; this.agentid = agentid; diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/pom.xml b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/pom.xml index a014f8b17c5c..df705e9ea22c 100644 --- a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/pom.xml +++ b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/pom.xml @@ -15,15 +15,14 @@ ~ See the License for the specific language governing permissions and ~ limitations under the License. --> - + 4.0.0 - dolphinscheduler-alert org.apache.dolphinscheduler + dolphinscheduler-alert dev-SNAPSHOT - 4.0.0 dolphinscheduler-alert-plugins pom diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/java/org/apache/dolphinscheduler/alert/AlertPluginManager.java b/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/java/org/apache/dolphinscheduler/alert/AlertPluginManager.java index 782bf0aa85bf..78ff13d98729 100644 --- a/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/java/org/apache/dolphinscheduler/alert/AlertPluginManager.java +++ b/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/java/org/apache/dolphinscheduler/alert/AlertPluginManager.java @@ -30,11 +30,6 @@ import org.apache.dolphinscheduler.spi.params.base.Validate; import org.apache.dolphinscheduler.spi.params.radio.RadioParam; import org.apache.dolphinscheduler.spi.plugin.PrioritySPIFactory; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.springframework.boot.context.event.ApplicationReadyEvent; -import org.springframework.context.event.EventListener; -import org.springframework.stereotype.Component; import java.util.ArrayList; import java.util.HashMap; @@ -42,10 +37,15 @@ import java.util.Map; import java.util.Optional; -import static java.lang.String.format; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.boot.context.event.ApplicationReadyEvent; +import org.springframework.context.event.EventListener; +import org.springframework.stereotype.Component; @Component public final class AlertPluginManager { + private static final Logger logger = LoggerFactory.getLogger(AlertPluginManager.class); private final PluginDao pluginDao; @@ -59,20 +59,22 @@ public AlertPluginManager(PluginDao pluginDao) { private final PluginParams warningTypeParams = getWarningTypeParams(); public PluginParams getWarningTypeParams() { - return - RadioParam.newBuilder(AlertConstants.NAME_WARNING_TYPE, AlertConstants.WARNING_TYPE) - .addParamsOptions(new ParamsOptions(WarningType.SUCCESS.getDescp(), WarningType.SUCCESS.getDescp(), false)) - .addParamsOptions(new ParamsOptions(WarningType.FAILURE.getDescp(), WarningType.FAILURE.getDescp(), false)) - .addParamsOptions(new ParamsOptions(WarningType.ALL.getDescp(), WarningType.ALL.getDescp(), false)) - .setValue(WarningType.ALL.getDescp()) - .addValidate(Validate.newBuilder().setRequired(true).build()) - .build(); + return RadioParam.newBuilder(AlertConstants.NAME_WARNING_TYPE, AlertConstants.WARNING_TYPE) + .addParamsOptions( + new ParamsOptions(WarningType.SUCCESS.getDescp(), WarningType.SUCCESS.getDescp(), false)) + .addParamsOptions( + new ParamsOptions(WarningType.FAILURE.getDescp(), WarningType.FAILURE.getDescp(), false)) + .addParamsOptions(new ParamsOptions(WarningType.ALL.getDescp(), WarningType.ALL.getDescp(), false)) + .setValue(WarningType.ALL.getDescp()) + .addValidate(Validate.newBuilder().setRequired(true).build()) + .build(); } @EventListener public void installPlugin(ApplicationReadyEvent readyEvent) { - PrioritySPIFactory prioritySPIFactory = new PrioritySPIFactory<>(AlertChannelFactory.class); + PrioritySPIFactory prioritySPIFactory = + new PrioritySPIFactory<>(AlertChannelFactory.class); for (Map.Entry entry : prioritySPIFactory.getSPIMap().entrySet()) { String name = entry.getKey(); AlertChannelFactory factory = entry.getValue(); diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/java/org/apache/dolphinscheduler/alert/AlertRequestProcessor.java b/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/java/org/apache/dolphinscheduler/alert/AlertRequestProcessor.java index 6bb4aa004993..930d9180bed9 100644 --- a/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/java/org/apache/dolphinscheduler/alert/AlertRequestProcessor.java +++ b/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/java/org/apache/dolphinscheduler/alert/AlertRequestProcessor.java @@ -34,6 +34,7 @@ @Component public final class AlertRequestProcessor implements NettyRequestProcessor { + private static final Logger logger = LoggerFactory.getLogger(AlertRequestProcessor.class); private final AlertSenderService alertSenderService; @@ -44,18 +45,19 @@ public AlertRequestProcessor(AlertSenderService alertSenderService) { @Override public void process(Channel channel, Command command) { - checkArgument(CommandType.ALERT_SEND_REQUEST == command.getType(), "invalid command type: %s", command.getType()); + checkArgument(CommandType.ALERT_SEND_REQUEST == command.getType(), "invalid command type: %s", + command.getType()); AlertSendRequestCommand alertSendRequestCommand = JsonSerializer.deserialize( - command.getBody(), AlertSendRequestCommand.class); + command.getBody(), AlertSendRequestCommand.class); logger.info("Received command : {}", alertSendRequestCommand); AlertSendResponseCommand alertSendResponseCommand = alertSenderService.syncHandler( - alertSendRequestCommand.getGroupId(), - alertSendRequestCommand.getTitle(), - alertSendRequestCommand.getContent(), - alertSendRequestCommand.getWarnType()); + alertSendRequestCommand.getGroupId(), + alertSendRequestCommand.getTitle(), + alertSendRequestCommand.getContent(), + alertSendRequestCommand.getWarnType()); channel.writeAndFlush(alertSendResponseCommand.convert2Command(command.getOpaque())); } } diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/java/org/apache/dolphinscheduler/alert/AlertServerMetrics.java b/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/java/org/apache/dolphinscheduler/alert/AlertServerMetrics.java index 0218e51d2bab..7a8b89a52d0f 100644 --- a/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/java/org/apache/dolphinscheduler/alert/AlertServerMetrics.java +++ b/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/java/org/apache/dolphinscheduler/alert/AlertServerMetrics.java @@ -19,10 +19,10 @@ import java.util.function.Supplier; +import lombok.experimental.UtilityClass; import io.micrometer.core.instrument.Counter; import io.micrometer.core.instrument.Gauge; import io.micrometer.core.instrument.Metrics; -import lombok.experimental.UtilityClass; @UtilityClass public class AlertServerMetrics { diff --git a/dolphinscheduler-alert/pom.xml b/dolphinscheduler-alert/pom.xml index ee7e3795361b..bcf840410d2d 100644 --- a/dolphinscheduler-alert/pom.xml +++ b/dolphinscheduler-alert/pom.xml @@ -17,18 +17,16 @@ ~ specific language governing permissions and limitations ~ under the License. --> - - + 4.0.0 - dolphinscheduler org.apache.dolphinscheduler + dolphinscheduler dev-SNAPSHOT - 4.0.0 - pom dolphinscheduler-alert + pom dolphinscheduler-alert-api diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/ApiApplicationServer.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/ApiApplicationServer.java index 08a73e69e3ab..2994208ea209 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/ApiApplicationServer.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/ApiApplicationServer.java @@ -21,7 +21,6 @@ import org.apache.dolphinscheduler.dao.PluginDao; import org.apache.dolphinscheduler.dao.entity.PluginDefine; import org.apache.dolphinscheduler.plugin.task.api.TaskChannelFactory; -import org.apache.dolphinscheduler.plugin.task.api.TaskPluginException; import org.apache.dolphinscheduler.service.task.TaskPluginManager; import org.apache.dolphinscheduler.spi.params.PluginParamsTransfer; import org.apache.dolphinscheduler.spi.params.base.PluginParams; diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/aspect/AccessLogAnnotation.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/aspect/AccessLogAnnotation.java index db62bcda3afc..dd62af88deac 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/aspect/AccessLogAnnotation.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/aspect/AccessLogAnnotation.java @@ -27,6 +27,7 @@ @Retention(RetentionPolicy.RUNTIME) @Documented public @interface AccessLogAnnotation { + // ignore request args String[] ignoreRequestArgs() default {"loginUser"}; diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/aspect/AccessLogAspect.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/aspect/AccessLogAspect.java index b20b6d84254a..6d78ada123d9 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/aspect/AccessLogAspect.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/aspect/AccessLogAspect.java @@ -47,6 +47,7 @@ @Aspect @Component public class AccessLogAspect { + private static final Logger logger = LoggerFactory.getLogger(AccessLogAspect.class); private static final String TRACE_ID = "traceId"; @@ -56,7 +57,7 @@ public class AccessLogAspect { private static final Pattern sensitiveDataPattern = Pattern.compile(sensitiveDataRegEx, Pattern.CASE_INSENSITIVE); @Pointcut("@annotation(org.apache.dolphinscheduler.api.aspect.AccessLogAnnotation)") - public void logPointCut(){ + public void logPointCut() { // Do nothing because of it's a pointcut } @@ -65,7 +66,7 @@ public Object doAround(ProceedingJoinPoint proceedingJoinPoint) throws Throwable long startTime = System.currentTimeMillis(); // fetch AccessLogAnnotation - MethodSignature sign = (MethodSignature) proceedingJoinPoint.getSignature(); + MethodSignature sign = (MethodSignature) proceedingJoinPoint.getSignature(); Method method = sign.getMethod(); AccessLogAnnotation annotation = method.getAnnotation(AccessLogAnnotation.class); @@ -73,7 +74,8 @@ public Object doAround(ProceedingJoinPoint proceedingJoinPoint) throws Throwable // log request if (!annotation.ignoreRequest()) { - ServletRequestAttributes attributes = (ServletRequestAttributes) RequestContextHolder.getRequestAttributes(); + ServletRequestAttributes attributes = + (ServletRequestAttributes) RequestContextHolder.getRequestAttributes(); if (attributes != null) { HttpServletRequest request = attributes.getRequest(); String traceIdFromHeader = request.getHeader(TRACE_ID); @@ -92,7 +94,8 @@ public Object doAround(ProceedingJoinPoint proceedingJoinPoint) throws Throwable userName, request.getRequestURI(), request.getMethod(), - proceedingJoinPoint.getSignature().getDeclaringTypeName() + "." + proceedingJoinPoint.getSignature().getName(), + proceedingJoinPoint.getSignature().getDeclaringTypeName() + "." + + proceedingJoinPoint.getSignature().getName(), argsString); } @@ -102,7 +105,8 @@ public Object doAround(ProceedingJoinPoint proceedingJoinPoint) throws Throwable // log response if (!annotation.ignoreResponse()) { - logger.info("RESPONSE TRACE_ID:{}, BODY:{}, REQUEST DURATION:{} milliseconds", traceId, ob, (System.currentTimeMillis() - startTime)); + logger.info("RESPONSE TRACE_ID:{}, BODY:{}, REQUEST DURATION:{} milliseconds", traceId, ob, + (System.currentTimeMillis() - startTime)); } return ob; @@ -134,7 +138,7 @@ protected String handleSensitiveData(String originalData) { boolean exists = false; while (matcher.find()) { if (matcher.groupCount() == 3) { - stream = IntStream.concat(stream, IntStream.range(matcher.end(1),matcher.end(2))); + stream = IntStream.concat(stream, IntStream.range(matcher.end(1), matcher.end(2))); exists = true; } } diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/audit/AuditMessage.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/audit/AuditMessage.java index 01ef5a28e884..0d936f0ad5e1 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/audit/AuditMessage.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/audit/AuditMessage.java @@ -24,6 +24,7 @@ import java.util.Date; public class AuditMessage { + private User user; private Date auditDate; @@ -34,7 +35,8 @@ public class AuditMessage { private Integer resourceId; - public AuditMessage(User user, Date auditDate, AuditResourceType resourceType, AuditOperationType operation, Integer resourceId) { + public AuditMessage(User user, Date auditDate, AuditResourceType resourceType, AuditOperationType operation, + Integer resourceId) { this.user = user; this.auditDate = auditDate; this.resourceType = resourceType; diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/configuration/AuditConfiguration.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/configuration/AuditConfiguration.java index 090b46888f18..451fcbc40ba9 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/configuration/AuditConfiguration.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/configuration/AuditConfiguration.java @@ -25,6 +25,7 @@ @EnableConfigurationProperties @ConfigurationProperties(value = "audit", ignoreUnknownFields = false) public class AuditConfiguration { + private boolean enabled; public boolean getEnabled() { @@ -34,4 +35,4 @@ public boolean getEnabled() { public void setEnabled(boolean enabled) { this.enabled = enabled; } -} \ No newline at end of file +} diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/configuration/OpenAPITranslationConfiguration.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/configuration/OpenAPITranslationConfiguration.java index 735f71f9ef9f..4e1c67bd0be7 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/configuration/OpenAPITranslationConfiguration.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/configuration/OpenAPITranslationConfiguration.java @@ -18,20 +18,24 @@ import static java.util.stream.Collectors.toList; import static java.util.stream.Collectors.toSet; + +import springfox.documentation.service.Operation; +import springfox.documentation.service.RequestParameter; +import springfox.documentation.spi.DocumentationType; +import springfox.documentation.spi.service.OperationBuilderPlugin; +import springfox.documentation.spi.service.contexts.OperationContext; + import java.util.Collection; import java.util.Locale; import java.util.Set; + +import lombok.RequiredArgsConstructor; + import org.springframework.context.MessageSource; import org.springframework.context.i18n.LocaleContextHolder; import org.springframework.core.Ordered; import org.springframework.core.annotation.Order; import org.springframework.stereotype.Component; -import lombok.RequiredArgsConstructor; -import springfox.documentation.service.Operation; -import springfox.documentation.service.RequestParameter; -import springfox.documentation.spi.DocumentationType; -import springfox.documentation.spi.service.OperationBuilderPlugin; -import springfox.documentation.spi.service.contexts.OperationContext; @Component public class OpenAPITranslationConfiguration { @@ -60,20 +64,20 @@ public void apply(OperationContext context) { .collect(toSet()); Collection parameters = operation.getRequestParameters().stream() - .map(it -> new RequestParameter( - it.getName(), - it.getIn(), - messageSource.getMessage(it.getDescription(), null, it.getDescription(), locale), - it.getRequired(), - it.getDeprecated(), - it.getHidden(), - it.getParameterSpecification(), - it.getScalarExample(), - it.getExamples(), - it.getPrecedence(), - it.getExtensions(), - it.getParameterIndex())) - .collect(toList()); + .map(it -> new RequestParameter( + it.getName(), + it.getIn(), + messageSource.getMessage(it.getDescription(), null, it.getDescription(), locale), + it.getRequired(), + it.getDeprecated(), + it.getHidden(), + it.getParameterSpecification(), + it.getScalarExample(), + it.getExamples(), + it.getPrecedence(), + it.getExtensions(), + it.getParameterIndex())) + .collect(toList()); context.operationBuilder() .notes(notes) diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/configuration/PythonGatewayConfiguration.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/configuration/PythonGatewayConfiguration.java index 5735e27fd2b9..fcfa973424d3 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/configuration/PythonGatewayConfiguration.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/configuration/PythonGatewayConfiguration.java @@ -25,6 +25,7 @@ @EnableConfigurationProperties @ConfigurationProperties(value = "python-gateway", ignoreUnknownFields = false) public class PythonGatewayConfiguration { + private boolean enabled; private String gatewayServerAddress; private int gatewayServerPort; diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/configuration/TaskTypeConfiguration.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/configuration/TaskTypeConfiguration.java index 64b6cc3e4380..649e32c088c3 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/configuration/TaskTypeConfiguration.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/configuration/TaskTypeConfiguration.java @@ -24,6 +24,9 @@ import java.util.ArrayList; import java.util.List; +import lombok.Getter; +import lombok.Setter; + import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.boot.context.properties.ConfigurationProperties; @@ -31,9 +34,6 @@ import org.springframework.context.annotation.PropertySource; import org.springframework.stereotype.Component; -import lombok.Getter; -import lombok.Setter; - @Component @EnableConfigurationProperties @PropertySource(value = {"classpath:task-type-config.yaml"}, factory = YamlPropertySourceFactory.class) @@ -62,9 +62,11 @@ public List getDefaultTaskTypes() { universal.forEach(task -> defaultTaskTypes.add(new FavTaskDto(task, false, Constants.TYPE_UNIVERSAL))); cloud.forEach(task -> defaultTaskTypes.add(new FavTaskDto(task, false, Constants.TYPE_CLOUD))); logic.forEach(task -> defaultTaskTypes.add(new FavTaskDto(task, false, Constants.TYPE_LOGIC))); - dataIntegration.forEach(task -> defaultTaskTypes.add(new FavTaskDto(task, false, Constants.TYPE_DATA_INTEGRATION))); + dataIntegration + .forEach(task -> defaultTaskTypes.add(new FavTaskDto(task, false, Constants.TYPE_DATA_INTEGRATION))); dataQuality.forEach(task -> defaultTaskTypes.add(new FavTaskDto(task, false, Constants.TYPE_DATA_QUALITY))); - machineLearning.forEach(task -> defaultTaskTypes.add(new FavTaskDto(task, false, Constants.TYPE_MACHINE_LEARNING))); + machineLearning + .forEach(task -> defaultTaskTypes.add(new FavTaskDto(task, false, Constants.TYPE_MACHINE_LEARNING))); other.forEach(task -> defaultTaskTypes.add(new FavTaskDto(task, false, Constants.TYPE_OTHER))); return defaultTaskTypes; } diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/configuration/TrafficConfiguration.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/configuration/TrafficConfiguration.java index 3b66a2d6ca3b..47cb5230f393 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/configuration/TrafficConfiguration.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/configuration/TrafficConfiguration.java @@ -17,11 +17,11 @@ package org.apache.dolphinscheduler.api.configuration; -import lombok.Data; - import java.util.HashMap; import java.util.Map; +import lombok.Data; + import org.springframework.boot.context.properties.ConfigurationProperties; import org.springframework.context.annotation.Configuration; diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/constants/ApiFuncIdentificationConstant.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/constants/ApiFuncIdentificationConstant.java index 035d582e0659..b93792307f73 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/constants/ApiFuncIdentificationConstant.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/constants/ApiFuncIdentificationConstant.java @@ -18,6 +18,7 @@ package org.apache.dolphinscheduler.api.constants; import org.apache.dolphinscheduler.api.enums.ExecuteType; + import java.util.HashMap; import java.util.Map; @@ -30,7 +31,7 @@ public class ApiFuncIdentificationConstant { public static final String ACCESS_TOKEN_DELETE = "security:token:delete"; public static final String ALERT_GROUP_VIEW = "security:alert-group:view"; public static final String ALERT_GROUP_CREATE = "security:alert-group:create"; - public static final String ALERT_GROUP_UPDATE = "security:alert-group:update"; + public static final String ALERT_GROUP_UPDATE = "security:alert-group:update"; public static final String ALERT_GROUP_DELETE = "security:alert-group:delete"; public static final String TENANT_MANAGER = "security:tenant:view"; public static final String TENANT_CREATE = "security:tenant:create"; @@ -156,28 +157,28 @@ public class ApiFuncIdentificationConstant { public static final String MONITOR_STATISTICS_VIEW = "monitor:statistics:view"; - public final static Map map = new HashMap(); + public final static Map map = new HashMap(); - static{ - for(ExecuteType type : ExecuteType.values()){ - switch (type){ + static { + for (ExecuteType type : ExecuteType.values()) { + switch (type) { case REPEAT_RUNNING: - map.put(type,RERUN); + map.put(type, RERUN); break; case RECOVER_SUSPENDED_PROCESS: - map.put(type,RECOVERY_SUSPEND); + map.put(type, RECOVERY_SUSPEND); break; case START_FAILURE_TASK_PROCESS: - map.put(type,FAILED_TO_RETRY); + map.put(type, FAILED_TO_RETRY); break; case STOP: - map.put(type,STOP); + map.put(type, STOP); break; case PAUSE: - map.put(type,PAUSE); + map.put(type, PAUSE); break; case NONE: - map.put(type,null); + map.put(type, null); break; default: } diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/AccessTokenController.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/AccessTokenController.java index 82142768ce63..8201eedfe26c 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/AccessTokenController.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/AccessTokenController.java @@ -29,8 +29,8 @@ import org.apache.dolphinscheduler.api.service.AccessTokenService; import org.apache.dolphinscheduler.api.utils.Result; import org.apache.dolphinscheduler.common.Constants; -import org.apache.dolphinscheduler.plugin.task.api.utils.ParameterUtils; import org.apache.dolphinscheduler.dao.entity.User; +import org.apache.dolphinscheduler.plugin.task.api.utils.ParameterUtils; import springfox.documentation.annotations.ApiIgnore; diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/AlertGroupController.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/AlertGroupController.java index 6453de0276e6..853336b3b716 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/AlertGroupController.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/AlertGroupController.java @@ -30,8 +30,8 @@ import org.apache.dolphinscheduler.api.service.AlertGroupService; import org.apache.dolphinscheduler.api.utils.Result; import org.apache.dolphinscheduler.common.Constants; -import org.apache.dolphinscheduler.plugin.task.api.utils.ParameterUtils; import org.apache.dolphinscheduler.dao.entity.User; +import org.apache.dolphinscheduler.plugin.task.api.utils.ParameterUtils; import springfox.documentation.annotations.ApiIgnore; diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/AlertPluginInstanceController.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/AlertPluginInstanceController.java index a2ac7e04108d..af805aa7e931 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/AlertPluginInstanceController.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/AlertPluginInstanceController.java @@ -32,6 +32,7 @@ import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.dao.entity.User; import org.apache.dolphinscheduler.plugin.task.api.utils.ParameterUtils; + import springfox.documentation.annotations.ApiIgnore; import java.util.Map; diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/ClusterController.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/ClusterController.java index c8dcdff0396f..e247688ec79f 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/ClusterController.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/ClusterController.java @@ -29,8 +29,8 @@ import org.apache.dolphinscheduler.api.service.ClusterService; import org.apache.dolphinscheduler.api.utils.Result; import org.apache.dolphinscheduler.common.Constants; -import org.apache.dolphinscheduler.plugin.task.api.utils.ParameterUtils; import org.apache.dolphinscheduler.dao.entity.User; +import org.apache.dolphinscheduler.plugin.task.api.utils.ParameterUtils; import springfox.documentation.annotations.ApiIgnore; diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/DataQualityController.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/DataQualityController.java index 04203651fb65..9b4e3712408f 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/DataQualityController.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/DataQualityController.java @@ -28,8 +28,8 @@ import org.apache.dolphinscheduler.api.service.DqRuleService; import org.apache.dolphinscheduler.api.utils.Result; import org.apache.dolphinscheduler.common.Constants; -import org.apache.dolphinscheduler.plugin.task.api.utils.ParameterUtils; import org.apache.dolphinscheduler.dao.entity.User; +import org.apache.dolphinscheduler.plugin.task.api.utils.ParameterUtils; import springfox.documentation.annotations.ApiIgnore; diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/DataSourceController.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/DataSourceController.java index b34407366446..3d9961e99bc5 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/DataSourceController.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/DataSourceController.java @@ -36,10 +36,10 @@ import org.apache.dolphinscheduler.api.service.DataSourceService; import org.apache.dolphinscheduler.api.utils.Result; import org.apache.dolphinscheduler.common.Constants; -import org.apache.dolphinscheduler.plugin.task.api.utils.ParameterUtils; import org.apache.dolphinscheduler.dao.entity.User; import org.apache.dolphinscheduler.plugin.datasource.api.datasource.BaseDataSourceParamDTO; import org.apache.dolphinscheduler.plugin.datasource.api.utils.DataSourceUtils; +import org.apache.dolphinscheduler.plugin.task.api.utils.ParameterUtils; import org.apache.dolphinscheduler.service.utils.CommonUtils; import org.apache.dolphinscheduler.spi.datasource.ConnectionParam; import org.apache.dolphinscheduler.spi.enums.DbType; diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/DsErrorController.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/DsErrorController.java index f42df4f4e162..e97ffcfe7bc0 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/DsErrorController.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/DsErrorController.java @@ -21,6 +21,7 @@ import javax.servlet.RequestDispatcher; import javax.servlet.http.HttpServletRequest; + import org.springframework.boot.web.servlet.error.ErrorController; import org.springframework.http.HttpStatus; import org.springframework.stereotype.Controller; @@ -29,6 +30,7 @@ @Controller public class DsErrorController implements ErrorController { + @RequestMapping("/error") public ModelAndView handleError(HttpServletRequest request) { Object status = request.getAttribute(RequestDispatcher.ERROR_STATUS_CODE); diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/EnvironmentController.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/EnvironmentController.java index 72ea72d1b057..fb1bf113801a 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/EnvironmentController.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/EnvironmentController.java @@ -29,8 +29,8 @@ import org.apache.dolphinscheduler.api.service.EnvironmentService; import org.apache.dolphinscheduler.api.utils.Result; import org.apache.dolphinscheduler.common.Constants; -import org.apache.dolphinscheduler.plugin.task.api.utils.ParameterUtils; import org.apache.dolphinscheduler.dao.entity.User; +import org.apache.dolphinscheduler.plugin.task.api.utils.ParameterUtils; import springfox.documentation.annotations.ApiIgnore; diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/ExecutorController.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/ExecutorController.java index 98e69ab1f4f1..f672702b8546 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/ExecutorController.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/ExecutorController.java @@ -121,7 +121,7 @@ public class ExecutorController extends BaseController { @ApiImplicitParam(name = "workerGroup", value = "WORKER_GROUP", dataType = "String", example = "default"), @ApiImplicitParam(name = "environmentCode", value = "ENVIRONMENT_CODE", dataType = "Long", example = "-1"), @ApiImplicitParam(name = "timeout", value = "TIMEOUT", dataType = "Int", example = "100"), - @ApiImplicitParam(name = "expectedParallelismNumber", value = "EXPECTED_PARALLELISM_NUMBER", dataType = "Int" , example = "8"), + @ApiImplicitParam(name = "expectedParallelismNumber", value = "EXPECTED_PARALLELISM_NUMBER", dataType = "Int", example = "8"), @ApiImplicitParam(name = "dryRun", value = "DRY_RUN", dataType = "Int", example = "0"), @ApiImplicitParam(name = "testFlag", value = "TEST_FLAG", dataType = "Int", example = "0"), @ApiImplicitParam(name = "complementDependentMode", value = "COMPLEMENT_DEPENDENT_MODE", dataType = "complementDependentMode") @@ -166,7 +166,8 @@ public Result startProcessInstance(@ApiIgnore @RequestAttribute(value = Constant Map result = execService.execProcessInstance(loginUser, projectCode, processDefinitionCode, scheduleTime, execType, failureStrategy, startNodeList, taskDependType, warningType, warningGroupId, runMode, processInstancePriority, - workerGroup, environmentCode, timeout, startParamMap, expectedParallelismNumber, dryRun, testFlag, complementDependentMode); + workerGroup, environmentCode, timeout, startParamMap, expectedParallelismNumber, dryRun, testFlag, + complementDependentMode); return returnDataList(result); } @@ -268,10 +269,12 @@ public Result batchStartProcessInstance(@ApiIgnore @RequestAttribute(value = Con complementDependentMode); if (!Status.SUCCESS.equals(result.get(Constants.STATUS))) { - logger.error("Process definition start failed, projectCode:{}, processDefinitionCode:{}.", projectCode, processDefinitionCode); + logger.error("Process definition start failed, projectCode:{}, processDefinitionCode:{}.", projectCode, + processDefinitionCode); startFailedProcessDefinitionCodeList.add(String.valueOf(processDefinitionCode)); } else - logger.info("Start process definition complete, projectCode:{}, processDefinitionCode:{}.", projectCode, processDefinitionCode); + logger.info("Start process definition complete, projectCode:{}, processDefinitionCode:{}.", projectCode, + processDefinitionCode); } if (!startFailedProcessDefinitionCodeList.isEmpty()) { @@ -303,9 +306,9 @@ public Result batchStartProcessInstance(@ApiIgnore @RequestAttribute(value = Con public Result execute(@ApiIgnore @RequestAttribute(value = Constants.SESSION_USER) User loginUser, @ApiParam(name = "projectCode", value = "PROJECT_CODE", required = true) @PathVariable long projectCode, @RequestParam("processInstanceId") Integer processInstanceId, - @RequestParam("executeType") ExecuteType executeType - ) { - logger.info("Start to execute process instance, projectCode:{}, processInstanceId:{}.", projectCode, processInstanceId); + @RequestParam("executeType") ExecuteType executeType) { + logger.info("Start to execute process instance, projectCode:{}, processInstanceId:{}.", projectCode, + processInstanceId); Map result = execService.execute(loginUser, projectCode, processInstanceId, executeType); return returnDataList(result); } @@ -344,10 +347,12 @@ public Result batchExecute(@RequestAttribute(value = Constants.SESSION_USER) Use Map singleResult = execService.execute(loginUser, projectCode, processInstanceId, executeType); if (!Status.SUCCESS.equals(singleResult.get(Constants.STATUS))) { - logger.error("Start to execute process instance error, projectCode:{}, processInstanceId:{}.", projectCode, processInstanceId); + logger.error("Start to execute process instance error, projectCode:{}, processInstanceId:{}.", + projectCode, processInstanceId); executeFailedIdList.add((String) singleResult.get(Constants.MSG)); } else - logger.info("Start to execute process instance complete, projectCode:{}, processInstanceId:{}.", projectCode, processInstanceId); + logger.info("Start to execute process instance complete, projectCode:{}, processInstanceId:{}.", + projectCode, processInstanceId); } catch (Exception e) { executeFailedIdList .add(MessageFormat.format(Status.PROCESS_INSTANCE_ERROR.getMsg(), strProcessInstanceId)); @@ -440,7 +445,8 @@ public Result startStreamTaskInstance(@ApiIgnore @RequestAttribute(value = Const startParamMap = JSONUtils.toMap(startParams); } - logger.info("Start to execute stream task instance, projectCode:{}, taskDefinitionCode:{}, taskVersion:{}.", projectCode, code, version); + logger.info("Start to execute stream task instance, projectCode:{}, taskDefinitionCode:{}, taskVersion:{}.", + projectCode, code, version); Map result = execService.execStreamTaskInstance(loginUser, projectCode, code, version, warningGroupId, workerGroup, environmentCode, startParamMap, dryRun); return returnDataList(result); diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/FavTaskController.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/FavTaskController.java index 98d38373f3a9..fb54b67283e9 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/FavTaskController.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/FavTaskController.java @@ -17,8 +17,10 @@ package org.apache.dolphinscheduler.api.controller; -import io.swagger.annotations.Api; -import io.swagger.annotations.ApiOperation; +import static org.apache.dolphinscheduler.api.enums.Status.ADD_TASK_TYPE_ERROR; +import static org.apache.dolphinscheduler.api.enums.Status.DELETE_TASK_TYPE_ERROR; +import static org.apache.dolphinscheduler.api.enums.Status.LIST_TASK_TYPE_ERROR; + import org.apache.dolphinscheduler.api.aspect.AccessLogAnnotation; import org.apache.dolphinscheduler.api.dto.FavTaskDto; import org.apache.dolphinscheduler.api.enums.Status; @@ -27,6 +29,13 @@ import org.apache.dolphinscheduler.api.utils.Result; import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.dao.entity.User; + +import springfox.documentation.annotations.ApiIgnore; + +import java.util.List; + +import javax.annotation.Resource; + import org.springframework.http.HttpStatus; import org.springframework.web.bind.annotation.DeleteMapping; import org.springframework.web.bind.annotation.GetMapping; @@ -36,14 +45,9 @@ import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.ResponseStatus; import org.springframework.web.bind.annotation.RestController; -import springfox.documentation.annotations.ApiIgnore; - -import javax.annotation.Resource; -import java.util.List; -import static org.apache.dolphinscheduler.api.enums.Status.ADD_TASK_TYPE_ERROR; -import static org.apache.dolphinscheduler.api.enums.Status.DELETE_TASK_TYPE_ERROR; -import static org.apache.dolphinscheduler.api.enums.Status.LIST_TASK_TYPE_ERROR; +import io.swagger.annotations.Api; +import io.swagger.annotations.ApiOperation; /** * fav controller diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/K8sNamespaceController.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/K8sNamespaceController.java index 06a5adfdeb49..9e95fd0e0937 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/K8sNamespaceController.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/K8sNamespaceController.java @@ -31,9 +31,9 @@ import org.apache.dolphinscheduler.api.service.K8sNamespaceService; import org.apache.dolphinscheduler.api.utils.Result; import org.apache.dolphinscheduler.common.Constants; -import org.apache.dolphinscheduler.plugin.task.api.utils.ParameterUtils; import org.apache.dolphinscheduler.dao.entity.K8sNamespace; import org.apache.dolphinscheduler.dao.entity.User; +import org.apache.dolphinscheduler.plugin.task.api.utils.ParameterUtils; import springfox.documentation.annotations.ApiIgnore; diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/MonitorController.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/MonitorController.java index 9240ac45d856..1acbc99a40f3 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/MonitorController.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/MonitorController.java @@ -28,6 +28,8 @@ import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.dao.entity.User; +import springfox.documentation.annotations.ApiIgnore; + import java.util.Map; import org.springframework.beans.factory.annotation.Autowired; @@ -40,7 +42,6 @@ import io.swagger.annotations.Api; import io.swagger.annotations.ApiOperation; -import springfox.documentation.annotations.ApiIgnore; /** * monitor controller diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/ProcessDefinitionController.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/ProcessDefinitionController.java index 55c66a6adb6e..dc29875ec460 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/ProcessDefinitionController.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/ProcessDefinitionController.java @@ -38,26 +38,19 @@ import org.apache.dolphinscheduler.api.aspect.AccessLogAnnotation; import org.apache.dolphinscheduler.api.enums.Status; import org.apache.dolphinscheduler.api.exceptions.ApiException; -import org.apache.dolphinscheduler.api.exceptions.ServiceException; import org.apache.dolphinscheduler.api.service.ProcessDefinitionService; import org.apache.dolphinscheduler.api.utils.PageInfo; import org.apache.dolphinscheduler.api.utils.Result; import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.enums.ProcessExecutionTypeEnum; import org.apache.dolphinscheduler.common.enums.ReleaseState; -import org.apache.dolphinscheduler.plugin.task.api.utils.ParameterUtils; import org.apache.dolphinscheduler.dao.entity.ProcessDefinition; import org.apache.dolphinscheduler.dao.entity.User; +import org.apache.dolphinscheduler.plugin.task.api.utils.ParameterUtils; import springfox.documentation.annotations.ApiIgnore; -import org.apache.commons.lang3.StringUtils; - -import java.text.MessageFormat; -import java.util.HashMap; -import java.util.HashSet; import java.util.Map; -import java.util.Set; import javax.servlet.http.HttpServletResponse; @@ -685,7 +678,8 @@ public Result batchDeleteProcessDefinitionByCodes(@ApiIgnore @RequestAttribute(v @ApiParam(name = "projectCode", value = "PROJECT_CODE", required = true) @PathVariable long projectCode, @RequestParam("codes") String codes) { - Map result = processDefinitionService.batchDeleteProcessDefinitionByCodes(loginUser, projectCode, codes); + Map result = + processDefinitionService.batchDeleteProcessDefinitionByCodes(loginUser, projectCode, codes); return returnDataList(result); } diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/ProcessInstanceController.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/ProcessInstanceController.java index b36ba8abd542..ebc5edde85ac 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/ProcessInstanceController.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/ProcessInstanceController.java @@ -17,8 +17,17 @@ package org.apache.dolphinscheduler.api.controller; -import io.swagger.annotations.*; -import org.apache.commons.lang3.StringUtils; +import static org.apache.dolphinscheduler.api.enums.Status.BATCH_DELETE_PROCESS_INSTANCE_BY_IDS_ERROR; +import static org.apache.dolphinscheduler.api.enums.Status.DELETE_PROCESS_INSTANCE_BY_ID_ERROR; +import static org.apache.dolphinscheduler.api.enums.Status.ENCAPSULATION_PROCESS_INSTANCE_GANTT_STRUCTURE_ERROR; +import static org.apache.dolphinscheduler.api.enums.Status.QUERY_PARENT_PROCESS_INSTANCE_DETAIL_INFO_BY_SUB_PROCESS_INSTANCE_ID_ERROR; +import static org.apache.dolphinscheduler.api.enums.Status.QUERY_PROCESS_INSTANCE_ALL_VARIABLES_ERROR; +import static org.apache.dolphinscheduler.api.enums.Status.QUERY_PROCESS_INSTANCE_BY_ID_ERROR; +import static org.apache.dolphinscheduler.api.enums.Status.QUERY_PROCESS_INSTANCE_LIST_PAGING_ERROR; +import static org.apache.dolphinscheduler.api.enums.Status.QUERY_SUB_PROCESS_INSTANCE_DETAIL_INFO_BY_TASK_ID_ERROR; +import static org.apache.dolphinscheduler.api.enums.Status.QUERY_TASK_LIST_BY_PROCESS_INSTANCE_ID_ERROR; +import static org.apache.dolphinscheduler.api.enums.Status.UPDATE_PROCESS_INSTANCE_ERROR; + import org.apache.dolphinscheduler.api.aspect.AccessLogAnnotation; import org.apache.dolphinscheduler.api.enums.Status; import org.apache.dolphinscheduler.api.exceptions.ApiException; @@ -26,14 +35,10 @@ import org.apache.dolphinscheduler.api.utils.Result; import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.enums.WorkflowExecutionStatus; -import org.apache.dolphinscheduler.plugin.task.api.utils.ParameterUtils; import org.apache.dolphinscheduler.dao.entity.ProcessInstance; import org.apache.dolphinscheduler.dao.entity.User; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.http.HttpStatus; -import org.springframework.web.bind.annotation.*; +import org.apache.dolphinscheduler.plugin.task.api.utils.ParameterUtils; + import springfox.documentation.annotations.ApiIgnore; import org.apache.commons.lang3.StringUtils; @@ -45,7 +50,26 @@ import java.util.List; import java.util.Map; -import static org.apache.dolphinscheduler.api.enums.Status.*; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.http.HttpStatus; +import org.springframework.web.bind.annotation.DeleteMapping; +import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.bind.annotation.PathVariable; +import org.springframework.web.bind.annotation.PostMapping; +import org.springframework.web.bind.annotation.PutMapping; +import org.springframework.web.bind.annotation.RequestAttribute; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RequestParam; +import org.springframework.web.bind.annotation.ResponseStatus; +import org.springframework.web.bind.annotation.RestController; + +import io.swagger.annotations.Api; +import io.swagger.annotations.ApiImplicitParam; +import io.swagger.annotations.ApiImplicitParams; +import io.swagger.annotations.ApiOperation; +import io.swagger.annotations.ApiParam; /** * process instance controller diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/ProjectController.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/ProjectController.java index 76997f75dc9a..14c2cde3ee61 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/ProjectController.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/ProjectController.java @@ -32,8 +32,8 @@ import org.apache.dolphinscheduler.api.service.ProjectService; import org.apache.dolphinscheduler.api.utils.Result; import org.apache.dolphinscheduler.common.Constants; -import org.apache.dolphinscheduler.plugin.task.api.utils.ParameterUtils; import org.apache.dolphinscheduler.dao.entity.User; +import org.apache.dolphinscheduler.plugin.task.api.utils.ParameterUtils; import springfox.documentation.annotations.ApiIgnore; diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/ProjectV2Controller.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/ProjectV2Controller.java index 1dd6324a4980..b167f9b47e96 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/ProjectV2Controller.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/ProjectV2Controller.java @@ -42,8 +42,8 @@ import org.apache.dolphinscheduler.api.service.ProjectService; import org.apache.dolphinscheduler.api.utils.Result; import org.apache.dolphinscheduler.common.Constants; -import org.apache.dolphinscheduler.plugin.task.api.utils.ParameterUtils; import org.apache.dolphinscheduler.dao.entity.User; +import org.apache.dolphinscheduler.plugin.task.api.utils.ParameterUtils; import springfox.documentation.annotations.ApiIgnore; diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/QueueController.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/QueueController.java index 32734e811e07..d71760be1103 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/QueueController.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/QueueController.java @@ -27,8 +27,8 @@ import org.apache.dolphinscheduler.api.service.QueueService; import org.apache.dolphinscheduler.api.utils.Result; import org.apache.dolphinscheduler.common.Constants; -import org.apache.dolphinscheduler.plugin.task.api.utils.ParameterUtils; import org.apache.dolphinscheduler.dao.entity.User; +import org.apache.dolphinscheduler.plugin.task.api.utils.ParameterUtils; import springfox.documentation.annotations.ApiIgnore; diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/QueueV2Controller.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/QueueV2Controller.java index d00f63cb20af..ec87292fbaf2 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/QueueV2Controller.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/QueueV2Controller.java @@ -36,8 +36,10 @@ import org.apache.dolphinscheduler.api.service.QueueService; import org.apache.dolphinscheduler.api.utils.Result; import org.apache.dolphinscheduler.common.Constants; -import org.apache.dolphinscheduler.plugin.task.api.utils.ParameterUtils; import org.apache.dolphinscheduler.dao.entity.User; +import org.apache.dolphinscheduler.plugin.task.api.utils.ParameterUtils; + +import springfox.documentation.annotations.ApiIgnore; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.http.HttpStatus; @@ -55,7 +57,6 @@ import io.swagger.annotations.ApiImplicitParam; import io.swagger.annotations.ApiImplicitParams; import io.swagger.annotations.ApiOperation; -import springfox.documentation.annotations.ApiIgnore; /** * queue controller @@ -93,9 +94,9 @@ public QueueListResponse queryList(@ApiIgnore @RequestAttribute(value = Constant */ @ApiOperation(value = "queryQueueListPaging", notes = "QUERY_QUEUE_LIST_PAGING_NOTES") @ApiImplicitParams({ - @ApiImplicitParam(name = "searchVal", value = "SEARCH_VAL", dataTypeClass = String.class), - @ApiImplicitParam(name = "pageNo", value = "PAGE_NO", required = true, dataTypeClass = int.class, example = "1"), - @ApiImplicitParam(name = "pageSize", value = "PAGE_SIZE", required = true, dataTypeClass = int.class, example = "20") + @ApiImplicitParam(name = "searchVal", value = "SEARCH_VAL", dataTypeClass = String.class), + @ApiImplicitParam(name = "pageNo", value = "PAGE_NO", required = true, dataTypeClass = int.class, example = "1"), + @ApiImplicitParam(name = "pageSize", value = "PAGE_SIZE", required = true, dataTypeClass = int.class, example = "20") }) @GetMapping() @ResponseStatus(HttpStatus.OK) @@ -109,7 +110,8 @@ public QueueListPagingResponse queryQueueListPaging(@ApiIgnore @RequestAttribute } String searchVal = ParameterUtils.handleEscapes(queueQueryRequest.getSearchVal()); - result = queueService.queryList(loginUser, searchVal, queueQueryRequest.getPageNo(), queueQueryRequest.getPageSize()); + result = queueService.queryList(loginUser, searchVal, queueQueryRequest.getPageNo(), + queueQueryRequest.getPageSize()); return new QueueListPagingResponse(result); } @@ -127,7 +129,8 @@ public QueueListPagingResponse queryQueueListPaging(@ApiIgnore @RequestAttribute @AccessLogAnnotation(ignoreRequestArgs = "loginUser") public QueueCreateResponse createQueue(@ApiIgnore @RequestAttribute(value = Constants.SESSION_USER) User loginUser, @RequestBody QueueCreateRequest queueCreateRequest) { - Result result = queueService.createQueue(loginUser, queueCreateRequest.getQueue(), queueCreateRequest.getQueueName()); + Result result = + queueService.createQueue(loginUser, queueCreateRequest.getQueue(), queueCreateRequest.getQueueName()); return new QueueCreateResponse(result); } @@ -141,16 +144,17 @@ public QueueCreateResponse createQueue(@ApiIgnore @RequestAttribute(value = Cons */ @ApiOperation(value = "updateQueue", notes = "UPDATE_QUEUE_NOTES") @ApiImplicitParams({ - @ApiImplicitParam(name = "id", value = "QUEUE_ID", required = true, dataTypeClass = int.class, example = "100") + @ApiImplicitParam(name = "id", value = "QUEUE_ID", required = true, dataTypeClass = int.class, example = "100") }) @PutMapping(value = "/{id}", consumes = {"application/json"}) @ResponseStatus(HttpStatus.CREATED) @ApiException(UPDATE_QUEUE_ERROR) @AccessLogAnnotation(ignoreRequestArgs = "loginUser") public QueueUpdateResponse updateQueue(@ApiIgnore @RequestAttribute(value = Constants.SESSION_USER) User loginUser, - @PathVariable(value = "id") int id, @RequestBody QueueUpdateRequest queueUpdateRequest) { + @PathVariable(value = "id") int id, + @RequestBody QueueUpdateRequest queueUpdateRequest) { Result result = queueService.updateQueue(loginUser, id, queueUpdateRequest.getQueue(), - queueUpdateRequest.getQueueName()); + queueUpdateRequest.getQueueName()); return new QueueUpdateResponse(result); } diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/ResourcesController.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/ResourcesController.java index ea2c59c41875..55ca04a1a0f7 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/ResourcesController.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/ResourcesController.java @@ -49,8 +49,8 @@ import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.enums.ProgramType; import org.apache.dolphinscheduler.common.enums.UdfType; -import org.apache.dolphinscheduler.plugin.task.api.utils.ParameterUtils; import org.apache.dolphinscheduler.dao.entity.User; +import org.apache.dolphinscheduler.plugin.task.api.utils.ParameterUtils; import org.apache.dolphinscheduler.spi.enums.ResourceType; import springfox.documentation.annotations.ApiIgnore; diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/SchedulerController.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/SchedulerController.java index 2f87a1bf0059..45a815262d57 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/SchedulerController.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/SchedulerController.java @@ -38,6 +38,7 @@ import org.apache.dolphinscheduler.common.enums.WarningType; import org.apache.dolphinscheduler.dao.entity.User; import org.apache.dolphinscheduler.plugin.task.api.utils.ParameterUtils; + import springfox.documentation.annotations.ApiIgnore; import java.util.Map; diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/TaskDefinitionController.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/TaskDefinitionController.java index 301fe085fb0c..cf96586084e2 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/TaskDefinitionController.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/TaskDefinitionController.java @@ -36,8 +36,8 @@ import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.enums.ReleaseState; import org.apache.dolphinscheduler.common.enums.TaskExecuteType; -import org.apache.dolphinscheduler.plugin.task.api.utils.ParameterUtils; import org.apache.dolphinscheduler.dao.entity.User; +import org.apache.dolphinscheduler.plugin.task.api.utils.ParameterUtils; import springfox.documentation.annotations.ApiIgnore; diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/TaskInstanceController.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/TaskInstanceController.java index 9b7b300d4fc7..78ab34762dc5 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/TaskInstanceController.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/TaskInstanceController.java @@ -28,9 +28,9 @@ import org.apache.dolphinscheduler.api.utils.Result; import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.enums.TaskExecuteType; -import org.apache.dolphinscheduler.plugin.task.api.utils.ParameterUtils; import org.apache.dolphinscheduler.dao.entity.User; import org.apache.dolphinscheduler.plugin.task.api.enums.TaskExecutionStatus; +import org.apache.dolphinscheduler.plugin.task.api.utils.ParameterUtils; import springfox.documentation.annotations.ApiIgnore; diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/TenantController.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/TenantController.java index 8c99eb8cfd47..c0724b6d3471 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/TenantController.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/TenantController.java @@ -29,8 +29,8 @@ import org.apache.dolphinscheduler.api.service.TenantService; import org.apache.dolphinscheduler.api.utils.Result; import org.apache.dolphinscheduler.common.Constants; -import org.apache.dolphinscheduler.plugin.task.api.utils.ParameterUtils; import org.apache.dolphinscheduler.dao.entity.User; +import org.apache.dolphinscheduler.plugin.task.api.utils.ParameterUtils; import springfox.documentation.annotations.ApiIgnore; diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/UsersController.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/UsersController.java index 1e427b0e2345..44a03483d5f0 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/UsersController.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/UsersController.java @@ -39,8 +39,8 @@ import org.apache.dolphinscheduler.api.service.UsersService; import org.apache.dolphinscheduler.api.utils.Result; import org.apache.dolphinscheduler.common.Constants; -import org.apache.dolphinscheduler.plugin.task.api.utils.ParameterUtils; import org.apache.dolphinscheduler.dao.entity.User; +import org.apache.dolphinscheduler.plugin.task.api.utils.ParameterUtils; import springfox.documentation.annotations.ApiIgnore; diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/WorkFlowLineageController.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/WorkFlowLineageController.java index f887f064b7d7..b22eda9eaea5 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/WorkFlowLineageController.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/WorkFlowLineageController.java @@ -28,9 +28,9 @@ import org.apache.dolphinscheduler.api.service.WorkFlowLineageService; import org.apache.dolphinscheduler.api.utils.Result; import org.apache.dolphinscheduler.common.Constants; -import org.apache.dolphinscheduler.plugin.task.api.utils.ParameterUtils; import org.apache.dolphinscheduler.dao.entity.User; import org.apache.dolphinscheduler.dao.entity.WorkFlowLineage; +import org.apache.dolphinscheduler.plugin.task.api.utils.ParameterUtils; import springfox.documentation.annotations.ApiIgnore; diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/WorkerGroupController.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/WorkerGroupController.java index 7314a38eed88..58681aed857e 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/WorkerGroupController.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/WorkerGroupController.java @@ -27,8 +27,8 @@ import org.apache.dolphinscheduler.api.service.WorkerGroupService; import org.apache.dolphinscheduler.api.utils.Result; import org.apache.dolphinscheduler.common.Constants; -import org.apache.dolphinscheduler.plugin.task.api.utils.ParameterUtils; import org.apache.dolphinscheduler.dao.entity.User; +import org.apache.dolphinscheduler.plugin.task.api.utils.ParameterUtils; import springfox.documentation.annotations.ApiIgnore; @@ -73,11 +73,11 @@ public class WorkerGroupController extends BaseController { */ @ApiOperation(value = "saveWorkerGroup", notes = "CREATE_WORKER_GROUP_NOTES") @ApiImplicitParams({ - @ApiImplicitParam(name = "id", value = "WORKER_GROUP_ID", dataType = "Int", example = "10", defaultValue = "0"), - @ApiImplicitParam(name = "name", value = "WORKER_GROUP_NAME", required = true, dataType = "String"), - @ApiImplicitParam(name = "addrList", value = "WORKER_ADDR_LIST", required = true, dataType = "String"), - @ApiImplicitParam(name = "description", value = "WORKER_DESC", required = false, dataType = "String"), - @ApiImplicitParam(name = "otherParamsJson", value = "WORKER_PARMS_JSON", required = false, dataType = "String"), + @ApiImplicitParam(name = "id", value = "WORKER_GROUP_ID", dataType = "Int", example = "10", defaultValue = "0"), + @ApiImplicitParam(name = "name", value = "WORKER_GROUP_NAME", required = true, dataType = "String"), + @ApiImplicitParam(name = "addrList", value = "WORKER_ADDR_LIST", required = true, dataType = "String"), + @ApiImplicitParam(name = "description", value = "WORKER_DESC", required = false, dataType = "String"), + @ApiImplicitParam(name = "otherParamsJson", value = "WORKER_PARMS_JSON", required = false, dataType = "String"), }) @PostMapping() @ResponseStatus(HttpStatus.OK) @@ -87,10 +87,10 @@ public Result saveWorkerGroup(@ApiIgnore @RequestAttribute(value = Constants.SES @RequestParam(value = "id", required = false, defaultValue = "0") int id, @RequestParam(value = "name") String name, @RequestParam(value = "addrList") String addrList, - @RequestParam(value = "description",required = false, defaultValue = "") String description, - @RequestParam(value = "otherParamsJson",required = false, defaultValue = "") String otherParamsJson - ) { - Map result = workerGroupService.saveWorkerGroup(loginUser, id, name, addrList, description, otherParamsJson); + @RequestParam(value = "description", required = false, defaultValue = "") String description, + @RequestParam(value = "otherParamsJson", required = false, defaultValue = "") String otherParamsJson) { + Map result = + workerGroupService.saveWorkerGroup(loginUser, id, name, addrList, description, otherParamsJson); return returnDataList(result); } diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/CommandStateCount.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/CommandStateCount.java index 5e6a51817350..b1b0eb66c964 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/CommandStateCount.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/CommandStateCount.java @@ -28,7 +28,8 @@ public class CommandStateCount { private int normalCount; private CommandType commandState; - public CommandStateCount(){} + public CommandStateCount() { + } public CommandStateCount(int errorCount, int normalCount, CommandType commandState) { this.errorCount = errorCount; diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/CreateTokenRequest.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/CreateTokenRequest.java index 03190f4b9ffb..c0b8520e2eb7 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/CreateTokenRequest.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/CreateTokenRequest.java @@ -17,10 +17,10 @@ package org.apache.dolphinscheduler.api.dto; - import io.swagger.annotations.ApiModelProperty; public class CreateTokenRequest { + @ApiModelProperty(example = "1", required = true) Integer userId; diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/CreateTokenResponse.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/CreateTokenResponse.java index e96e2b2690a0..d7fa93c40cc4 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/CreateTokenResponse.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/CreateTokenResponse.java @@ -21,6 +21,7 @@ import org.apache.dolphinscheduler.dao.entity.AccessToken; public class CreateTokenResponse extends Result { + private AccessToken data; public CreateTokenResponse(Result result) { diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/RuleDefinition.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/RuleDefinition.java index d702e88b48bf..2ef0f11cbd10 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/RuleDefinition.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/RuleDefinition.java @@ -40,7 +40,7 @@ public class RuleDefinition { public RuleDefinition() { } - public RuleDefinition(List ruleInputEntryList,List executeSqlList) { + public RuleDefinition(List ruleInputEntryList, List executeSqlList) { this.ruleInputEntryList = ruleInputEntryList; this.executeSqlList = executeSqlList; } diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/TaskCountDto.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/TaskCountDto.java index 66a42857a4c3..9e456109bede 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/TaskCountDto.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/TaskCountDto.java @@ -17,9 +17,6 @@ package org.apache.dolphinscheduler.api.dto; -import lombok.AllArgsConstructor; -import lombok.Data; -import lombok.NoArgsConstructor; import org.apache.dolphinscheduler.dao.entity.ExecuteStatusCount; import org.apache.dolphinscheduler.plugin.task.api.enums.TaskExecutionStatus; @@ -28,10 +25,15 @@ import java.util.Map; import java.util.stream.Collectors; +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; + @Data @NoArgsConstructor @AllArgsConstructor public class TaskCountDto { + /** * total count */ diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/TaskStateCount.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/TaskStateCount.java index 95dc18a49f15..eb935659c91e 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/TaskStateCount.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/TaskStateCount.java @@ -17,10 +17,11 @@ package org.apache.dolphinscheduler.api.dto; +import org.apache.dolphinscheduler.plugin.task.api.enums.TaskExecutionStatus; + import lombok.AllArgsConstructor; import lombok.Data; import lombok.NoArgsConstructor; -import org.apache.dolphinscheduler.plugin.task.api.enums.TaskExecutionStatus; @Data @NoArgsConstructor diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/gantt/GanttDto.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/gantt/GanttDto.java index c32454a7847b..401e3f6319cf 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/gantt/GanttDto.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/gantt/GanttDto.java @@ -44,14 +44,13 @@ public class GanttDto { /** * task status map */ - private Map taskStatus; + private Map taskStatus; - - public GanttDto(){ + public GanttDto() { this.taskStatus = new HashMap<>(); - taskStatus.put("success","success"); + taskStatus.put("success", "success"); } - public GanttDto(int height, List tasks, List taskNames){ + public GanttDto(int height, List tasks, List taskNames) { this(); this.height = height; this.tasks = tasks; diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/gantt/Task.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/gantt/Task.java index 07243db3fda2..5370f61e8a4d 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/gantt/Task.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/gantt/Task.java @@ -25,6 +25,7 @@ * Task */ public class Task { + /** * task name */ diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/project/ProjectCreateRequest.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/project/ProjectCreateRequest.java index aca6556db497..21ad4d75be42 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/project/ProjectCreateRequest.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/project/ProjectCreateRequest.java @@ -17,8 +17,8 @@ package org.apache.dolphinscheduler.api.dto.project; -import io.swagger.annotations.ApiModelProperty; import lombok.Data; +import io.swagger.annotations.ApiModelProperty; /** * project create request diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/project/ProjectDeleteResponse.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/project/ProjectDeleteResponse.java index a7ee4bd97bb1..b174adda8459 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/project/ProjectDeleteResponse.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/project/ProjectDeleteResponse.java @@ -35,4 +35,4 @@ public ProjectDeleteResponse(Result result) { this.setMsg(result.getMsg()); this.setData((Boolean) result.getData()); } -} \ No newline at end of file +} diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/project/ProjectQueryRequest.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/project/ProjectQueryRequest.java index 2512650a7303..942be6857d9e 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/project/ProjectQueryRequest.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/project/ProjectQueryRequest.java @@ -19,12 +19,13 @@ import org.apache.dolphinscheduler.api.dto.PageQueryDto; +import lombok.Data; + import com.fasterxml.jackson.annotation.JsonIgnoreProperties; import com.fasterxml.jackson.annotation.JsonInclude; import io.swagger.annotations.ApiModel; import io.swagger.annotations.ApiModelProperty; -import lombok.Data; /** * project query request diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/project/ProjectUpdateRequest.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/project/ProjectUpdateRequest.java index 32e36ac95a07..69cb3b563c52 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/project/ProjectUpdateRequest.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/project/ProjectUpdateRequest.java @@ -17,11 +17,11 @@ package org.apache.dolphinscheduler.api.dto.project; +import lombok.Data; + import com.fasterxml.jackson.annotation.JsonIgnoreProperties; import com.fasterxml.jackson.annotation.JsonInclude; - import io.swagger.annotations.ApiModelProperty; -import lombok.Data; /** * project update request diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/queue/QueueCreateRequest.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/queue/QueueCreateRequest.java index 702826fe3237..c318200f5acf 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/queue/QueueCreateRequest.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/queue/QueueCreateRequest.java @@ -17,9 +17,9 @@ package org.apache.dolphinscheduler.api.dto.queue; +import lombok.Data; import io.swagger.annotations.ApiModel; import io.swagger.annotations.ApiModelProperty; -import lombok.Data; /** * queue create request diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/queue/QueueQueryRequest.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/queue/QueueQueryRequest.java index 727d8b12c6a4..3bdd2293cfbf 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/queue/QueueQueryRequest.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/queue/QueueQueryRequest.java @@ -19,12 +19,13 @@ import org.apache.dolphinscheduler.api.dto.PageQueryDto; +import lombok.Data; + import com.fasterxml.jackson.annotation.JsonIgnoreProperties; import com.fasterxml.jackson.annotation.JsonInclude; import io.swagger.annotations.ApiModel; import io.swagger.annotations.ApiModelProperty; -import lombok.Data; /** * queue query request diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/queue/QueueUpdateRequest.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/queue/QueueUpdateRequest.java index fd04318a033f..7d2bbdc42f35 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/queue/QueueUpdateRequest.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/queue/QueueUpdateRequest.java @@ -17,12 +17,13 @@ package org.apache.dolphinscheduler.api.dto.queue; +import lombok.Data; + import com.fasterxml.jackson.annotation.JsonIgnoreProperties; import com.fasterxml.jackson.annotation.JsonInclude; import io.swagger.annotations.ApiModel; import io.swagger.annotations.ApiModelProperty; -import lombok.Data; /** * queue update request diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/queue/QueueVerifyRequest.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/queue/QueueVerifyRequest.java index 88e53598438b..6b07a7bb7dd2 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/queue/QueueVerifyRequest.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/queue/QueueVerifyRequest.java @@ -17,9 +17,9 @@ package org.apache.dolphinscheduler.api.dto.queue; +import lombok.Data; import io.swagger.annotations.ApiModel; import io.swagger.annotations.ApiModelProperty; -import lombok.Data; /** * queue verify request diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/resources/FileLeaf.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/resources/FileLeaf.java index 92aaf1332074..bee5bc859079 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/resources/FileLeaf.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/resources/FileLeaf.java @@ -19,6 +19,6 @@ /** * file leaf */ -public class FileLeaf extends ResourceComponent{ +public class FileLeaf extends ResourceComponent { } diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/resources/filter/IFilter.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/resources/filter/IFilter.java index ce6ce3a011ac..2bcc2822a866 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/resources/filter/IFilter.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/resources/filter/IFilter.java @@ -24,5 +24,6 @@ * interface filter */ public interface IFilter { + List filter(); } diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/resources/filter/ResourceFilter.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/resources/filter/ResourceFilter.java index bf5a49597c83..6206769f7f18 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/resources/filter/ResourceFilter.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/resources/filter/ResourceFilter.java @@ -28,6 +28,7 @@ * resource filter */ public class ResourceFilter implements IFilter { + /** * resource suffix */ @@ -40,7 +41,7 @@ public class ResourceFilter implements IFilter { /** * parent list */ - //Set parentList = new HashSet<>(); + // Set parentList = new HashSet<>(); /** * constructor @@ -56,7 +57,7 @@ public ResourceFilter(String suffix, List resourceList) { * file filter * @return file filtered by suffix */ - public Set fileFilter(){ + public Set fileFilter() { return resourceList.stream().filter(t -> { String alias = t.getAlias(); return alias.endsWith(suffix); @@ -67,12 +68,12 @@ public Set fileFilter(){ * list all parent dir * @return parent resource dir set */ - Set listAllParent(){ - Set parentList = new HashSet<>(); + Set listAllParent() { + Set parentList = new HashSet<>(); Set filterFileList = fileFilter(); - for(Resource file:filterFileList){ + for (Resource file : filterFileList) { parentList.add(file); - setAllParent(file,parentList); + setAllParent(file, parentList); } return parentList; @@ -83,11 +84,11 @@ Set listAllParent(){ * @param resource resource * @return parent resource dir set */ - private void setAllParent(Resource resource,Set parentList){ + private void setAllParent(Resource resource, Set parentList) { for (Resource resourceTemp : resourceList) { if (resourceTemp.getId() == resource.getPid()) { parentList.add(resourceTemp); - setAllParent(resourceTemp,parentList); + setAllParent(resourceTemp, parentList); } } } diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/resources/visitor/ResourceTreeVisitor.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/resources/visitor/ResourceTreeVisitor.java index 391822e53505..5b84fde1a567 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/resources/visitor/ResourceTreeVisitor.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/resources/visitor/ResourceTreeVisitor.java @@ -16,7 +16,6 @@ */ package org.apache.dolphinscheduler.api.dto.resources.visitor; - import org.apache.dolphinscheduler.api.dto.resources.Directory; import org.apache.dolphinscheduler.api.dto.resources.FileLeaf; import org.apache.dolphinscheduler.api.dto.resources.ResourceComponent; @@ -28,7 +27,7 @@ /** * resource tree visitor */ -public class ResourceTreeVisitor implements Visitor{ +public class ResourceTreeVisitor implements Visitor { /** * resource list @@ -55,10 +54,10 @@ public ResourceComponent visit() { ResourceComponent rootDirectory = new Directory(); for (Resource resource : resourceList) { // judge whether is root node - if (rootNode(resource)){ + if (rootNode(resource)) { ResourceComponent tempResourceComponent = getResourceComponent(resource); rootDirectory.add(tempResourceComponent); - tempResourceComponent.setChildren(setChildren(tempResourceComponent.getId(),resourceList)); + tempResourceComponent.setChildren(setChildren(tempResourceComponent.getId(), resourceList)); } } return rootDirectory; @@ -70,18 +69,18 @@ public ResourceComponent visit() { * @param list resource list * @return resource component list */ - public static List setChildren(int id, List list ){ + public static List setChildren(int id, List list) { List childList = new ArrayList<>(); for (Resource resource : list) { - if (id == resource.getPid()){ + if (id == resource.getPid()) { ResourceComponent tempResourceComponent = getResourceComponent(resource); childList.add(tempResourceComponent); } } for (ResourceComponent resourceComponent : childList) { - resourceComponent.setChildren(setChildren(resourceComponent.getId(),list)); + resourceComponent.setChildren(setChildren(resourceComponent.getId(), list)); } - if (childList.size()==0){ + if (childList.size() == 0) { return new ArrayList<>(); } return childList; @@ -95,7 +94,7 @@ public static List setChildren(int id, List list ){ public boolean rootNode(Resource resource) { boolean isRootNode = true; - if(resource.getPid() != -1 ){ + if (resource.getPid() != -1) { for (Resource parent : resourceList) { if (resource.getPid() == parent.getId()) { isRootNode = false; @@ -113,17 +112,17 @@ public boolean rootNode(Resource resource) { */ private static ResourceComponent getResourceComponent(Resource resource) { ResourceComponent tempResourceComponent; - if(resource.isDirectory()){ + if (resource.isDirectory()) { tempResourceComponent = new Directory(); - }else{ + } else { tempResourceComponent = new FileLeaf(); } - + tempResourceComponent.setName(resource.getAlias()); - tempResourceComponent.setFullName(resource.getFullName().replaceFirst("/","")); + tempResourceComponent.setFullName(resource.getFullName().replaceFirst("/", "")); tempResourceComponent.setId(resource.getId()); tempResourceComponent.setPid(resource.getPid()); - tempResourceComponent.setIdValue(resource.getId(),resource.isDirectory()); + tempResourceComponent.setIdValue(resource.getId(), resource.isDirectory()); tempResourceComponent.setDescription(resource.getDescription()); tempResourceComponent.setType(resource.getType()); return tempResourceComponent; diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/resources/visitor/Visitor.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/resources/visitor/Visitor.java index defbed13bb95..382bcc5e16bc 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/resources/visitor/Visitor.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/resources/visitor/Visitor.java @@ -23,6 +23,7 @@ * Visitor */ public interface Visitor { + /** * visit * @return resource component diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/schedule/ScheduleCreateRequest.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/schedule/ScheduleCreateRequest.java index ff42d75845a6..f324c23c5a3e 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/schedule/ScheduleCreateRequest.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/schedule/ScheduleCreateRequest.java @@ -33,9 +33,6 @@ import com.google.gson.GsonBuilder; import io.swagger.annotations.ApiModelProperty; -import com.google.gson.Gson; -import com.google.gson.GsonBuilder; - /** * schedule create request */ diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/enums/ExecuteType.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/enums/ExecuteType.java index bfac40458228..755daa1dd107 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/enums/ExecuteType.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/enums/ExecuteType.java @@ -33,7 +33,7 @@ public enum ExecuteType { NONE, REPEAT_RUNNING, RECOVER_SUSPENDED_PROCESS, START_FAILURE_TASK_PROCESS, STOP, PAUSE; public static ExecuteType getEnum(int value) { - for (ExecuteType e: ExecuteType.values()) { + for (ExecuteType e : ExecuteType.values()) { if (e.ordinal() == value) { return e; } diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/exceptions/ApiException.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/exceptions/ApiException.java index 8cdf2b81f869..b4812a0307d4 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/exceptions/ApiException.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/exceptions/ApiException.java @@ -31,5 +31,6 @@ @Retention(RUNTIME) @Target(METHOD) public @interface ApiException { + Status value(); } diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/exceptions/ApiExceptionHandler.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/exceptions/ApiExceptionHandler.java index e1d5a4713803..863ad79a06af 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/exceptions/ApiExceptionHandler.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/exceptions/ApiExceptionHandler.java @@ -55,4 +55,3 @@ public Result exceptionHandler(Exception e, HandlerMethod hm) { } } - diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/interceptor/RateLimitInterceptor.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/interceptor/RateLimitInterceptor.java index 20041b64d47f..19f121fff24e 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/interceptor/RateLimitInterceptor.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/interceptor/RateLimitInterceptor.java @@ -56,13 +56,15 @@ public class RateLimitInterceptor implements HandlerInterceptor { .maximumSize(100) .expireAfterAccess(10, TimeUnit.MINUTES) .build(new CacheLoader() { + @Override public RateLimiter load(String token) { // use tenant customize rate limit Map customizeTenantQpsRate = trafficConfiguration.getCustomizeTenantQpsRate(); int tenantQuota = trafficConfiguration.getDefaultTenantQpsRate(); if (MapUtils.isNotEmpty(customizeTenantQpsRate)) { - tenantQuota = customizeTenantQpsRate.getOrDefault(token, trafficConfiguration.getDefaultTenantQpsRate()); + tenantQuota = customizeTenantQpsRate.getOrDefault(token, + trafficConfiguration.getDefaultTenantQpsRate()); } // use tenant default rate limit return RateLimiter.create(tenantQuota, 1, TimeUnit.SECONDS); @@ -70,7 +72,8 @@ public RateLimiter load(String token) { }); @Override - public boolean preHandle(HttpServletRequest request, HttpServletResponse response, Object handler) throws ExecutionException { + public boolean preHandle(HttpServletRequest request, HttpServletResponse response, + Object handler) throws ExecutionException { // tenant-level rate limit if (trafficConfiguration.isTenantSwitch()) { String token = request.getHeader("token"); @@ -78,7 +81,8 @@ public boolean preHandle(HttpServletRequest request, HttpServletResponse respons RateLimiter tenantRateLimiter = tenantRateLimiterCache.get(token); if (!tenantRateLimiter.tryAcquire()) { response.setStatus(HttpStatus.TOO_MANY_REQUESTS.value()); - logger.warn("Too many request, reach tenant rate limit, current tenant:{} qps is {}", token, tenantRateLimiter.getRate()); + logger.warn("Too many request, reach tenant rate limit, current tenant:{} qps is {}", token, + tenantRateLimiter.getRate()); return false; } } @@ -87,7 +91,8 @@ public boolean preHandle(HttpServletRequest request, HttpServletResponse respons if (trafficConfiguration.isGlobalSwitch()) { if (!globalRateLimiter.tryAcquire()) { response.setStatus(HttpStatus.TOO_MANY_REQUESTS.value()); - logger.warn("Too many request, reach global rate limit, current qps is {}", globalRateLimiter.getRate()); + logger.warn("Too many request, reach global rate limit, current qps is {}", + globalRateLimiter.getRate()); return false; } } @@ -97,7 +102,8 @@ public boolean preHandle(HttpServletRequest request, HttpServletResponse respons public RateLimitInterceptor(TrafficConfiguration trafficConfiguration) { this.trafficConfiguration = trafficConfiguration; if (trafficConfiguration.isGlobalSwitch()) { - this.globalRateLimiter = RateLimiter.create(trafficConfiguration.getMaxGlobalQpsRate(), 1, TimeUnit.SECONDS); + this.globalRateLimiter = + RateLimiter.create(trafficConfiguration.getMaxGlobalQpsRate(), 1, TimeUnit.SECONDS); } } diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/k8s/K8sClientService.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/k8s/K8sClientService.java index dcfc6e734aaa..393f440ee47e 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/k8s/K8sClientService.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/k8s/K8sClientService.java @@ -42,7 +42,8 @@ public class K8sClientService { @Autowired private K8sManager k8sManager; - public ResourceQuota upsertNamespaceAndResourceToK8s(K8sNamespace k8sNamespace, String yamlStr) throws RemotingException { + public ResourceQuota upsertNamespaceAndResourceToK8s(K8sNamespace k8sNamespace, + String yamlStr) throws RemotingException { upsertNamespaceToK8s(k8sNamespace.getNamespace(), k8sNamespace.getClusterCode()); return upsertNamespacedResourceToK8s(k8sNamespace, yamlStr); } @@ -61,48 +62,48 @@ public Optional deleteNamespaceToK8s(String name, Long clusterCode) t return getNamespaceFromK8s(name, clusterCode); } - private ResourceQuota upsertNamespacedResourceToK8s(K8sNamespace k8sNamespace, String yamlStr) throws RemotingException { + private ResourceQuota upsertNamespacedResourceToK8s(K8sNamespace k8sNamespace, + String yamlStr) throws RemotingException { KubernetesClient client = k8sManager.getK8sClient(k8sNamespace.getClusterCode()); - //创建资源 + // 创建资源 ResourceQuota queryExist = client.resourceQuotas() - .inNamespace(k8sNamespace.getNamespace()) - .withName(k8sNamespace.getNamespace()) - .get(); + .inNamespace(k8sNamespace.getNamespace()) + .withName(k8sNamespace.getNamespace()) + .get(); ResourceQuota body = yaml.loadAs(yamlStr, ResourceQuota.class); if (queryExist != null) { if (k8sNamespace.getLimitsCpu() == null && k8sNamespace.getLimitsMemory() == null) { client.resourceQuotas().inNamespace(k8sNamespace.getNamespace()) - .withName(k8sNamespace.getNamespace()) - .delete(); + .withName(k8sNamespace.getNamespace()) + .delete(); return null; } } return client.resourceQuotas().inNamespace(k8sNamespace.getNamespace()) - .withName(k8sNamespace.getNamespace()) - .createOrReplace(body); + .withName(k8sNamespace.getNamespace()) + .createOrReplace(body); } private Optional getNamespaceFromK8s(String name, Long clusterCode) throws RemotingException { NamespaceList listNamespace = - k8sManager.getK8sClient(clusterCode).namespaces().list(); + k8sManager.getK8sClient(clusterCode).namespaces().list(); Optional list = - listNamespace.getItems().stream() - .filter((Namespace namespace) -> - namespace.getMetadata().getName().equals(name)) - .findFirst(); + listNamespace.getItems().stream() + .filter((Namespace namespace) -> namespace.getMetadata().getName().equals(name)) + .findFirst(); return list; } private Namespace upsertNamespaceToK8s(String name, Long clusterCode) throws RemotingException { Optional result = getNamespaceFromK8s(name, clusterCode); - //if not exist create + // if not exist create if (!result.isPresent()) { KubernetesClient client = k8sManager.getK8sClient(clusterCode); Namespace body = new Namespace(); @@ -115,4 +116,4 @@ private Namespace upsertNamespaceToK8s(String name, Long clusterCode) throws Rem return result.get(); } -} \ No newline at end of file +} diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/k8s/K8sManager.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/k8s/K8sManager.java index fd9a2493284f..422075cbd750 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/k8s/K8sManager.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/k8s/K8sManager.java @@ -67,7 +67,8 @@ public synchronized KubernetesClient getK8sClient(Long clusterCode) throws Remot * @param clusterCode * @return new client if need updated */ - public synchronized KubernetesClient getAndUpdateK8sClient(Long clusterCode, boolean update) throws RemotingException { + public synchronized KubernetesClient getAndUpdateK8sClient(Long clusterCode, + boolean update) throws RemotingException { if (null == clusterCode) { return null; } @@ -84,7 +85,6 @@ public synchronized KubernetesClient getAndUpdateK8sClient(Long clusterCode, boo return clientMap.get(clusterCode); } - private void deleteK8sClientInner(Long clusterCode) { if (clusterCode == null) { return; diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/permission/PermissionCheck.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/permission/PermissionCheck.java index 41d69e590ef9..3d2e42b81bc1 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/permission/PermissionCheck.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/permission/PermissionCheck.java @@ -31,6 +31,7 @@ import org.slf4j.Logger; public class PermissionCheck { + /** * logger */ @@ -74,7 +75,8 @@ public PermissionCheck(AuthorizationType authorizationType, ProcessService proce /** * permission check */ - public PermissionCheck(AuthorizationType authorizationType, ProcessService processService, T[] needChecks, int userId) { + public PermissionCheck(AuthorizationType authorizationType, ProcessService processService, T[] needChecks, + int userId) { this.authorizationType = authorizationType; this.processService = processService; this.needChecks = needChecks; @@ -84,7 +86,8 @@ public PermissionCheck(AuthorizationType authorizationType, ProcessService proce /** * permission check */ - public PermissionCheck(AuthorizationType authorizationType, ProcessService processService, T[] needChecks, int userId, Logger logger) { + public PermissionCheck(AuthorizationType authorizationType, ProcessService processService, T[] needChecks, + int userId, Logger logger) { this.authorizationType = authorizationType; this.processService = processService; this.needChecks = needChecks; @@ -95,7 +98,8 @@ public PermissionCheck(AuthorizationType authorizationType, ProcessService proce /** * permission check */ - public PermissionCheck(AuthorizationType authorizationType, ProcessService processService, List resourceList, int userId, Logger logger) { + public PermissionCheck(AuthorizationType authorizationType, ProcessService processService, + List resourceList, int userId, Logger logger) { this.authorizationType = authorizationType; this.processService = processService; this.resourceList = resourceList; diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/permission/ResourcePermissionCheckService.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/permission/ResourcePermissionCheckService.java index eea1f14f98ad..b8016c5926a7 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/permission/ResourcePermissionCheckService.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/permission/ResourcePermissionCheckService.java @@ -16,13 +16,13 @@ */ package org.apache.dolphinscheduler.api.permission; -import org.apache.dolphinscheduler.common.enums.AuthorizationType; -import org.slf4j.Logger; - import java.util.List; import java.util.Set; -public interface ResourcePermissionCheckService{ +import org.slf4j.Logger; + +public interface ResourcePermissionCheckService { + /** * resourcePermissionCheck * @param authorizationType @@ -51,7 +51,8 @@ public interface ResourcePermissionCheckService{ * @param logger * @return */ - boolean operationPermissionCheck(Object authorizationType, Object[] projectIds, Integer userId, String permissionKey, Logger logger); + boolean operationPermissionCheck(Object authorizationType, Object[] projectIds, Integer userId, + String permissionKey, Logger logger); /** * functionDisabled diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/permission/ResourcePermissionCheckServiceImpl.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/permission/ResourcePermissionCheckServiceImpl.java index 1ebd187369d2..49c35242b293 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/permission/ResourcePermissionCheckServiceImpl.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/permission/ResourcePermissionCheckServiceImpl.java @@ -35,7 +35,6 @@ import static java.util.stream.Collectors.toSet; -import org.apache.commons.collections.CollectionUtils; import org.apache.dolphinscheduler.common.enums.AuthorizationType; import org.apache.dolphinscheduler.common.enums.UserType; import org.apache.dolphinscheduler.dao.entity.AccessToken; @@ -52,9 +51,7 @@ import org.apache.dolphinscheduler.dao.mapper.AccessTokenMapper; import org.apache.dolphinscheduler.dao.mapper.AlertGroupMapper; import org.apache.dolphinscheduler.dao.mapper.AlertPluginInstanceMapper; -import org.apache.dolphinscheduler.dao.mapper.CommandMapper; import org.apache.dolphinscheduler.dao.mapper.DataSourceMapper; -import org.apache.dolphinscheduler.dao.mapper.DqRuleMapper; import org.apache.dolphinscheduler.dao.mapper.EnvironmentMapper; import org.apache.dolphinscheduler.dao.mapper.K8sNamespaceMapper; import org.apache.dolphinscheduler.dao.mapper.ProjectMapper; @@ -67,6 +64,8 @@ import org.apache.dolphinscheduler.dao.mapper.WorkerGroupMapper; import org.apache.dolphinscheduler.service.process.ProcessService; +import org.apache.commons.collections.CollectionUtils; + import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; @@ -86,23 +85,29 @@ import org.springframework.stereotype.Component; @Component -public class ResourcePermissionCheckServiceImpl implements ResourcePermissionCheckService, ApplicationContextAware { +public class ResourcePermissionCheckServiceImpl + implements + ResourcePermissionCheckService, + ApplicationContextAware { @Autowired private ProcessService processService; - public static final Map> RESOURCE_LIST_MAP = new ConcurrentHashMap<>(); + public static final Map> RESOURCE_LIST_MAP = + new ConcurrentHashMap<>(); @Override public void setApplicationContext(ApplicationContext applicationContext) throws BeansException { - for (ResourceAcquisitionAndPermissionCheck authorizedResourceList : applicationContext.getBeansOfType(ResourceAcquisitionAndPermissionCheck.class).values()) { + for (ResourceAcquisitionAndPermissionCheck authorizedResourceList : applicationContext + .getBeansOfType(ResourceAcquisitionAndPermissionCheck.class).values()) { List authorizationTypes = authorizedResourceList.authorizationTypes(); authorizationTypes.forEach(auth -> RESOURCE_LIST_MAP.put(auth, authorizedResourceList)); } - } + } @Override - public boolean resourcePermissionCheck(Object authorizationType, Object[] needChecks, Integer userId, Logger logger) { + public boolean resourcePermissionCheck(Object authorizationType, Object[] needChecks, Integer userId, + Logger logger) { if (Objects.nonNull(needChecks) && needChecks.length > 0) { Set originResSet = new HashSet<>(Arrays.asList(needChecks)); Set ownResSets = RESOURCE_LIST_MAP.get(authorizationType).listAuthorizedResource(userId, logger); @@ -115,7 +120,8 @@ public boolean resourcePermissionCheck(Object authorizationType, Object[] needCh } @Override - public boolean operationPermissionCheck(Object authorizationType, Object[] projectIds, Integer userId, String permissionKey, Logger logger) { + public boolean operationPermissionCheck(Object authorizationType, Object[] projectIds, Integer userId, + String permissionKey, Logger logger) { User user = processService.getUserById(userId); if (user == null) { logger.error("User does not exist, userId:{}.", userId); @@ -229,7 +235,8 @@ public Set listAuthorizedResource(int userId, Logger logger) { } else { // query resource relation List resIds = resourceUserMapper.queryResourcesIdListByUserIdAndPerm(userId, 0); - relationResources = CollectionUtils.isEmpty(resIds) ? new ArrayList<>() : resourceMapper.queryResourceListById(resIds); + relationResources = CollectionUtils.isEmpty(resIds) ? new ArrayList<>() + : resourceMapper.queryResourceListById(resIds); } List ownResourceList = resourceMapper.queryResourceListAuthored(userId, -1); relationResources.addAll(ownResourceList); @@ -259,7 +266,7 @@ public List authorizationTypes() { @Override public Set listAuthorizedResource(int userId, Logger logger) { List udfFuncList = udfFuncMapper.listAuthorizedUdfByUserId(userId); - if (udfFuncList.isEmpty()){ + if (udfFuncList.isEmpty()) { return Collections.emptySet(); } return udfFuncList.stream().map(UdfFunc::getId).collect(toSet()); @@ -325,7 +332,6 @@ public Set listAuthorizedResource(int userId, Logger logger) { } } - @Component public static class EnvironmentResourceList implements ResourceAcquisitionAndPermissionCheck { @@ -342,7 +348,7 @@ public List authorizationTypes() { @Override public boolean permissionCheck(int userId, String url, Logger logger) { - return true; + return true; } @Override @@ -374,7 +380,6 @@ public boolean permissionCheck(int userId, String url, Logger logger) { return false; } - @Override public Set listAuthorizedResource(int userId, Logger logger) { return Collections.emptySet(); @@ -403,7 +408,6 @@ public boolean permissionCheck(int userId, String url, Logger logger) { return false; } - @Override public Set listAuthorizedResource(int userId, Logger logger) { return Collections.emptySet(); @@ -429,10 +433,9 @@ public List authorizationTypes() { @Override public boolean permissionCheck(int userId, String url, Logger logger) { - return false; + return false; } - @Override public Set listAuthorizedResource(int userId, Logger logger) { List alertGroupList = alertGroupMapper.queryAllGroupList(); @@ -462,7 +465,6 @@ public boolean permissionCheck(int userId, String url, Logger logger) { return false; } - @Override public Set listAuthorizedResource(int userId, Logger logger) { if (userId != 0) { @@ -495,10 +497,10 @@ public boolean permissionCheck(int userId, String url, Logger logger) { return true; } - @Override public Set listAuthorizedResource(int userId, Logger logger) { - return dataSourceMapper.listAuthorizedDataSource(userId, null).stream().map(DataSource::getId).collect(toSet()); + return dataSourceMapper.listAuthorizedDataSource(userId, null).stream().map(DataSource::getId) + .collect(toSet()); } } @@ -526,11 +528,11 @@ public boolean permissionCheck(int userId, String url, Logger logger) { @Override public Set listAuthorizedResource(int userId, Logger logger) { - return accessTokenMapper.listAuthorizedAccessToken(userId, null).stream().map(AccessToken::getId).collect(toSet()); + return accessTokenMapper.listAuthorizedAccessToken(userId, null).stream().map(AccessToken::getId) + .collect(toSet()); } } - interface ResourceAcquisitionAndPermissionCheck { /** diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/python/PythonGateway.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/python/PythonGateway.java index 89fcc5fcf327..c465c3f9fdaa 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/python/PythonGateway.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/python/PythonGateway.java @@ -17,19 +17,6 @@ package org.apache.dolphinscheduler.api.python; -import java.io.IOException; -import java.net.InetAddress; -import java.net.UnknownHostException; -import java.util.Collections; -import java.util.Date; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.stream.Collectors; - -import javax.annotation.PostConstruct; - -import org.apache.commons.collections.CollectionUtils; import org.apache.dolphinscheduler.api.configuration.PythonGatewayConfiguration; import org.apache.dolphinscheduler.api.dto.EnvironmentDto; import org.apache.dolphinscheduler.api.dto.resources.ResourceComponent; @@ -45,7 +32,6 @@ import org.apache.dolphinscheduler.api.service.UsersService; import org.apache.dolphinscheduler.api.utils.Result; import org.apache.dolphinscheduler.common.Constants; -import org.apache.dolphinscheduler.common.enums.AuthorizationType; import org.apache.dolphinscheduler.common.enums.ComplementDependentMode; import org.apache.dolphinscheduler.common.enums.FailureStrategy; import org.apache.dolphinscheduler.common.enums.Priority; @@ -57,7 +43,6 @@ import org.apache.dolphinscheduler.common.enums.UserType; import org.apache.dolphinscheduler.common.enums.WarningType; import org.apache.dolphinscheduler.common.utils.CodeGenerateUtils; -import org.apache.dolphinscheduler.common.utils.PropertyUtils; import org.apache.dolphinscheduler.dao.entity.DataSource; import org.apache.dolphinscheduler.dao.entity.ProcessDefinition; import org.apache.dolphinscheduler.dao.entity.Project; @@ -75,14 +60,30 @@ import org.apache.dolphinscheduler.dao.mapper.ScheduleMapper; import org.apache.dolphinscheduler.dao.mapper.TaskDefinitionMapper; import org.apache.dolphinscheduler.spi.enums.ResourceType; + +import py4j.GatewayServer; + +import org.apache.commons.collections.CollectionUtils; + +import java.io.IOException; +import java.net.InetAddress; +import java.net.UnknownHostException; +import java.util.Date; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + +import javax.annotation.PostConstruct; + import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; -import py4j.GatewayServer; @Component public class PythonGateway { + private static final Logger logger = LoggerFactory.getLogger(PythonGateway.class); private static final FailureStrategy DEFAULT_FAILURE_STRATEGY = FailureStrategy.CONTINUE; @@ -147,6 +148,7 @@ public class PythonGateway { // TODO replace this user to build in admin user if we make sure build in one could not be change private final User dummyAdminUser = new User() { + { setId(ADMIN_USER_ID); setUserName("dummyUser"); @@ -155,6 +157,7 @@ public class PythonGateway { }; private final Queue queuePythonGateway = new Queue() { + { setId(Integer.MAX_VALUE); setQueueName("queuePythonGateway"); @@ -171,7 +174,8 @@ public Map genTaskCodeList(Integer genNum) { return taskDefinitionService.genTaskCodeList(genNum); } - public Map getCodeAndVersion(String projectName, String processDefinitionName, String taskName) throws CodeGenerateUtils.CodeGenerateException { + public Map getCodeAndVersion(String projectName, String processDefinitionName, + String taskName) throws CodeGenerateUtils.CodeGenerateException { Project project = projectMapper.queryByName(projectName); Map result = new HashMap<>(); // project do not exists, mean task not exists too, so we should directly return init value @@ -181,15 +185,18 @@ public Map getCodeAndVersion(String projectName, String processDef return result; } - ProcessDefinition processDefinition = processDefinitionMapper.queryByDefineName(project.getCode(), processDefinitionName); - // In the case project exists, but current process definition still not created, we should also return the init version of it + ProcessDefinition processDefinition = + processDefinitionMapper.queryByDefineName(project.getCode(), processDefinitionName); + // In the case project exists, but current process definition still not created, we should also return the init + // version of it if (processDefinition == null) { result.put("code", CodeGenerateUtils.getInstance().genCode()); result.put("version", 0L); return result; } - TaskDefinition taskDefinition = taskDefinitionMapper.queryByName(project.getCode(), processDefinition.getCode(), taskName); + TaskDefinition taskDefinition = + taskDefinitionMapper.queryByName(project.getCode(), processDefinition.getCode(), taskName); if (taskDefinition == null) { result.put("code", CodeGenerateUtils.getInstance().genCode()); result.put("version", 0L); @@ -249,21 +256,26 @@ public Long createOrUpdateProcessDefinition(String userName, if (processDefinition != null) { processDefinitionCode = processDefinition.getCode(); // make sure process definition offline which could edit - processDefinitionService.releaseProcessDefinition(user, projectCode, processDefinitionCode, ReleaseState.OFFLINE); - Map result = processDefinitionService.updateProcessDefinition(user, projectCode, name, processDefinitionCode, description, globalParams, + processDefinitionService.releaseProcessDefinition(user, projectCode, processDefinitionCode, + ReleaseState.OFFLINE); + Map result = processDefinitionService.updateProcessDefinition(user, projectCode, name, + processDefinitionCode, description, globalParams, null, timeout, tenantCode, taskRelationJson, taskDefinitionJson, otherParamsJson, executionType); } else { - Map result = processDefinitionService.createProcessDefinition(user, projectCode, name, description, globalParams, - null, timeout, tenantCode, taskRelationJson, taskDefinitionJson, otherParamsJson, executionType); + Map result = processDefinitionService.createProcessDefinition(user, projectCode, name, + description, globalParams, + null, timeout, tenantCode, taskRelationJson, taskDefinitionJson, otherParamsJson, executionType); processDefinition = (ProcessDefinition) result.get(Constants.DATA_LIST); processDefinitionCode = processDefinition.getCode(); } // Fresh process definition schedule if (schedule != null) { - createOrUpdateSchedule(user, projectCode, processDefinitionCode, schedule, workerGroup, warningType, warningGroupId); + createOrUpdateSchedule(user, projectCode, processDefinitionCode, schedule, workerGroup, warningType, + warningGroupId); } - processDefinitionService.releaseProcessDefinition(user, projectCode, processDefinitionCode, ReleaseState.getEnum(releaseState)); + processDefinitionService.releaseProcessDefinition(user, projectCode, processDefinitionCode, + ReleaseState.getEnum(releaseState)); return processDefinitionCode; } @@ -275,14 +287,16 @@ public Long createOrUpdateProcessDefinition(String userName, * @param processDefinitionName process definition name */ private ProcessDefinition getProcessDefinition(User user, long projectCode, String processDefinitionName) { - Map verifyProcessDefinitionExists = processDefinitionService.verifyProcessDefinitionName(user, projectCode, processDefinitionName, 0); + Map verifyProcessDefinitionExists = + processDefinitionService.verifyProcessDefinitionName(user, projectCode, processDefinitionName, 0); Status verifyStatus = (Status) verifyProcessDefinitionExists.get(Constants.STATUS); ProcessDefinition processDefinition = null; if (verifyStatus == Status.PROCESS_DEFINITION_NAME_EXIST) { processDefinition = processDefinitionMapper.queryByDefineName(projectCode, processDefinitionName); } else if (verifyStatus != Status.SUCCESS) { - String msg = "Verify process definition exists status is invalid, neither SUCCESS or PROCESS_DEFINITION_NAME_EXIST."; + String msg = + "Verify process definition exists status is invalid, neither SUCCESS or PROCESS_DEFINITION_NAME_EXIST."; logger.error(msg); throw new RuntimeException(msg); } @@ -314,13 +328,16 @@ private void createOrUpdateSchedule(User user, // create or update schedule int scheduleId; if (scheduleObj == null) { - processDefinitionService.releaseProcessDefinition(user, projectCode, processDefinitionCode, ReleaseState.ONLINE); - Map result = schedulerService.insertSchedule(user, projectCode, processDefinitionCode, schedule, WarningType.valueOf(warningType), + processDefinitionService.releaseProcessDefinition(user, projectCode, processDefinitionCode, + ReleaseState.ONLINE); + Map result = schedulerService.insertSchedule(user, projectCode, processDefinitionCode, + schedule, WarningType.valueOf(warningType), warningGroupId, DEFAULT_FAILURE_STRATEGY, DEFAULT_PRIORITY, workerGroup, DEFAULT_ENVIRONMENT_CODE); scheduleId = (int) result.get("scheduleId"); } else { scheduleId = scheduleObj.getId(); - processDefinitionService.releaseProcessDefinition(user, projectCode, processDefinitionCode, ReleaseState.OFFLINE); + processDefinitionService.releaseProcessDefinition(user, projectCode, processDefinitionCode, + ReleaseState.OFFLINE); schedulerService.updateSchedule(user, projectCode, scheduleId, schedule, WarningType.valueOf(warningType), warningGroupId, DEFAULT_FAILURE_STRATEGY, DEFAULT_PRIORITY, workerGroup, DEFAULT_ENVIRONMENT_CODE); } @@ -334,14 +351,15 @@ public void execProcessInstance(String userName, String workerGroup, String warningType, Integer warningGroupId, - Integer timeout - ) { + Integer timeout) { User user = usersService.queryUser(userName); Project project = projectMapper.queryByName(projectName); - ProcessDefinition processDefinition = processDefinitionMapper.queryByDefineName(project.getCode(), processDefinitionName); + ProcessDefinition processDefinition = + processDefinitionMapper.queryByDefineName(project.getCode(), processDefinitionName); // make sure process definition online - processDefinitionService.releaseProcessDefinition(user, project.getCode(), processDefinition.getCode(), ReleaseState.ONLINE); + processDefinitionService.releaseProcessDefinition(user, project.getCode(), processDefinition.getCode(), + ReleaseState.ONLINE); executorService.execProcessInstance(user, project.getCode(), @@ -362,14 +380,13 @@ public void execProcessInstance(String userName, null, DEFAULT_DRY_RUN, DEFAULT_TEST_FLAG, - COMPLEMENT_DEPENDENT_MODE - ); + COMPLEMENT_DEPENDENT_MODE); } // side object /* - Grant project's permission to user. Use when project's created user not current but - Python API use it to change process definition. + * Grant project's permission to user. Use when project's created user not current but Python API use it to change + * process definition. */ private Integer grantProjectToUser(Project project, User user) { Date now = new Date(); @@ -383,8 +400,8 @@ private Integer grantProjectToUser(Project project, User user) { } /* - Grant or create project. Create a new project if project do not exists, and grant the project - permission to user if project exists but without permission to this user. + * Grant or create project. Create a new project if project do not exists, and grant the project permission to user + * if project exists but without permission to this user. */ public void createOrGrantProject(String userName, String name, String desc) { User user = usersService.queryUser(userName); @@ -452,7 +469,8 @@ public User queryUser(int id) { return user; } - public User updateUser(String userName, String userPassword, String email, String phone, String tenantCode, String queue, int state) throws Exception { + public User updateUser(String userName, String userPassword, String email, String phone, String tenantCode, + String queue, int state) throws Exception { return usersService.createUserIfNotExists(userName, userPassword, email, phone, tenantCode, queue, state); } @@ -496,7 +514,8 @@ public Map getDatasourceInfo(String datasourceName) { * @param projectName project name which process definition belongs to * @param processDefinitionName process definition name */ - public Map getProcessDefinitionInfo(String userName, String projectName, String processDefinitionName) { + public Map getProcessDefinitionInfo(String userName, String projectName, + String processDefinitionName) { Map result = new HashMap<>(); User user = usersService.queryUser(userName); @@ -506,7 +525,8 @@ public Map getProcessDefinitionInfo(String userName, String proj // get process definition info if (processDefinition != null) { // make sure process definition online - processDefinitionService.releaseProcessDefinition(user, projectCode, processDefinition.getCode(), ReleaseState.ONLINE); + processDefinitionService.releaseProcessDefinition(user, projectCode, processDefinition.getCode(), + ReleaseState.ONLINE); result.put("id", processDefinition.getId()); result.put("name", processDefinition.getName()); result.put("code", processDefinition.getCode()); @@ -539,7 +559,8 @@ public Map getDependentInfo(String projectName, String processDe long projectCode = project.getCode(); result.put("projectCode", projectCode); - ProcessDefinition processDefinition = processDefinitionMapper.queryByDefineName(projectCode, processDefinitionName); + ProcessDefinition processDefinition = + processDefinitionMapper.queryByDefineName(projectCode, processDefinitionName); if (processDefinition == null) { String msg = String.format("Can not find valid process definition by name %s", processDefinitionName); logger.error(msg); @@ -548,7 +569,8 @@ public Map getDependentInfo(String projectName, String processDe result.put("processDefinitionCode", processDefinition.getCode()); if (taskName != null) { - TaskDefinition taskDefinition = taskDefinitionMapper.queryByName(projectCode, processDefinition.getCode(), taskName); + TaskDefinition taskDefinition = + taskDefinitionMapper.queryByName(projectCode, processDefinition.getCode(), taskName); result.put("taskDefinitionCode", taskDefinition.getCode()); } return result; @@ -564,11 +586,14 @@ public Map getDependentInfo(String projectName, String processDe public Map getResourcesFileInfo(String programType, String fullName) { Map result = new HashMap<>(); - Result resources = resourceService.queryResourceByProgramType(dummyAdminUser, ResourceType.FILE, ProgramType.valueOf(programType)); + Result resources = resourceService.queryResourceByProgramType(dummyAdminUser, ResourceType.FILE, + ProgramType.valueOf(programType)); List resourcesComponent = (List) resources.getData(); - List namedResources = resourcesComponent.stream().filter(s -> fullName.equals(s.getFullName())).collect(Collectors.toList()); + List namedResources = + resourcesComponent.stream().filter(s -> fullName.equals(s.getFullName())).collect(Collectors.toList()); if (CollectionUtils.isEmpty(namedResources)) { - String msg = String.format("Can not find valid resource by program type %s and name %s", programType, fullName); + String msg = + String.format("Can not find valid resource by program type %s and name %s", programType, fullName); logger.error(msg); throw new IllegalArgumentException(msg); } @@ -596,7 +621,6 @@ public Long getEnvironmentInfo(String environmentName) { return environmentDto.getCode(); } - /** * Get resource by given resource type and full name. It return map contain resource id, name. * Useful in Python API create task which need processDefinition information. @@ -623,7 +647,8 @@ public String getGatewayVersion() { * @return id of resource */ public Integer createOrUpdateResource( - String userName, String fullName, String description, String resourceContent) { + String userName, String fullName, String description, + String resourceContent) { return resourceService.createOrUpdateResource(userName, fullName, description, resourceContent); } @@ -640,15 +665,14 @@ private void start() { InetAddress gatewayHost = InetAddress.getByName(pythonGatewayConfiguration.getGatewayServerAddress()); InetAddress pythonHost = InetAddress.getByName(pythonGatewayConfiguration.getPythonAddress()); server = new GatewayServer( - this, - pythonGatewayConfiguration.getGatewayServerPort(), - pythonGatewayConfiguration.getPythonPort(), - gatewayHost, - pythonHost, - pythonGatewayConfiguration.getConnectTimeout(), - pythonGatewayConfiguration.getReadTimeout(), - null - ); + this, + pythonGatewayConfiguration.getGatewayServerPort(), + pythonGatewayConfiguration.getPythonPort(), + gatewayHost, + pythonHost, + pythonGatewayConfiguration.getConnectTimeout(), + pythonGatewayConfiguration.getReadTimeout(), + null); GatewayServer.turnLoggingOn(); logger.info("PythonGatewayService started on: " + gatewayHost.toString()); server.start(); diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/security/Authenticator.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/security/Authenticator.java index 99769b840bdc..5c2a556bb7f7 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/security/Authenticator.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/security/Authenticator.java @@ -25,6 +25,7 @@ import javax.servlet.http.HttpServletRequest; public interface Authenticator { + /** * Verifying legality via username and password * @param username user name diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/security/SecurityConfig.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/security/SecurityConfig.java index 6a17068e220e..3e3bb1ae796f 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/security/SecurityConfig.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/security/SecurityConfig.java @@ -32,6 +32,7 @@ @Configuration public class SecurityConfig { + private static final Logger logger = LoggerFactory.getLogger(SecurityConfig.class); @Value("${security.authentication.type:PASSWORD}") diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/security/impl/AbstractAuthenticator.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/security/impl/AbstractAuthenticator.java index 8512aaf248f5..24587948c1b0 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/security/impl/AbstractAuthenticator.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/security/impl/AbstractAuthenticator.java @@ -38,6 +38,7 @@ import org.springframework.beans.factory.annotation.Autowired; public abstract class AbstractAuthenticator implements Authenticator { + private static final Logger logger = LoggerFactory.getLogger(AbstractAuthenticator.class); @Autowired @@ -106,7 +107,7 @@ public User getAuthUser(HttpServletRequest request) { logger.info("session info is null "); return null; } - //get user object from session + // get user object from session return userService.queryUser(session.getUserId()); } diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/security/impl/ldap/LdapAuthenticator.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/security/impl/ldap/LdapAuthenticator.java index 394ed440657f..e849f884a550 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/security/impl/ldap/LdapAuthenticator.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/security/impl/ldap/LdapAuthenticator.java @@ -32,7 +32,7 @@ public User login(String userId, String password, String extra) { User user = null; String ldapEmail = ldapService.ldapLogin(userId, password); if (ldapEmail != null) { - //check if user exist + // check if user exist user = userService.getUserByUserName(userId); if (user == null && ldapService.createIfUserNotExists()) { user = userService.createUser(ldapService.getUserType(userId), userId, ldapEmail); diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/AccessTokenService.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/AccessTokenService.java index 1668cffe3732..2dbfc4dce51d 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/AccessTokenService.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/AccessTokenService.java @@ -57,7 +57,6 @@ public interface AccessTokenService { */ Result createToken(User loginUser, int userId, String expireTime, String token); - /** * generate token * diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/AlertGroupService.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/AlertGroupService.java index 8d7e49af510f..538518b5ff22 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/AlertGroupService.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/AlertGroupService.java @@ -76,7 +76,8 @@ public interface AlertGroupService { * @param alertInstanceIds alertInstanceIds * @return update result code */ - Map updateAlertgroup(User loginUser, int id, String groupName, String desc, String alertInstanceIds); + Map updateAlertgroup(User loginUser, int id, String groupName, String desc, + String alertInstanceIds); /** * delete alert group by id diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/AlertPluginInstanceService.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/AlertPluginInstanceService.java index 5296f31e11ac..24c7ce510080 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/AlertPluginInstanceService.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/AlertPluginInstanceService.java @@ -36,7 +36,7 @@ public interface AlertPluginInstanceService { * @param pluginInstanceParams plugin instance params * @return result */ - Map create(User loginUser,int pluginDefineId,String instanceName,String pluginInstanceParams); + Map create(User loginUser, int pluginDefineId, String instanceName, String pluginInstanceParams); /** * update @@ -46,7 +46,8 @@ public interface AlertPluginInstanceService { * @param pluginInstanceParams plugin instance params * @return result */ - Map update(User loginUser, int alertPluginInstanceId,String instanceName,String pluginInstanceParams); + Map update(User loginUser, int alertPluginInstanceId, String instanceName, + String pluginInstanceParams); /** * delete alert plugin instance diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/BaseService.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/BaseService.java index a1fa21a9f1e3..6c86aeb0b2f1 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/BaseService.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/BaseService.java @@ -17,13 +17,14 @@ package org.apache.dolphinscheduler.api.service; -import java.util.List; -import java.util.Map; - import org.apache.dolphinscheduler.api.enums.Status; import org.apache.dolphinscheduler.api.utils.Result; import org.apache.dolphinscheduler.common.enums.AuthorizationType; import org.apache.dolphinscheduler.dao.entity.User; + +import java.util.List; +import java.util.Map; + import org.slf4j.Logger; /** @@ -39,7 +40,6 @@ public interface BaseService { */ boolean isAdmin(User user); - /** * isNotAdmin * @@ -86,7 +86,6 @@ public interface BaseService { */ boolean check(Map result, boolean bool, Status userNoOperationPerm); - /** * Verify that the operator has permissions * diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ClusterService.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ClusterService.java index 0787e5db419c..0b542579db32 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ClusterService.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ClusterService.java @@ -96,4 +96,3 @@ public interface ClusterService { Map verifyCluster(String clusterName); } - diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/DataAnalysisService.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/DataAnalysisService.java index cb4639acfc89..389731682e81 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/DataAnalysisService.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/DataAnalysisService.java @@ -19,6 +19,7 @@ import org.apache.dolphinscheduler.dao.entity.ExecuteStatusCount; import org.apache.dolphinscheduler.dao.entity.User; + import org.apache.ibatis.annotations.Param; import java.util.Date; @@ -50,7 +51,8 @@ public interface DataAnalysisService { * @param endDate end date * @return process instance state count data */ - Map countProcessInstanceStateByProject(User loginUser, long projectCode, String startDate, String endDate); + Map countProcessInstanceStateByProject(User loginUser, long projectCode, String startDate, + String endDate); /** * statistics the process definition quantities of a certain person diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/DataSourceService.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/DataSourceService.java index eaf1552caa03..c1454f948ba5 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/DataSourceService.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/DataSourceService.java @@ -142,5 +142,5 @@ public interface DataSourceService { * @param tableName * @return */ - Map getTableColumns(Integer datasourceId,String tableName); + Map getTableColumns(Integer datasourceId, String tableName); } diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/DqRuleService.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/DqRuleService.java index d0350556ac4a..df6d68d71e17 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/DqRuleService.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/DqRuleService.java @@ -38,5 +38,5 @@ Result queryRuleListPaging(User loginUser, String endTime, Integer pageNo, Integer pageSize); - Map getDatasourceOptionsById(int datasourceId); + Map getDatasourceOptionsById(int datasourceId); } diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/EnvironmentService.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/EnvironmentService.java index 617d2b26acff..ebedea43fea0 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/EnvironmentService.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/EnvironmentService.java @@ -52,7 +52,6 @@ public interface EnvironmentService { */ Map queryEnvironmentByCode(Long code); - /** * delete environment * @@ -71,7 +70,8 @@ public interface EnvironmentService { * @param desc environment desc * @param workerGroups worker groups */ - Map updateEnvironmentByCode(User loginUser, Long code, String name, String config, String desc, String workerGroups); + Map updateEnvironmentByCode(User loginUser, Long code, String name, String config, String desc, + String workerGroups); /** * query environment paging @@ -100,4 +100,3 @@ public interface EnvironmentService { Map verifyEnvironment(String environmentName); } - diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/EnvironmentWorkerGroupRelationService.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/EnvironmentWorkerGroupRelationService.java index 9db770158d15..4d22814e5f09 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/EnvironmentWorkerGroupRelationService.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/EnvironmentWorkerGroupRelationService.java @@ -38,4 +38,3 @@ public interface EnvironmentWorkerGroupRelationService { */ Map queryAllEnvironmentWorkerGroupRelationList(); } - diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ExecutorService.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ExecutorService.java index 9aaabb36b671..6f19a29caf81 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ExecutorService.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ExecutorService.java @@ -61,7 +61,8 @@ public interface ExecutorService { Map execProcessInstance(User loginUser, long projectCode, long processDefinitionCode, String cronTime, CommandType commandType, FailureStrategy failureStrategy, String startNodeList, - TaskDependType taskDependType, WarningType warningType, Integer warningGroupId, + TaskDependType taskDependType, WarningType warningType, + Integer warningGroupId, RunMode runMode, Priority processInstancePriority, String workerGroup, Long environmentCode, Integer timeout, diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/K8sNamespaceService.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/K8sNamespaceService.java index d3f15a4dd69e..7a14169acc04 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/K8sNamespaceService.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/K8sNamespaceService.java @@ -50,7 +50,8 @@ public interface K8sNamespaceService { * @param limitsMemory limits memory, can null means not limit * @return */ - Map createK8sNamespace(User loginUser, String namespace, Long clusterCode, Double limitsCpu, Integer limitsMemory); + Map createK8sNamespace(User loginUser, String namespace, Long clusterCode, Double limitsCpu, + Integer limitsMemory); /** * update K8s Namespace tag and resource limit @@ -61,7 +62,8 @@ public interface K8sNamespaceService { * @param limitsMemory max memory * @return */ - Map updateK8sNamespace(User loginUser, int id, String userName, Double limitsCpu, Integer limitsMemory); + Map updateK8sNamespace(User loginUser, int id, String userName, Double limitsCpu, + Integer limitsMemory); /** * verify namespace and k8s @@ -106,4 +108,4 @@ public interface K8sNamespaceService { * @return namespace list */ List queryNamespaceAvailable(User loginUser); -} \ No newline at end of file +} diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/LoggerService.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/LoggerService.java index f7ba946e4c12..33477c509e37 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/LoggerService.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/LoggerService.java @@ -38,7 +38,6 @@ public interface LoggerService { */ Result queryLog(int taskInstId, int skipLineNum, int limit); - /** * get log size * diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/MonitorService.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/MonitorService.java index d301957955f8..e881b2e346dc 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/MonitorService.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/MonitorService.java @@ -34,23 +34,23 @@ public interface MonitorService { * @param loginUser login user * @return data base state */ - Map queryDatabaseState(User loginUser); - + Map queryDatabaseState(User loginUser); + /** * query master list * * @param loginUser login user * @return master information list */ - Map queryMaster(User loginUser); - + Map queryMaster(User loginUser); + /** * query worker list * * @param loginUser login user * @return worker information list */ - Map queryWorker(User loginUser); - + Map queryWorker(User loginUser); + List getServerListFromRegistry(boolean isMaster); } diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ProcessDefinitionService.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ProcessDefinitionService.java index e887d0f01738..8019cc40ae71 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ProcessDefinitionService.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ProcessDefinitionService.java @@ -245,8 +245,8 @@ Map verifyProcessDefinitionName(User loginUser, * @return delete result code */ Map batchDeleteProcessDefinitionByCodes(User loginUser, - long projectCode, - String codes); + long projectCode, + String codes); /** * delete process definition by code diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ProcessInstanceService.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ProcessInstanceService.java index ca03353740e7..d84ca5d3cb0d 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ProcessInstanceService.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ProcessInstanceService.java @@ -18,16 +18,16 @@ package org.apache.dolphinscheduler.api.service; -import java.io.IOException; -import java.util.List; -import java.util.Map; - import org.apache.dolphinscheduler.api.utils.Result; import org.apache.dolphinscheduler.common.enums.WorkflowExecutionStatus; import org.apache.dolphinscheduler.dao.entity.ProcessInstance; import org.apache.dolphinscheduler.dao.entity.User; import org.apache.dolphinscheduler.plugin.task.api.enums.DependResult; +import java.io.IOException; +import java.util.List; +import java.util.Map; + /** * process instance service */ @@ -201,4 +201,4 @@ List queryByProcessDefineCodeAndStatus(Long processDefinitionCo */ List queryByProcessDefineCode(Long processDefinitionCode, int size); -} \ No newline at end of file +} diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/QueueService.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/QueueService.java index 553cde34b5b4..0f62ea902882 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/QueueService.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/QueueService.java @@ -21,8 +21,6 @@ import org.apache.dolphinscheduler.dao.entity.Queue; import org.apache.dolphinscheduler.dao.entity.User; -import java.util.Map; - /** * queue service */ diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ResourcesService.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ResourcesService.java index 00c6fa151b7e..4de830c07490 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ResourcesService.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ResourcesService.java @@ -98,7 +98,8 @@ Result updateResource(User loginUser, * @param pageSize page size * @return resource list page */ - Result queryResourceListPaging(User loginUser, int directoryId, ResourceType type, String searchVal, Integer pageNo, Integer pageSize); + Result queryResourceListPaging(User loginUser, int directoryId, ResourceType type, String searchVal, Integer pageNo, + Integer pageSize); /** * query resource list @@ -135,7 +136,7 @@ Result updateResource(User loginUser, * @param type resource type * @return true if the resource name not exists, otherwise return false */ - Result verifyResourceName(String fullName, ResourceType type,User loginUser); + Result verifyResourceName(String fullName, ResourceType type, User loginUser); /** * verify resource by full name or pid and type @@ -144,7 +145,7 @@ Result updateResource(User loginUser, * @param type resource type * @return true if the resource full name or pid not exists, otherwise return false */ - Result queryResource(User loginUser,String fullName,Integer id,ResourceType type); + Result queryResource(User loginUser, String fullName, Integer id, ResourceType type); /** * view resource file online @@ -154,7 +155,7 @@ Result updateResource(User loginUser, * @param limit limit * @return resource content */ - Result readResource(User loginUser,int resourceId, int skipLineNum, int limit); + Result readResource(User loginUser, int resourceId, int skipLineNum, int limit); /** * create resource file online @@ -167,7 +168,8 @@ Result updateResource(User loginUser, * @param content content * @return create result code */ - Result onlineCreateResource(User loginUser, ResourceType type, String fileName, String fileSuffix, String desc, String content,int pid,String currentDirectory); + Result onlineCreateResource(User loginUser, ResourceType type, String fileName, String fileSuffix, + String desc, String content, int pid, String currentDirectory); /** * create or update resource. @@ -179,7 +181,8 @@ Result updateResource(User loginUser, * @param content content of resource * @return create result code */ - Result onlineCreateOrUpdateResourceWithDir(User loginUser, String fileFullName, String desc, String content); + Result onlineCreateOrUpdateResourceWithDir(User loginUser, String fileFullName, String desc, + String content); /** * create or update resource. @@ -200,7 +203,7 @@ Result updateResource(User loginUser, * @param content content * @return update result cod */ - Result updateResourceContent(User loginUser,int resourceId, String content); + Result updateResourceContent(User loginUser, int resourceId, String content); /** * download file diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/TaskGroupQueueService.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/TaskGroupQueueService.java index e6e81d950b11..a0d05c5b8d70 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/TaskGroupQueueService.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/TaskGroupQueueService.java @@ -35,11 +35,11 @@ public interface TaskGroupQueueService { * @param status Task queue status * @param pageNo page no * @param pageSize page size - + * @return tasks list */ - Map queryTasksByGroupId(User loginUser, String taskName - , String processName, Integer status, int groupId, int pageNo,int pageSize); + Map queryTasksByGroupId(User loginUser, String taskName, String processName, Integer status, + int groupId, int pageNo, int pageSize); /** * query tasks in task group queue by project id @@ -68,7 +68,7 @@ Map queryTasksByProcessId(User loginUser, int pageNo, */ boolean deleteByTaskId(int taskId); - void forceStartTask(int queueId,int forceStart); + void forceStartTask(int queueId, int forceStart); void modifyPriority(Integer queueId, Integer priority); } diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/TaskGroupService.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/TaskGroupService.java index 4b15ca878fd2..398710973386 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/TaskGroupService.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/TaskGroupService.java @@ -66,7 +66,7 @@ Map updateTaskGroup(User loginUser, int id, String name, * @param pageSize page size * @return the result code and msg */ - Map queryAllTaskGroup(User loginUser, String name,Integer status, int pageNo, int pageSize); + Map queryAllTaskGroup(User loginUser, String name, Integer status, int pageNo, int pageSize); /** * query all task group by status diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/TenantService.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/TenantService.java index 0d795a10c4bd..fc5abf4b6b19 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/TenantService.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/TenantService.java @@ -66,7 +66,7 @@ Map createTenant(User loginUser, * @throws Exception exception */ Map updateTenant(User loginUser, int id, String tenantCode, int queueId, - String desc) throws Exception; + String desc) throws Exception; /** * delete tenant diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/UdfFuncService.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/UdfFuncService.java index 1f5d1ab3390f..0cac41663fb5 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/UdfFuncService.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/UdfFuncService.java @@ -21,8 +21,6 @@ import org.apache.dolphinscheduler.common.enums.UdfType; import org.apache.dolphinscheduler.dao.entity.User; -import java.util.Map; - /** * udf func service */ @@ -56,7 +54,7 @@ Result createUdfFunction(User loginUser, * @param id udf function id * @return udf function detail */ - Result queryUdfFuncDetail(User loginUser,int id); + Result queryUdfFuncDetail(User loginUser, int id); /** * updateProcessInstance udf function @@ -72,14 +70,14 @@ Result createUdfFunction(User loginUser, * @return update result code */ Result updateUdfFunc(User loginUser, - int udfFuncId, - String funcName, - String className, - String argTypes, - String database, - String desc, - UdfType type, - int resourceId); + int udfFuncId, + String funcName, + String className, + String argTypes, + String database, + String desc, + UdfType type, + int resourceId); /** * query udf function list paging @@ -117,4 +115,4 @@ Result updateUdfFunc(User loginUser, */ Result verifyUdfFuncByName(User loginUser, String name); -} \ No newline at end of file +} diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/UsersService.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/UsersService.java index b4555ad50574..d854e9afddb3 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/UsersService.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/UsersService.java @@ -130,7 +130,8 @@ User createUser(String userName, String userPassword, String email, * @throws Exception exception */ Map updateUser(User loginUser, int userId, String userName, String userPassword, String email, - int tenantId, String phone, String queue, int state, String timeZone) throws IOException; + int tenantId, String phone, String queue, int state, + String timeZone) throws IOException; /** * delete user @@ -152,7 +153,6 @@ Map updateUser(User loginUser, int userId, String userName, Stri */ Map grantProject(User loginUser, int userId, String projectIds); - /** * grant project by code * @@ -182,7 +182,6 @@ Map updateUser(User loginUser, int userId, String userName, Stri */ Map grantResources(User loginUser, int userId, String resourceIds); - /** * grant udf function * @@ -193,7 +192,6 @@ Map updateUser(User loginUser, int userId, String userName, Stri */ Map grantUDFFunction(User loginUser, int userId, String udfIds); - /** * grant namespace * @@ -204,7 +202,6 @@ Map updateUser(User loginUser, int userId, String userName, Stri */ Map grantNamespaces(User loginUser, int userId, String namespaceIds); - /** * grant datasource * @@ -231,7 +228,6 @@ Map updateUser(User loginUser, int userId, String userName, Stri */ Map queryAllGeneralUsers(User loginUser); - /** * query user list * @@ -248,7 +244,6 @@ Map updateUser(User loginUser, int userId, String userName, Stri */ Result verifyUserName(String userName); - /** * unauthorized user * @@ -258,7 +253,6 @@ Map updateUser(User loginUser, int userId, String userName, Stri */ Map unauthorizedUser(User loginUser, Integer alertGroupId); - /** * authorized user * diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/WorkerGroupService.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/WorkerGroupService.java index 4c474b75cda1..b2f9d7817804 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/WorkerGroupService.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/WorkerGroupService.java @@ -38,7 +38,8 @@ public interface WorkerGroupService { * @param otherParamsJson otherParamsJson * @return create or update result code */ - Map saveWorkerGroup(User loginUser, int id, String name, String addrList, String description, String otherParamsJson); + Map saveWorkerGroup(User loginUser, int id, String name, String addrList, String description, + String otherParamsJson); /** * query worker group paging diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/AccessTokenServiceImpl.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/AccessTokenServiceImpl.java index 42b56a090902..17e0729f9c02 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/AccessTokenServiceImpl.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/AccessTokenServiceImpl.java @@ -125,7 +125,8 @@ public Result createToken(User loginUser, int userId, String expireTime, String Result result = new Result(); // 1. check permission - if (!(canOperatorPermissions(loginUser,null, AuthorizationType.ACCESS_TOKEN,ACCESS_TOKEN_CREATE) || loginUser.getId() == userId)) { + if (!(canOperatorPermissions(loginUser, null, AuthorizationType.ACCESS_TOKEN, ACCESS_TOKEN_CREATE) + || loginUser.getId() == userId)) { putMsg(result, Status.USER_NO_OPERATION_PERM); return result; } @@ -190,7 +191,7 @@ public Map generateToken(User loginUser, int userId, String expi @Override public Map delAccessTokenById(User loginUser, int id) { Map result = new HashMap<>(); - if (!canOperatorPermissions(loginUser, null, AuthorizationType.ACCESS_TOKEN,ACCESS_TOKEN_DELETE)) { + if (!canOperatorPermissions(loginUser, null, AuthorizationType.ACCESS_TOKEN, ACCESS_TOKEN_DELETE)) { putMsg(result, Status.USER_NO_OPERATION_PERM); return result; } @@ -226,7 +227,7 @@ public Map updateToken(User loginUser, int id, int userId, Strin Map result = new HashMap<>(); // 1. check permission - if (!canOperatorPermissions(loginUser, null,AuthorizationType.ACCESS_TOKEN,ACCESS_TOKEN_UPDATE)) { + if (!canOperatorPermissions(loginUser, null, AuthorizationType.ACCESS_TOKEN, ACCESS_TOKEN_UPDATE)) { putMsg(result, Status.USER_NO_OPERATION_PERM); return result; } diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/AlertGroupServiceImpl.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/AlertGroupServiceImpl.java index 2b528acf465e..145c9aee8405 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/AlertGroupServiceImpl.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/AlertGroupServiceImpl.java @@ -17,6 +17,11 @@ package org.apache.dolphinscheduler.api.service.impl; +import static org.apache.dolphinscheduler.api.constants.ApiFuncIdentificationConstant.ALERT_GROUP_CREATE; +import static org.apache.dolphinscheduler.api.constants.ApiFuncIdentificationConstant.ALERT_GROUP_DELETE; +import static org.apache.dolphinscheduler.api.constants.ApiFuncIdentificationConstant.ALERT_GROUP_UPDATE; +import static org.apache.dolphinscheduler.api.constants.ApiFuncIdentificationConstant.ALERT_GROUP_VIEW; + import org.apache.dolphinscheduler.api.enums.Status; import org.apache.dolphinscheduler.api.service.AlertGroupService; import org.apache.dolphinscheduler.api.utils.PageInfo; @@ -48,8 +53,6 @@ import com.baomidou.mybatisplus.core.metadata.IPage; import com.baomidou.mybatisplus.extension.plugins.pagination.Page; -import static org.apache.dolphinscheduler.api.constants.ApiFuncIdentificationConstant.*; - /** * alert group service impl */ @@ -74,7 +77,8 @@ public Map queryAlertgroup(User loginUser) { if (loginUser.getUserType().equals(UserType.ADMIN_USER)) { alertGroups = alertGroupMapper.queryAllGroupList(); } else { - Set ids = resourcePermissionCheckService.userOwnedResourceIdsAcquisition(AuthorizationType.ALERT_GROUP, loginUser.getId(), logger); + Set ids = resourcePermissionCheckService + .userOwnedResourceIdsAcquisition(AuthorizationType.ALERT_GROUP, loginUser.getId(), logger); if (ids.isEmpty()) { result.put(Constants.DATA_LIST, Collections.emptyList()); putMsg(result, Status.SUCCESS); @@ -99,12 +103,12 @@ public Map queryAlertGroupById(User loginUser, Integer id) { Map result = new HashMap<>(); result.put(Constants.STATUS, false); - //only admin can operate - if (!canOperatorPermissions(loginUser,new Object[]{id}, AuthorizationType.ALERT_GROUP,ALERT_GROUP_VIEW)) { + // only admin can operate + if (!canOperatorPermissions(loginUser, new Object[]{id}, AuthorizationType.ALERT_GROUP, ALERT_GROUP_VIEW)) { putMsg(result, Status.USER_NO_OPERATION_PERM); return result; } - //check if exist + // check if exist AlertGroup alertGroup = alertGroupMapper.selectById(id); if (alertGroup == null) { putMsg(result, Status.ALERT_GROUP_NOT_EXIST); @@ -134,7 +138,8 @@ public Result listPaging(User loginUser, String searchVal, Integer pageNo, Integ if (loginUser.getUserType().equals(UserType.ADMIN_USER)) { alertGroupPage = alertGroupMapper.queryAlertGroupPage(page, searchVal); } else { - Set ids = resourcePermissionCheckService.userOwnedResourceIdsAcquisition(AuthorizationType.ALERT_GROUP, loginUser.getId(), logger); + Set ids = resourcePermissionCheckService + .userOwnedResourceIdsAcquisition(AuthorizationType.ALERT_GROUP, loginUser.getId(), logger); if (ids.isEmpty()) { result.setData(pageInfo); putMsg(result, Status.SUCCESS); @@ -161,14 +166,15 @@ public Result listPaging(User loginUser, String searchVal, Integer pageNo, Integ */ @Override @Transactional - public Map createAlertgroup(User loginUser, String groupName, String desc, String alertInstanceIds) { + public Map createAlertgroup(User loginUser, String groupName, String desc, + String alertInstanceIds) { Map result = new HashMap<>(); - //only admin can operate - if (!canOperatorPermissions(loginUser,null, AuthorizationType.ALERT_GROUP, ALERT_GROUP_CREATE)) { + // only admin can operate + if (!canOperatorPermissions(loginUser, null, AuthorizationType.ALERT_GROUP, ALERT_GROUP_CREATE)) { putMsg(result, Status.USER_NO_OPERATION_PERM); return result; } - if(checkDescriptionLength(desc)){ + if (checkDescriptionLength(desc)) { logger.warn("Parameter description is too long."); putMsg(result, Status.DESCRIPTION_TOO_LONG_ERROR); return result; @@ -189,7 +195,8 @@ public Map createAlertgroup(User loginUser, String groupName, St if (insert > 0) { result.put(Constants.DATA_LIST, alertGroup); putMsg(result, Status.SUCCESS); - permissionPostHandle(AuthorizationType.ALERT_GROUP, loginUser.getId(), Collections.singletonList(alertGroup.getId()), logger); + permissionPostHandle(AuthorizationType.ALERT_GROUP, loginUser.getId(), + Collections.singletonList(alertGroup.getId()), logger); logger.info("Create alert group complete, groupName:{}", alertGroup.getGroupName()); } else { logger.error("Create alert group error, groupName:{}", alertGroup.getGroupName()); @@ -214,14 +221,15 @@ public Map createAlertgroup(User loginUser, String groupName, St * @return update result code */ @Override - public Map updateAlertgroup(User loginUser, int id, String groupName, String desc, String alertInstanceIds) { + public Map updateAlertgroup(User loginUser, int id, String groupName, String desc, + String alertInstanceIds) { Map result = new HashMap<>(); - if (!canOperatorPermissions(loginUser, new Object[]{id},AuthorizationType.ALERT_GROUP,ALERT_GROUP_UPDATE)) { + if (!canOperatorPermissions(loginUser, new Object[]{id}, AuthorizationType.ALERT_GROUP, ALERT_GROUP_UPDATE)) { putMsg(result, Status.USER_NO_OPERATION_PERM); return result; } - if(checkDescriptionLength(desc)){ + if (checkDescriptionLength(desc)) { logger.warn("Parameter description is too long."); putMsg(result, Status.DESCRIPTION_TOO_LONG_ERROR); return result; @@ -268,8 +276,8 @@ public Map delAlertgroupById(User loginUser, int id) { Map result = new HashMap<>(); result.put(Constants.STATUS, false); - //only admin can operate - if (!canOperatorPermissions(loginUser,new Object[]{id}, AuthorizationType.ALERT_GROUP,ALERT_GROUP_DELETE)) { + // only admin can operate + if (!canOperatorPermissions(loginUser, new Object[]{id}, AuthorizationType.ALERT_GROUP, ALERT_GROUP_DELETE)) { putMsg(result, Status.USER_NO_OPERATION_PERM); return result; } @@ -281,7 +289,7 @@ public Map delAlertgroupById(User loginUser, int id) { return result; } - //check exist + // check exist AlertGroup alertGroup = alertGroupMapper.selectById(id); if (alertGroup == null) { logger.error("Alert group does not exist, id:{}.", id); diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/AlertPluginInstanceServiceImpl.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/AlertPluginInstanceServiceImpl.java index ee95249209b7..fa7b44362a8e 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/AlertPluginInstanceServiceImpl.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/AlertPluginInstanceServiceImpl.java @@ -17,6 +17,10 @@ package org.apache.dolphinscheduler.api.service.impl; +import static org.apache.dolphinscheduler.api.constants.ApiFuncIdentificationConstant.ALART_INSTANCE_CREATE; +import static org.apache.dolphinscheduler.api.constants.ApiFuncIdentificationConstant.ALERT_PLUGIN_DELETE; +import static org.apache.dolphinscheduler.api.constants.ApiFuncIdentificationConstant.ALERT_PLUGIN_UPDATE; + import org.apache.dolphinscheduler.api.constants.ApiFuncIdentificationConstant; import org.apache.dolphinscheduler.api.enums.Status; import org.apache.dolphinscheduler.api.service.AlertPluginInstanceService; @@ -55,8 +59,6 @@ import com.baomidou.mybatisplus.core.metadata.IPage; import com.baomidou.mybatisplus.extension.plugins.pagination.Page; -import static org.apache.dolphinscheduler.api.constants.ApiFuncIdentificationConstant.*; - /** * alert plugin instance service impl */ @@ -84,7 +86,8 @@ public class AlertPluginInstanceServiceImpl extends BaseServiceImpl implements A * @param pluginInstanceParams plugin instance params */ @Override - public Map create(User loginUser, int pluginDefineId, String instanceName, String pluginInstanceParams) { + public Map create(User loginUser, int pluginDefineId, String instanceName, + String pluginInstanceParams) { AlertPluginInstance alertPluginInstance = new AlertPluginInstance(); String paramsMapJson = parsePluginParamsMap(pluginInstanceParams); alertPluginInstance.setPluginInstanceParams(paramsMapJson); @@ -92,12 +95,13 @@ public Map create(User loginUser, int pluginDefineId, String ins alertPluginInstance.setPluginDefineId(pluginDefineId); Map result = new HashMap<>(); - if (!canOperatorPermissions(loginUser,null, AuthorizationType.ALERT_PLUGIN_INSTANCE,ALART_INSTANCE_CREATE)) { + if (!canOperatorPermissions(loginUser, null, AuthorizationType.ALERT_PLUGIN_INSTANCE, ALART_INSTANCE_CREATE)) { putMsg(result, Status.USER_NO_OPERATION_PERM); return result; } if (alertPluginInstanceMapper.existInstanceName(alertPluginInstance.getInstanceName()) == Boolean.TRUE) { - logger.error("Plugin instance with the same name already exists, name:{}.", alertPluginInstance.getInstanceName()); + logger.error("Plugin instance with the same name already exists, name:{}.", + alertPluginInstance.getInstanceName()); putMsg(result, Status.PLUGIN_INSTANCE_ALREADY_EXIT); return result; } @@ -123,14 +127,16 @@ public Map create(User loginUser, int pluginDefineId, String ins * @param pluginInstanceParams plugin instance params */ @Override - public Map update(User loginUser, int pluginInstanceId, String instanceName, String pluginInstanceParams) { + public Map update(User loginUser, int pluginInstanceId, String instanceName, + String pluginInstanceParams) { String paramsMapJson = parsePluginParamsMap(pluginInstanceParams); - AlertPluginInstance alertPluginInstance = new AlertPluginInstance(pluginInstanceId, paramsMapJson, instanceName, new Date()); + AlertPluginInstance alertPluginInstance = + new AlertPluginInstance(pluginInstanceId, paramsMapJson, instanceName, new Date()); Map result = new HashMap<>(); - if (!canOperatorPermissions(loginUser,null, AuthorizationType.ALERT_PLUGIN_INSTANCE,ALERT_PLUGIN_UPDATE)) { + if (!canOperatorPermissions(loginUser, null, AuthorizationType.ALERT_PLUGIN_INSTANCE, ALERT_PLUGIN_UPDATE)) { putMsg(result, Status.USER_NO_OPERATION_PERM); return result; } @@ -158,14 +164,14 @@ public Map update(User loginUser, int pluginInstanceId, String i @Override public Map delete(User loginUser, int id) { Map result = new HashMap<>(); - //check if there is an associated alert group + // check if there is an associated alert group boolean hasAssociatedAlertGroup = checkHasAssociatedAlertGroup(String.valueOf(id)); if (hasAssociatedAlertGroup) { logger.warn("Delete alert plugin failed because alert group is using it, pluginId:{}.", id); putMsg(result, Status.DELETE_ALERT_PLUGIN_INSTANCE_ERROR_HAS_ALERT_GROUP_ASSOCIATED); return result; } - if (!canOperatorPermissions(loginUser,null, AuthorizationType.ALERT_PLUGIN_INSTANCE,ALERT_PLUGIN_DELETE)) { + if (!canOperatorPermissions(loginUser, null, AuthorizationType.ALERT_PLUGIN_INSTANCE, ALERT_PLUGIN_DELETE)) { putMsg(result, Status.USER_NO_OPERATION_PERM); return result; } @@ -190,7 +196,8 @@ public Map delete(User loginUser, int id) { public Map get(User loginUser, int id) { Map result = new HashMap<>(); AlertPluginInstance alertPluginInstance = alertPluginInstanceMapper.selectById(id); - if (!canOperatorPermissions(loginUser,null, AuthorizationType.ALERT_PLUGIN_INSTANCE,ApiFuncIdentificationConstant.ALARM_INSTANCE_MANAGE)) { + if (!canOperatorPermissions(loginUser, null, AuthorizationType.ALERT_PLUGIN_INSTANCE, + ApiFuncIdentificationConstant.ALARM_INSTANCE_MANAGE)) { putMsg(result, Status.USER_NO_OPERATION_PERM); return result; } @@ -225,7 +232,8 @@ public Result listPaging(User loginUser, String searchVal, int pageNo, int pageS Result result = new Result(); Page page = new Page<>(pageNo, pageSize); - IPage alertPluginInstanceIPage = alertPluginInstanceMapper.queryByInstanceNamePage(page, searchVal); + IPage alertPluginInstanceIPage = + alertPluginInstanceMapper.queryByInstanceNamePage(page, searchVal); PageInfo pageInfo = new PageInfo<>(pageNo, pageSize); pageInfo.setTotal((int) alertPluginInstanceIPage.getTotal()); @@ -243,7 +251,8 @@ private List buildPluginInstanceVOList(List pluginDefineMap = pluginDefineList.stream().collect(Collectors.toMap(PluginDefine::getId, Function.identity())); + Map pluginDefineMap = + pluginDefineList.stream().collect(Collectors.toMap(PluginDefine::getId, Function.identity())); List alertPluginInstanceVOS = new ArrayList<>(); alertPluginInstances.forEach(alertPluginInstance -> { AlertPluginInstanceVO alertPluginInstanceVO = new AlertPluginInstanceVO(); @@ -254,12 +263,13 @@ private List buildPluginInstanceVOList(List> pluginParamsList = PluginParamsTransfer.generatePluginParams(pluginParamsMapString, pluginUiParams); + List> pluginParamsList = + PluginParamsTransfer.generatePluginParams(pluginParamsMapString, pluginUiParams); return JSONUtils.toJsonString(pluginParamsList); } @@ -297,7 +308,8 @@ private boolean checkHasAssociatedAlertGroup(String id) { if (CollectionUtils.isEmpty(idsList)) { return false; } - Optional first = idsList.stream().filter(k -> null != k && Arrays.asList(k.split(",")).contains(id)).findFirst(); + Optional first = + idsList.stream().filter(k -> null != k && Arrays.asList(k.split(",")).contains(id)).findFirst(); return first.isPresent(); } diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/AuditServiceImpl.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/AuditServiceImpl.java index 2ae6740838c6..bc798c9eea95 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/AuditServiceImpl.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/AuditServiceImpl.java @@ -88,7 +88,7 @@ public Result queryLogListPaging(User loginUser, AuditResourceType resourceType, Map checkAndParseDateResult = checkAndParseDateParameters(startDate, endDate); Status resultEnum = (Status) checkAndParseDateResult.get(Constants.STATUS); if (resultEnum != Status.SUCCESS) { - putMsg(result,resultEnum); + putMsg(result, resultEnum); return result; } diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/BaseServiceImpl.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/BaseServiceImpl.java index d8d24925b293..ad0a9fe82894 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/BaseServiceImpl.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/BaseServiceImpl.java @@ -17,15 +17,6 @@ package org.apache.dolphinscheduler.api.service.impl; -import java.io.IOException; -import java.text.MessageFormat; -import java.util.Date; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Objects; - -import org.apache.commons.lang3.StringUtils; import org.apache.dolphinscheduler.api.enums.Status; import org.apache.dolphinscheduler.api.permission.ResourcePermissionCheckService; import org.apache.dolphinscheduler.api.service.BaseService; @@ -35,6 +26,16 @@ import org.apache.dolphinscheduler.common.enums.UserType; import org.apache.dolphinscheduler.common.utils.DateUtils; import org.apache.dolphinscheduler.dao.entity.User; + +import org.apache.commons.lang3.StringUtils; + +import java.text.MessageFormat; +import java.util.Date; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; + import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; @@ -43,16 +44,18 @@ * base service impl */ public class BaseServiceImpl implements BaseService { + private static final Logger logger = LoggerFactory.getLogger(BaseServiceImpl.class); @Autowired protected ResourcePermissionCheckService resourcePermissionCheckService; @Override - public void permissionPostHandle(AuthorizationType authorizationType, Integer userId, List ids, Logger logger) { - try{ + public void permissionPostHandle(AuthorizationType authorizationType, Integer userId, List ids, + Logger logger) { + try { resourcePermissionCheckService.postHandle(authorizationType, userId, ids, logger); - }catch (Exception e){ + } catch (Exception e) { logger.error("Post handle error, userId:{}.", userId, e); throw new RuntimeException("Resource association user error", e); } @@ -78,7 +81,7 @@ public boolean isAdmin(User user) { */ @Override public boolean isNotAdmin(User loginUser, Map result) { - //only admin can operate + // only admin can operate if (!isAdmin(loginUser)) { putMsg(result, Status.USER_NO_OPERATION_PERM); return true; @@ -145,14 +148,14 @@ public boolean check(Map result, boolean bool, Status userNoOper * @param tenantCode tenant code * @throws IOException if hdfs operation exception */ -// @Override -// public void createTenantDirIfNotExists(String tenantCode) throws IOException { -// String resourcePath = HadoopUtils.getHdfsResDir(tenantCode); -// String udfsPath = HadoopUtils.getHdfsUdfDir(tenantCode); -// // init resource path and udf path -// HadoopUtils.getInstance().mkdir(tenantCode,resourcePath); -// HadoopUtils.getInstance().mkdir(tenantCode,udfsPath); -// } + // @Override + // public void createTenantDirIfNotExists(String tenantCode) throws IOException { + // String resourcePath = HadoopUtils.getHdfsResDir(tenantCode); + // String udfsPath = HadoopUtils.getHdfsUdfDir(tenantCode); + // // init resource path and udf path + // HadoopUtils.getInstance().mkdir(tenantCode,resourcePath); + // HadoopUtils.getInstance().mkdir(tenantCode,udfsPath); + // } /** * Verify that the operator has permissions @@ -173,9 +176,11 @@ public boolean canOperator(User operateUser, int createUserId) { * @return boolean */ @Override - public boolean canOperatorPermissions(User user, Object[] ids,AuthorizationType type,String permissionKey) { - boolean operationPermissionCheck = resourcePermissionCheckService.operationPermissionCheck(type, type.equals(AuthorizationType.PROJECTS) ? ids : null, user.getId(), permissionKey, logger); - boolean resourcePermissionCheck = resourcePermissionCheckService.resourcePermissionCheck(type, ids, user.getUserType().equals(UserType.ADMIN_USER) ? 0 : user.getId(), logger); + public boolean canOperatorPermissions(User user, Object[] ids, AuthorizationType type, String permissionKey) { + boolean operationPermissionCheck = resourcePermissionCheckService.operationPermissionCheck(type, + type.equals(AuthorizationType.PROJECTS) ? ids : null, user.getId(), permissionKey, logger); + boolean resourcePermissionCheck = resourcePermissionCheckService.resourcePermissionCheck(type, ids, + user.getUserType().equals(UserType.ADMIN_USER) ? 0 : user.getId(), logger); return operationPermissionCheck && resourcePermissionCheck; } @@ -217,6 +222,6 @@ public Map checkAndParseDateParameters(String startDateStr, Stri @Override public boolean checkDescriptionLength(String description) { - return description!=null && description.codePointCount(0, description.length()) > 255; + return description != null && description.codePointCount(0, description.length()) > 255; } } diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ClusterServiceImpl.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ClusterServiceImpl.java index b7d4e01d5e70..cdd40cbc0f6f 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ClusterServiceImpl.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ClusterServiceImpl.java @@ -258,7 +258,8 @@ public Map deleteClusterByCode(User loginUser, Long code) { .selectCount(new QueryWrapper().lambda().eq(K8sNamespace::getClusterCode, code)); if (relatedNamespaceNumber > 0) { - logger.warn("Delete cluster failed because {} namespace(s) is(are) using it, clusterCode:{}.", relatedNamespaceNumber, code); + logger.warn("Delete cluster failed because {} namespace(s) is(are) using it, clusterCode:{}.", + relatedNamespaceNumber, code); putMsg(result, Status.DELETE_CLUSTER_RELATED_NAMESPACE_EXISTS); return result; } @@ -274,7 +275,6 @@ public Map deleteClusterByCode(User loginUser, Long code) { return result; } - /** * update cluster * diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/DataAnalysisServiceImpl.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/DataAnalysisServiceImpl.java index 276e1ddb4266..da6b5c156e8e 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/DataAnalysisServiceImpl.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/DataAnalysisServiceImpl.java @@ -17,8 +17,8 @@ package org.apache.dolphinscheduler.api.service.impl; -import org.apache.commons.lang3.StringUtils; -import org.apache.commons.lang3.tuple.Pair; +import static org.apache.dolphinscheduler.api.constants.ApiFuncIdentificationConstant.PROJECT_OVERVIEW; + import org.apache.dolphinscheduler.api.dto.CommandStateCount; import org.apache.dolphinscheduler.api.dto.DefineUserDto; import org.apache.dolphinscheduler.api.dto.TaskCountDto; @@ -44,10 +44,9 @@ import org.apache.dolphinscheduler.dao.mapper.TaskInstanceMapper; import org.apache.dolphinscheduler.plugin.task.api.enums.TaskExecutionStatus; import org.apache.dolphinscheduler.service.process.ProcessService; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.stereotype.Service; + +import org.apache.commons.lang3.StringUtils; +import org.apache.commons.lang3.tuple.Pair; import java.util.ArrayList; import java.util.Arrays; @@ -60,7 +59,10 @@ import java.util.Set; import java.util.stream.Collectors; -import static org.apache.dolphinscheduler.api.constants.ApiFuncIdentificationConstant.PROJECT_OVERVIEW; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; /** * data analysis service impl diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/DataSourceServiceImpl.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/DataSourceServiceImpl.java index 808f85311da0..6547d83490e1 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/DataSourceServiceImpl.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/DataSourceServiceImpl.java @@ -71,8 +71,6 @@ import com.baomidou.mybatisplus.extension.plugins.pagination.Page; import com.fasterxml.jackson.databind.node.ObjectNode; - - /** * data source service impl */ @@ -87,7 +85,6 @@ public class DataSourceServiceImpl extends BaseServiceImpl implements DataSource @Autowired private DataSourceUserMapper datasourceUserMapper; - private static final String TABLE = "TABLE"; private static final String VIEW = "VIEW"; private static final String[] TABLE_TYPES = new String[]{TABLE, VIEW}; @@ -106,7 +103,8 @@ public class DataSourceServiceImpl extends BaseServiceImpl implements DataSource public Result createDataSource(User loginUser, BaseDataSourceParamDTO datasourceParam) { DataSourceUtils.checkDatasourceParam(datasourceParam); Result result = new Result<>(); - if (!canOperatorPermissions(loginUser,null, AuthorizationType.DATASOURCE, ApiFuncIdentificationConstant.DATASOURCE_CREATE_DATASOURCE)) { + if (!canOperatorPermissions(loginUser, null, AuthorizationType.DATASOURCE, + ApiFuncIdentificationConstant.DATASOURCE_CREATE_DATASOURCE)) { putMsg(result, Status.USER_NO_OPERATION_PERM); return result; } @@ -116,7 +114,7 @@ public Result createDataSource(User loginUser, BaseDataSourceParamDTO da putMsg(result, Status.DATASOURCE_EXIST); return result; } - if(checkDescriptionLength(datasourceParam.getNote())){ + if (checkDescriptionLength(datasourceParam.getNote())) { logger.warn("Parameter description is too long, description:{}.", datasourceParam.getNote()); putMsg(result, Status.DESCRIPTION_TOO_LONG_ERROR); return result; @@ -146,8 +144,10 @@ public Result createDataSource(User loginUser, BaseDataSourceParamDTO da try { dataSourceMapper.insert(dataSource); putMsg(result, Status.SUCCESS); - permissionPostHandle(AuthorizationType.DATASOURCE, loginUser.getId(), Collections.singletonList(dataSource.getId()), logger); - logger.info("Datasource create complete, dbType:{}, datasourceName:{}.", dataSource.getType().getDescp(), dataSource.getName()); + permissionPostHandle(AuthorizationType.DATASOURCE, loginUser.getId(), + Collections.singletonList(dataSource.getId()), logger); + logger.info("Datasource create complete, dbType:{}, datasourceName:{}.", dataSource.getType().getDescp(), + dataSource.getName()); } catch (DuplicateKeyException ex) { logger.error("Datasource create error.", ex); putMsg(result, Status.DATASOURCE_EXIST); @@ -175,24 +175,26 @@ public Result updateDataSource(int id, User loginUser, BaseDataSourcePar return result; } - if (!canOperatorPermissions(loginUser,new Object[]{dataSource.getId()}, AuthorizationType.DATASOURCE, DATASOURCE_UPDATE)) { + if (!canOperatorPermissions(loginUser, new Object[]{dataSource.getId()}, AuthorizationType.DATASOURCE, + DATASOURCE_UPDATE)) { putMsg(result, Status.USER_NO_OPERATION_PERM); return result; } - //check name can use or not + // check name can use or not if (!dataSource.getName().trim().equals(dataSource.getName()) && checkName(dataSource.getName())) { logger.warn("Datasource with the same name already exists, name:{}.", dataSource.getName()); putMsg(result, Status.DATASOURCE_EXIST); return result; } - if(checkDescriptionLength(dataSourceParam.getNote())){ + if (checkDescriptionLength(dataSourceParam.getNote())) { logger.warn("Parameter description is too long, description:{}.", dataSourceParam.getNote()); putMsg(result, Status.DESCRIPTION_TOO_LONG_ERROR); return result; } - //check password，if the password is not updated, set to the old password. - BaseConnectionParam connectionParam = (BaseConnectionParam) DataSourceUtils.buildConnectionParams(dataSourceParam); + // check password，if the password is not updated, set to the old password. + BaseConnectionParam connectionParam = + (BaseConnectionParam) DataSourceUtils.buildConnectionParams(dataSourceParam); String password = connectionParam.getPassword(); if (StringUtils.isBlank(password)) { String oldConnectionParams = dataSource.getConnectionParams(); @@ -213,17 +215,19 @@ public Result updateDataSource(int id, User loginUser, BaseDataSourcePar dataSource.setType(dataSource.getType()); dataSource.setConnectionParams(JSONUtils.toJsonString(connectionParam)); dataSource.setUpdateTime(now); - if(dataSource.getTestFlag() == 1 && dataSourceParam.getTestFlag() == 0){ + if (dataSource.getTestFlag() == 1 && dataSourceParam.getTestFlag() == 0) { clearBindTestId(id); } dataSource.setTestFlag(dataSourceParam.getTestFlag()); dataSource.setBindTestId(dataSourceParam.getBindTestId()); try { dataSourceMapper.updateById(dataSource); - logger.info("Update datasource complete, datasourceId:{}, datasourceName:{}.", dataSource.getId(), dataSource.getName()); + logger.info("Update datasource complete, datasourceId:{}, datasourceName:{}.", dataSource.getId(), + dataSource.getName()); putMsg(result, Status.SUCCESS); } catch (DuplicateKeyException ex) { - logger.error("Update datasource error, datasourceId:{}, datasourceName:{}.", dataSource.getId(), dataSource.getName()); + logger.error("Update datasource error, datasourceId:{}, datasourceName:{}.", dataSource.getId(), + dataSource.getName()); putMsg(result, Status.DATASOURCE_EXIST); } return result; @@ -279,9 +283,11 @@ public Result queryDataSourceListPaging(User loginUser, String searchVal, Intege Page dataSourcePage = new Page<>(pageNo, pageSize); PageInfo pageInfo = new PageInfo<>(pageNo, pageSize); if (loginUser.getUserType().equals(UserType.ADMIN_USER)) { - dataSourceList = dataSourceMapper.selectPaging(dataSourcePage, UserType.ADMIN_USER.equals(loginUser.getUserType()) ? 0 : loginUser.getId(), searchVal); + dataSourceList = dataSourceMapper.selectPaging(dataSourcePage, + UserType.ADMIN_USER.equals(loginUser.getUserType()) ? 0 : loginUser.getId(), searchVal); } else { - Set ids = resourcePermissionCheckService.userOwnedResourceIdsAcquisition(AuthorizationType.DATASOURCE, loginUser.getId(), logger); + Set ids = resourcePermissionCheckService + .userOwnedResourceIdsAcquisition(AuthorizationType.DATASOURCE, loginUser.getId(), logger); if (ids.isEmpty()) { result.setData(pageInfo); putMsg(result, Status.SUCCESS); @@ -335,13 +341,16 @@ public Map queryDataSourceList(User loginUser, Integer type, int if (loginUser.getUserType().equals(UserType.ADMIN_USER)) { datasourceList = dataSourceMapper.queryDataSourceByType(0, type, testFlag); } else { - Set ids = resourcePermissionCheckService.userOwnedResourceIdsAcquisition(AuthorizationType.DATASOURCE, loginUser.getId(), logger); + Set ids = resourcePermissionCheckService + .userOwnedResourceIdsAcquisition(AuthorizationType.DATASOURCE, loginUser.getId(), logger); if (ids.isEmpty()) { result.put(Constants.DATA_LIST, Collections.emptyList()); putMsg(result, Status.SUCCESS); return result; } - datasourceList = dataSourceMapper.selectBatchIds(ids).stream().filter(dataSource -> dataSource.getType().getCode() == type).filter(dataSource -> dataSource.getTestFlag() == testFlag).collect(Collectors.toList()); + datasourceList = dataSourceMapper.selectBatchIds(ids).stream() + .filter(dataSource -> dataSource.getType().getCode() == type) + .filter(dataSource -> dataSource.getTestFlag() == testFlag).collect(Collectors.toList()); } result.put(Constants.DATA_LIST, datasourceList); @@ -360,7 +369,8 @@ public Result verifyDataSourceName(String name) { Result result = new Result<>(); List dataSourceList = dataSourceMapper.queryDataSourceByName(name); if (dataSourceList != null && !dataSourceList.isEmpty()) { - logger.warn("Datasource with the same name already exists, dataSourceName:{}.", dataSourceList.get(0).getName()); + logger.warn("Datasource with the same name already exists, dataSourceName:{}.", + dataSourceList.get(0).getName()); putMsg(result, Status.DATASOURCE_EXIST); } else { putMsg(result, Status.SUCCESS); @@ -382,18 +392,21 @@ public Result checkConnection(DbType type, ConnectionParam connectionPar Result result = new Result<>(); try (Connection connection = DataSourceClientProvider.getInstance().getConnection(type, connectionParam)) { if (connection == null) { - logger.error("Connection test to {} datasource failed, connectionParam:{}.", type.getDescp(), connectionParam); + logger.error("Connection test to {} datasource failed, connectionParam:{}.", type.getDescp(), + connectionParam); putMsg(result, Status.CONNECTION_TEST_FAILURE); return result; } - logger.info("Connection test to {} datasource success, connectionParam:{}", type.getDescp(), connectionParam); + logger.info("Connection test to {} datasource success, connectionParam:{}", type.getDescp(), + connectionParam); putMsg(result, Status.SUCCESS); return result; } catch (Exception e) { String message = Optional.of(e).map(Throwable::getCause) .map(Throwable::getMessage) .orElse(e.getMessage()); - logger.error("Datasource test connection error, dbType:{}, connectionParam:{}, message:{}.", type, connectionParam, message); + logger.error("Datasource test connection error, dbType:{}, connectionParam:{}, message:{}.", type, + connectionParam, message); return new Result<>(Status.CONNECTION_TEST_FAILURE.getCode(), message); } } @@ -413,7 +426,8 @@ public Result connectionTest(int id) { putMsg(result, Status.RESOURCE_NOT_EXIST); return result; } - return checkConnection(dataSource.getType(), DataSourceUtils.buildConnectionParams(dataSource.getType(), dataSource.getConnectionParams())); + return checkConnection(dataSource.getType(), + DataSourceUtils.buildConnectionParams(dataSource.getType(), dataSource.getConnectionParams())); } /** @@ -428,14 +442,15 @@ public Result connectionTest(int id) { public Result delete(User loginUser, int datasourceId) { Result result = new Result<>(); try { - //query datasource by id + // query datasource by id DataSource dataSource = dataSourceMapper.selectById(datasourceId); if (dataSource == null) { logger.warn("Datasource does not exist, datasourceId:{}.", datasourceId); putMsg(result, Status.RESOURCE_NOT_EXIST); return result; } - if (!canOperatorPermissions(loginUser, new Object[]{dataSource.getId()},AuthorizationType.DATASOURCE,DATASOURCE_DELETE)) { + if (!canOperatorPermissions(loginUser, new Object[]{dataSource.getId()}, AuthorizationType.DATASOURCE, + DATASOURCE_DELETE)) { putMsg(result, Status.USER_NO_OPERATION_PERM); return result; } @@ -462,7 +477,7 @@ public Result delete(User loginUser, int datasourceId) { public Map unauthDatasource(User loginUser, Integer userId) { Map result = new HashMap<>(); List datasourceList; - if (canOperatorPermissions(loginUser,null,AuthorizationType.DATASOURCE,null)) { + if (canOperatorPermissions(loginUser, null, AuthorizationType.DATASOURCE, null)) { // admin gets all data sources except userId datasourceList = dataSourceMapper.queryDatasourceExceptUserId(userId); } else { @@ -543,7 +558,7 @@ public Map getTables(Integer datasourceId) { tables = metaData.getTables( connectionParam.getDatabase(), - getDbSchemaPattern(dataSource.getType(),schema,connectionParam), + getDbSchemaPattern(dataSource.getType(), schema, connectionParam), "%", TABLE_TYPES); if (null == tables) { logger.error("Get datasource tables error, datasourceId:{}.", datasourceId); @@ -574,7 +589,7 @@ public Map getTables(Integer datasourceId) { } @Override - public Map getTableColumns(Integer datasourceId,String tableName) { + public Map getTableColumns(Integer datasourceId, String tableName) { Map result = new HashMap<>(); DataSource dataSource = dataSourceMapper.selectById(datasourceId); @@ -640,7 +655,7 @@ private List getParamsOptions(List columnList) { return options; } - private String getDbSchemaPattern(DbType dbType,String schema,BaseConnectionParam connectionParam) { + private String getDbSchemaPattern(DbType dbType, String schema, BaseConnectionParam connectionParam) { if (dbType == null) { return null; } @@ -689,7 +704,7 @@ private static void closeResult(ResultSet rs) { } } } - private void clearBindTestId(Integer bindTestId){ + private void clearBindTestId(Integer bindTestId) { dataSourceMapper.clearBindTestId(bindTestId); } diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/DqExecuteResultServiceImpl.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/DqExecuteResultServiceImpl.java index 417103f9f312..7d16d3d4d1ba 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/DqExecuteResultServiceImpl.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/DqExecuteResultServiceImpl.java @@ -21,7 +21,6 @@ import org.apache.dolphinscheduler.api.service.DqExecuteResultService; import org.apache.dolphinscheduler.api.utils.PageInfo; import org.apache.dolphinscheduler.api.utils.Result; -import org.apache.dolphinscheduler.common.enums.AuthorizationType; import org.apache.dolphinscheduler.common.utils.DateUtils; import org.apache.dolphinscheduler.dao.entity.DqExecuteResult; import org.apache.dolphinscheduler.dao.entity.User; diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/EnvironmentServiceImpl.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/EnvironmentServiceImpl.java index 0f5efcfa3b77..4023843c4bf7 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/EnvironmentServiceImpl.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/EnvironmentServiceImpl.java @@ -17,7 +17,9 @@ package org.apache.dolphinscheduler.api.service.impl; -import static org.apache.dolphinscheduler.api.constants.ApiFuncIdentificationConstant.*; +import static org.apache.dolphinscheduler.api.constants.ApiFuncIdentificationConstant.ENVIRONMENT_CREATE; +import static org.apache.dolphinscheduler.api.constants.ApiFuncIdentificationConstant.ENVIRONMENT_DELETE; +import static org.apache.dolphinscheduler.api.constants.ApiFuncIdentificationConstant.ENVIRONMENT_UPDATE; import org.apache.dolphinscheduler.api.dto.EnvironmentDto; import org.apache.dolphinscheduler.api.enums.Status; @@ -151,7 +153,8 @@ public Map createEnvironment(User loginUser, String name, String relation.setCreateTime(new Date()); relation.setUpdateTime(new Date()); relationMapper.insert(relation); - logger.info("Environment-WorkerGroup relation create complete, environmentName:{}, workerGroup:{}.", + logger.info( + "Environment-WorkerGroup relation create complete, environmentName:{}, workerGroup:{}.", env.getName(), relation.getWorkerGroup()); } }); @@ -159,7 +162,8 @@ public Map createEnvironment(User loginUser, String name, String } result.put(Constants.DATA_LIST, env.getCode()); putMsg(result, Status.SUCCESS); - permissionPostHandle(AuthorizationType.ENVIRONMENT, loginUser.getId(), Collections.singletonList(env.getId()), logger); + permissionPostHandle(AuthorizationType.ENVIRONMENT, loginUser.getId(), + Collections.singletonList(env.getId()), logger); logger.info("Environment create complete, name:{}.", env.getName()); } else { logger.error("Environment create error, name:{}.", env.getName()); @@ -186,7 +190,8 @@ public Result queryEnvironmentListPaging(User loginUser, Integer pageNo, Integer if (loginUser.getUserType().equals(UserType.ADMIN_USER)) { environmentIPage = environmentMapper.queryEnvironmentListPaging(page, searchVal); } else { - Set ids = resourcePermissionCheckService.userOwnedResourceIdsAcquisition(AuthorizationType.ENVIRONMENT, loginUser.getId(), logger); + Set ids = resourcePermissionCheckService + .userOwnedResourceIdsAcquisition(AuthorizationType.ENVIRONMENT, loginUser.getId(), logger); if (ids.isEmpty()) { result.setData(pageInfo); putMsg(result, Status.SUCCESS); @@ -199,12 +204,13 @@ public Result queryEnvironmentListPaging(User loginUser, Integer pageNo, Integer if (CollectionUtils.isNotEmpty(environmentIPage.getRecords())) { Map> relationMap = relationMapper.selectList(null).stream() - .collect(Collectors.groupingBy(EnvironmentWorkerGroupRelation::getEnvironmentCode,Collectors.mapping(EnvironmentWorkerGroupRelation::getWorkerGroup,Collectors.toList()))); + .collect(Collectors.groupingBy(EnvironmentWorkerGroupRelation::getEnvironmentCode, + Collectors.mapping(EnvironmentWorkerGroupRelation::getWorkerGroup, Collectors.toList()))); List dtoList = environmentIPage.getRecords().stream().map(environment -> { EnvironmentDto dto = new EnvironmentDto(); - BeanUtils.copyProperties(environment,dto); - List workerGroups = relationMap.getOrDefault(environment.getCode(),new ArrayList()); + BeanUtils.copyProperties(environment, dto); + List workerGroups = relationMap.getOrDefault(environment.getCode(), new ArrayList()); dto.setWorkerGroups(workerGroups); return dto; }).collect(Collectors.toList()); @@ -227,31 +233,33 @@ public Result queryEnvironmentListPaging(User loginUser, Integer pageNo, Integer */ @Override public Map queryAllEnvironmentList(User loginUser) { - Map result = new HashMap<>(); - Set ids = resourcePermissionCheckService.userOwnedResourceIdsAcquisition(AuthorizationType.ENVIRONMENT, loginUser.getId(), logger); + Map result = new HashMap<>(); + Set ids = resourcePermissionCheckService.userOwnedResourceIdsAcquisition(AuthorizationType.ENVIRONMENT, + loginUser.getId(), logger); if (ids.isEmpty()) { result.put(Constants.DATA_LIST, Collections.emptyList()); - putMsg(result,Status.SUCCESS); + putMsg(result, Status.SUCCESS); return result; } List environmentList = environmentMapper.selectBatchIds(ids); if (CollectionUtils.isNotEmpty(environmentList)) { Map> relationMap = relationMapper.selectList(null).stream() - .collect(Collectors.groupingBy(EnvironmentWorkerGroupRelation::getEnvironmentCode,Collectors.mapping(EnvironmentWorkerGroupRelation::getWorkerGroup,Collectors.toList()))); + .collect(Collectors.groupingBy(EnvironmentWorkerGroupRelation::getEnvironmentCode, + Collectors.mapping(EnvironmentWorkerGroupRelation::getWorkerGroup, Collectors.toList()))); List dtoList = environmentList.stream().map(environment -> { EnvironmentDto dto = new EnvironmentDto(); - BeanUtils.copyProperties(environment,dto); - List workerGroups = relationMap.getOrDefault(environment.getCode(),new ArrayList()); + BeanUtils.copyProperties(environment, dto); + List workerGroups = relationMap.getOrDefault(environment.getCode(), new ArrayList()); dto.setWorkerGroups(workerGroups); return dto; }).collect(Collectors.toList()); - result.put(Constants.DATA_LIST,dtoList); + result.put(Constants.DATA_LIST, dtoList); } else { result.put(Constants.DATA_LIST, new ArrayList<>()); } - putMsg(result,Status.SUCCESS); + putMsg(result, Status.SUCCESS); return result; } @@ -274,7 +282,7 @@ public Map queryEnvironmentByCode(Long code) { .collect(Collectors.toList()); EnvironmentDto dto = new EnvironmentDto(); - BeanUtils.copyProperties(env,dto); + BeanUtils.copyProperties(env, dto); dto.setWorkerGroups(workerGroups); result.put(Constants.DATA_LIST, dto); putMsg(result, Status.SUCCESS); @@ -300,7 +308,7 @@ public Map queryEnvironmentByName(String name) { .collect(Collectors.toList()); EnvironmentDto dto = new EnvironmentDto(); - BeanUtils.copyProperties(env,dto); + BeanUtils.copyProperties(env, dto); dto.setWorkerGroups(workerGroups); result.put(Constants.DATA_LIST, dto); putMsg(result, Status.SUCCESS); @@ -327,7 +335,8 @@ public Map deleteEnvironmentByCode(User loginUser, Long code) { .selectCount(new QueryWrapper().lambda().eq(TaskDefinition::getEnvironmentCode, code)); if (relatedTaskNumber > 0) { - logger.warn("Delete environment failed because {} tasks is using it, environmentCode:{}.", relatedTaskNumber, code); + logger.warn("Delete environment failed because {} tasks is using it, environmentCode:{}.", + relatedTaskNumber, code); putMsg(result, Status.DELETE_ENVIRONMENT_RELATED_TASK_EXISTS); return result; } diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/EnvironmentWorkerGroupRelationServiceImpl.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/EnvironmentWorkerGroupRelationServiceImpl.java index 7fa7104ebf0c..51e8fd458c6e 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/EnvironmentWorkerGroupRelationServiceImpl.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/EnvironmentWorkerGroupRelationServiceImpl.java @@ -36,8 +36,9 @@ * task definition service impl */ @Service -public class EnvironmentWorkerGroupRelationServiceImpl extends BaseServiceImpl implements - EnvironmentWorkerGroupRelationService { +public class EnvironmentWorkerGroupRelationServiceImpl extends BaseServiceImpl + implements + EnvironmentWorkerGroupRelationService { private static final Logger logger = LoggerFactory.getLogger(EnvironmentWorkerGroupRelationServiceImpl.class); @@ -52,7 +53,8 @@ public class EnvironmentWorkerGroupRelationServiceImpl extends BaseServiceImpl i @Override public Map queryEnvironmentWorkerGroupRelation(Long environmentCode) { Map result = new HashMap<>(); - List relations = environmentWorkerGroupRelationMapper.queryByEnvironmentCode(environmentCode); + List relations = + environmentWorkerGroupRelationMapper.queryByEnvironmentCode(environmentCode); result.put(Constants.DATA_LIST, relations); putMsg(result, Status.SUCCESS); return result; @@ -69,8 +71,8 @@ public Map queryAllEnvironmentWorkerGroupRelationList() { List relations = environmentWorkerGroupRelationMapper.selectList(null); - result.put(Constants.DATA_LIST,relations); - putMsg(result,Status.SUCCESS); + result.put(Constants.DATA_LIST, relations); + putMsg(result, Status.SUCCESS); return result; } } diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/FavTaskServiceImpl.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/FavTaskServiceImpl.java index f04bf89d3f3a..21e3737101a0 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/FavTaskServiceImpl.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/FavTaskServiceImpl.java @@ -23,13 +23,15 @@ import org.apache.dolphinscheduler.dao.entity.FavTask; import org.apache.dolphinscheduler.dao.entity.User; import org.apache.dolphinscheduler.dao.mapper.FavTaskMapper; -import org.springframework.stereotype.Service; -import javax.annotation.Resource; import java.util.ArrayList; import java.util.List; import java.util.Set; +import javax.annotation.Resource; + +import org.springframework.stereotype.Service; + @Service public class FavTaskServiceImpl extends BaseServiceImpl implements FavTaskService { diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/K8SNamespaceServiceImpl.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/K8SNamespaceServiceImpl.java index 97d4a67f2be4..7316b3e3375a 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/K8SNamespaceServiceImpl.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/K8SNamespaceServiceImpl.java @@ -31,8 +31,8 @@ import org.apache.dolphinscheduler.dao.mapper.K8sNamespaceMapper; import org.apache.dolphinscheduler.remote.exceptions.RemotingException; -import org.apache.commons.lang3.StringUtils; import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.lang3.StringUtils; import java.util.ArrayList; import java.util.Date; @@ -445,7 +445,7 @@ public List queryNamespaceAvailable(User loginUser) { if (isAdmin(loginUser)) { k8sNamespaces = k8sNamespaceMapper.selectList(null); } else { - k8sNamespaces = k8sNamespaceMapper.queryNamespaceAvailable(loginUser.getId()); + k8sNamespaces = k8sNamespaceMapper.queryNamespaceAvailable(loginUser.getId()); } setClusterName(k8sNamespaces); return k8sNamespaces; diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/LoggerServiceImpl.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/LoggerServiceImpl.java index e3f83f3c1f03..f12fc27cbb52 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/LoggerServiceImpl.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/LoggerServiceImpl.java @@ -17,6 +17,9 @@ package org.apache.dolphinscheduler.api.service.impl; +import static org.apache.dolphinscheduler.api.constants.ApiFuncIdentificationConstant.DOWNLOAD_LOG; +import static org.apache.dolphinscheduler.api.constants.ApiFuncIdentificationConstant.VIEW_LOG; + import org.apache.dolphinscheduler.api.enums.Status; import org.apache.dolphinscheduler.api.exceptions.ServiceException; import org.apache.dolphinscheduler.api.service.LoggerService; @@ -38,10 +41,6 @@ import java.nio.charset.StandardCharsets; import java.util.Map; -import java.util.Objects; - -import javax.annotation.PostConstruct; -import javax.annotation.PreDestroy; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -50,9 +49,6 @@ import com.google.common.primitives.Bytes; -import static org.apache.dolphinscheduler.api.constants.ApiFuncIdentificationConstant.DOWNLOAD_LOG; -import static org.apache.dolphinscheduler.api.constants.ApiFuncIdentificationConstant.VIEW_LOG; - /** * logger service impl */ @@ -107,7 +103,6 @@ public Result queryLog(int taskInstId, int skipLineNum, int lim return result; } - /** * get log size * @@ -137,7 +132,7 @@ public byte[] getLogBytes(int taskInstId) { @SuppressWarnings("unchecked") public Map queryLog(User loginUser, long projectCode, int taskInstId, int skipLineNum, int limit) { Project project = projectMapper.queryByCode(projectCode); - //check user access for project + // check user access for project Map result = projectService.checkProjectAndAuth(loginUser, project, projectCode, VIEW_LOG); if (result.get(Constants.STATUS) != Status.SUCCESS) { return result; @@ -170,8 +165,8 @@ public Map queryLog(User loginUser, long projectCode, int taskIn @Override public byte[] getLogBytes(User loginUser, long projectCode, int taskInstId) { Project project = projectMapper.queryByCode(projectCode); - //check user access for project - Map result = projectService.checkProjectAndAuth(loginUser, project, projectCode,DOWNLOAD_LOG); + // check user access for project + Map result = projectService.checkProjectAndAuth(loginUser, project, projectCode, DOWNLOAD_LOG); if (result.get(Constants.STATUS) != Status.SUCCESS) { throw new ServiceException("user has no permission"); } diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/MonitorServiceImpl.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/MonitorServiceImpl.java index 04debec55504..fb10f49cb1b7 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/MonitorServiceImpl.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/MonitorServiceImpl.java @@ -97,32 +97,30 @@ public Map queryWorker(User loginUser) { Map result = new HashMap<>(); List workerServers = getServerListFromRegistry(false) - .stream() - .map((Server server) -> { - WorkerServerModel model = new WorkerServerModel(); - model.setId(server.getId()); - model.setHost(server.getHost()); - model.setPort(server.getPort()); - model.setZkDirectories(Sets.newHashSet(server.getZkDirectory())); - model.setResInfo(server.getResInfo()); - model.setCreateTime(server.getCreateTime()); - model.setLastHeartbeatTime(server.getLastHeartbeatTime()); - return model; - }) - .collect(Collectors.toList()); + .stream() + .map((Server server) -> { + WorkerServerModel model = new WorkerServerModel(); + model.setId(server.getId()); + model.setHost(server.getHost()); + model.setPort(server.getPort()); + model.setZkDirectories(Sets.newHashSet(server.getZkDirectory())); + model.setResInfo(server.getResInfo()); + model.setCreateTime(server.getCreateTime()); + model.setLastHeartbeatTime(server.getLastHeartbeatTime()); + return model; + }) + .collect(Collectors.toList()); Map workerHostPortServerMapping = workerServers - .stream() - .collect(Collectors.toMap( - (WorkerServerModel worker) -> { - String[] s = worker.getZkDirectories().iterator().next().split("/"); - return s[s.length - 1]; - } - , Function.identity() - , (WorkerServerModel oldOne, WorkerServerModel newOne) -> { - oldOne.getZkDirectories().addAll(newOne.getZkDirectories()); - return oldOne; - })); + .stream() + .collect(Collectors.toMap( + (WorkerServerModel worker) -> { + String[] s = worker.getZkDirectories().iterator().next().split("/"); + return s[s.length - 1]; + }, Function.identity(), (WorkerServerModel oldOne, WorkerServerModel newOne) -> { + oldOne.getZkDirectories().addAll(newOne.getZkDirectories()); + return oldOne; + })); result.put(Constants.DATA_LIST, workerHostPortServerMapping.values()); putMsg(result, Status.SUCCESS); @@ -133,8 +131,8 @@ public Map queryWorker(User loginUser) { @Override public List getServerListFromRegistry(boolean isMaster) { return isMaster - ? registryClient.getServerList(NodeType.MASTER) - : registryClient.getServerList(NodeType.WORKER); + ? registryClient.getServerList(NodeType.MASTER) + : registryClient.getServerList(NodeType.WORKER); } } diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ProcessDefinitionServiceImpl.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ProcessDefinitionServiceImpl.java index 37659730d1bb..17979139d5fa 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ProcessDefinitionServiceImpl.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ProcessDefinitionServiceImpl.java @@ -973,8 +973,7 @@ public Map batchDeleteProcessDefinitionByCodes(User loginUser, l diffCode.stream().map(String::valueOf).collect(Collectors.joining(Constants.COMMA))); throw new ServiceException(Status.BATCH_DELETE_PROCESS_DEFINE_BY_CODES_ERROR, diffCode.stream().map(code -> code + "[process definition not exist]") - .collect(Collectors.joining(Constants.COMMA)) - ); + .collect(Collectors.joining(Constants.COMMA))); } for (ProcessDefinition process : processDefinitionList) { @@ -1003,7 +1002,8 @@ private void processDefinitionUsedInOtherTaskValid(ProcessDefinition processDefi // check process instances is already running List processInstances = processInstanceService - .queryByProcessDefineCodeAndStatus(processDefinition.getCode(), org.apache.dolphinscheduler.service.utils.Constants.NOT_TERMINATED_STATES); + .queryByProcessDefineCodeAndStatus(processDefinition.getCode(), + org.apache.dolphinscheduler.service.utils.Constants.NOT_TERMINATED_STATES); if (CollectionUtils.isNotEmpty(processInstances)) { throw new ServiceException(Status.DELETE_PROCESS_DEFINITION_EXECUTING_FAIL, processInstances.size()); } diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ProcessInstanceServiceImpl.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ProcessInstanceServiceImpl.java index 4f146a8c2f16..b969ea6dcca2 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ProcessInstanceServiceImpl.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ProcessInstanceServiceImpl.java @@ -17,10 +17,20 @@ package org.apache.dolphinscheduler.api.service.impl; -import com.baomidou.mybatisplus.core.metadata.IPage; -import com.baomidou.mybatisplus.core.toolkit.CollectionUtils; -import com.baomidou.mybatisplus.extension.plugins.pagination.Page; -import org.apache.commons.lang3.StringUtils; +import static org.apache.dolphinscheduler.api.constants.ApiFuncIdentificationConstant.INSTANCE_DELETE; +import static org.apache.dolphinscheduler.api.constants.ApiFuncIdentificationConstant.INSTANCE_UPDATE; +import static org.apache.dolphinscheduler.api.constants.ApiFuncIdentificationConstant.WORKFLOW_INSTANCE; +import static org.apache.dolphinscheduler.api.enums.Status.PROCESS_INSTANCE_NOT_EXIST; +import static org.apache.dolphinscheduler.api.enums.Status.PROCESS_INSTANCE_STATE_OPERATION_ERROR; +import static org.apache.dolphinscheduler.common.Constants.DATA_LIST; +import static org.apache.dolphinscheduler.common.Constants.DEPENDENT_SPLIT; +import static org.apache.dolphinscheduler.common.Constants.GLOBAL_PARAMS; +import static org.apache.dolphinscheduler.common.Constants.LOCAL_PARAMS; +import static org.apache.dolphinscheduler.common.Constants.PROCESS_INSTANCE_STATE; +import static org.apache.dolphinscheduler.common.Constants.TASK_LIST; +import static org.apache.dolphinscheduler.plugin.task.api.TaskConstants.TASK_TYPE_DEPENDENT; +import static org.apache.dolphinscheduler.plugin.task.api.TaskConstants.TASK_TYPE_SUB_PROCESS; + import org.apache.dolphinscheduler.api.dto.gantt.GanttDto; import org.apache.dolphinscheduler.api.dto.gantt.Task; import org.apache.dolphinscheduler.api.enums.Status; @@ -40,7 +50,6 @@ import org.apache.dolphinscheduler.common.model.TaskNodeRelation; import org.apache.dolphinscheduler.common.utils.DateUtils; import org.apache.dolphinscheduler.common.utils.JSONUtils; -import org.apache.dolphinscheduler.plugin.task.api.utils.ParameterUtils; import org.apache.dolphinscheduler.common.utils.placeholder.BusinessTimeUtils; import org.apache.dolphinscheduler.dao.entity.ProcessDefinition; import org.apache.dolphinscheduler.dao.entity.ProcessInstance; @@ -66,16 +75,13 @@ import org.apache.dolphinscheduler.plugin.task.api.enums.DependResult; import org.apache.dolphinscheduler.plugin.task.api.model.Property; import org.apache.dolphinscheduler.plugin.task.api.parameters.ParametersNode; +import org.apache.dolphinscheduler.plugin.task.api.utils.ParameterUtils; import org.apache.dolphinscheduler.service.expand.CuringParamsService; import org.apache.dolphinscheduler.service.model.TaskNode; import org.apache.dolphinscheduler.service.process.ProcessService; import org.apache.dolphinscheduler.service.task.TaskPluginManager; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.stereotype.Service; -import org.springframework.transaction.annotation.Transactional; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; +import org.apache.commons.lang3.StringUtils; import java.io.BufferedReader; import java.io.ByteArrayInputStream; @@ -92,20 +98,15 @@ import java.util.function.Function; import java.util.stream.Collectors; -import static org.apache.dolphinscheduler.api.constants.ApiFuncIdentificationConstant.INSTANCE_DELETE; -import static org.apache.dolphinscheduler.api.constants.ApiFuncIdentificationConstant.INSTANCE_UPDATE; -import static org.apache.dolphinscheduler.api.constants.ApiFuncIdentificationConstant.WORKFLOW_INSTANCE; -import static org.apache.dolphinscheduler.api.enums.Status.PROCESS_INSTANCE_NOT_EXIST; -import static org.apache.dolphinscheduler.api.enums.Status.PROCESS_INSTANCE_STATE_OPERATION_ERROR; -import static org.apache.dolphinscheduler.common.Constants.DATA_LIST; -import static org.apache.dolphinscheduler.common.Constants.DEPENDENT_SPLIT; -import static org.apache.dolphinscheduler.common.Constants.GLOBAL_PARAMS; -import static org.apache.dolphinscheduler.common.Constants.LOCAL_PARAMS; -import static org.apache.dolphinscheduler.common.Constants.PROCESS_INSTANCE_STATE; -import static org.apache.dolphinscheduler.common.Constants.TASK_LIST; -import static org.apache.dolphinscheduler.plugin.task.api.TaskConstants.TASK_TYPE_DEPENDENT; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; +import org.springframework.transaction.annotation.Transactional; -import static org.apache.dolphinscheduler.plugin.task.api.TaskConstants.TASK_TYPE_SUB_PROCESS; +import com.baomidou.mybatisplus.core.metadata.IPage; +import com.baomidou.mybatisplus.core.toolkit.CollectionUtils; +import com.baomidou.mybatisplus.extension.plugins.pagination.Page; /** * process instance service impl @@ -348,20 +349,24 @@ public Result queryProcessInstanceList(User loginUser, long projectCode, long pr public Map queryTaskListByProcessId(User loginUser, long projectCode, Integer processId) throws IOException { Project project = projectMapper.queryByCode(projectCode); - //check user access for project - Map result = projectService.checkProjectAndAuth(loginUser, project, projectCode, WORKFLOW_INSTANCE); + // check user access for project + Map result = + projectService.checkProjectAndAuth(loginUser, project, projectCode, WORKFLOW_INSTANCE); if (result.get(Constants.STATUS) != Status.SUCCESS) { return result; } ProcessInstance processInstance = processService.findProcessInstanceDetailById(processId) .orElseThrow(() -> new ServiceException(PROCESS_INSTANCE_NOT_EXIST, processId)); - ProcessDefinition processDefinition = processDefineMapper.queryByCode(processInstance.getProcessDefinitionCode()); + ProcessDefinition processDefinition = + processDefineMapper.queryByCode(processInstance.getProcessDefinitionCode()); if (processDefinition != null && projectCode != processDefinition.getProjectCode()) { - logger.error("Process definition does not exist, projectCode:{}, processDefinitionId:{}.", projectCode, processId); + logger.error("Process definition does not exist, projectCode:{}, processDefinitionId:{}.", projectCode, + processId); putMsg(result, PROCESS_INSTANCE_NOT_EXIST, processId); return result; } - List taskInstanceList = processService.findValidTaskListByProcessId(processId, processInstance.getTestFlag()); + List taskInstanceList = + processService.findValidTaskListByProcessId(processId, processInstance.getTestFlag()); addDependResultForTaskList(taskInstanceList); Map resultMap = new HashMap<>(); resultMap.put(PROCESS_INSTANCE_STATE, processInstance.getState().toString()); @@ -378,7 +383,8 @@ public Map queryTaskListByProcessId(User loginUser, long project private void addDependResultForTaskList(List taskInstanceList) throws IOException { for (TaskInstance taskInstance : taskInstanceList) { if (TASK_TYPE_DEPENDENT.equalsIgnoreCase(taskInstance.getTaskType())) { - logger.info("DEPENDENT type task instance need to set dependent result, taskCode:{}, taskInstanceId:{}", taskInstance.getTaskCode(), taskInstance.getId()); + logger.info("DEPENDENT type task instance need to set dependent result, taskCode:{}, taskInstanceId:{}", + taskInstance.getTaskCode(), taskInstance.getId()); Result logResult = loggerService.queryLog( taskInstance.getId(), Constants.LOG_QUERY_SKIP_LINE_NUMBER, Constants.LOG_QUERY_LIMIT); if (logResult.getCode() == Status.SUCCESS.ordinal()) { @@ -447,13 +453,15 @@ public Map querySubProcessInstanceByTaskId(User loginUser, long TaskDefinition taskDefinition = taskDefinitionMapper.queryByCode(taskInstance.getTaskCode()); if (taskDefinition != null && projectCode != taskDefinition.getProjectCode()) { - logger.error("Task definition does not exist, projectCode:{}, taskDefinitionCode:{}.", projectCode, taskInstance.getTaskCode()); + logger.error("Task definition does not exist, projectCode:{}, taskDefinitionCode:{}.", projectCode, + taskInstance.getTaskCode()); putMsg(result, Status.TASK_INSTANCE_NOT_EXISTS, taskId); return result; } if (!taskInstance.isSubProcess()) { - logger.warn("Task instance is not {} type instance, projectCode:{}, taskInstanceId:{}.", TASK_TYPE_SUB_PROCESS, projectCode, taskId); + logger.warn("Task instance is not {} type instance, projectCode:{}, taskInstanceId:{}.", + TASK_TYPE_SUB_PROCESS, projectCode, taskId); putMsg(result, Status.TASK_INSTANCE_NOT_SUB_WORKFLOW_INSTANCE, taskInstance.getName()); return result; } @@ -461,7 +469,8 @@ public Map querySubProcessInstanceByTaskId(User loginUser, long ProcessInstance subWorkflowInstance = processService.findSubProcessInstance( taskInstance.getProcessInstanceId(), taskInstance.getId()); if (subWorkflowInstance == null) { - logger.error("SubProcess instance does not exist, projectCode:{}, taskInstanceId:{}.", projectCode, taskInstance.getId()); + logger.error("SubProcess instance does not exist, projectCode:{}, taskInstanceId:{}.", projectCode, + taskInstance.getId()); putMsg(result, Status.SUB_PROCESS_INSTANCE_NOT_EXIST, taskId); return result; } @@ -496,8 +505,9 @@ public Map updateProcessInstance(User loginUser, long projectCod String globalParams, String locations, int timeout, String tenantCode) { Project project = projectMapper.queryByCode(projectCode); - //check user access for project - Map result = projectService.checkProjectAndAuth(loginUser, project, projectCode, INSTANCE_UPDATE); + // check user access for project + Map result = + projectService.checkProjectAndAuth(loginUser, project, projectCode, INSTANCE_UPDATE); if (result.get(Constants.STATUS) != Status.SUCCESS) { return result; } @@ -505,15 +515,18 @@ public Map updateProcessInstance(User loginUser, long projectCod ProcessInstance processInstance = processService.findProcessInstanceDetailById(processInstanceId) .orElseThrow(() -> new ServiceException(PROCESS_INSTANCE_NOT_EXIST, processInstanceId)); // check process instance exists in project - ProcessDefinition processDefinition0 = processDefineMapper.queryByCode(processInstance.getProcessDefinitionCode()); + ProcessDefinition processDefinition0 = + processDefineMapper.queryByCode(processInstance.getProcessDefinitionCode()); if (processDefinition0 != null && projectCode != processDefinition0.getProjectCode()) { - logger.error("Process definition does not exist, projectCode:{}, processDefinitionCode:{}.", projectCode, processInstance.getProcessDefinitionCode()); + logger.error("Process definition does not exist, projectCode:{}, processDefinitionCode:{}.", projectCode, + processInstance.getProcessDefinitionCode()); putMsg(result, PROCESS_INSTANCE_NOT_EXIST, processInstanceId); return result; } - //check process instance status + // check process instance status if (!processInstance.getState().isFinished()) { - logger.warn("Process Instance state is {} so can not update process instance, processInstanceId:{}.", processInstance.getState().getDesc(), processInstanceId); + logger.warn("Process Instance state is {} so can not update process instance, processInstanceId:{}.", + processInstance.getState().getDesc(), processInstanceId); putMsg(result, PROCESS_INSTANCE_STATE_OPERATION_ERROR, processInstance.getName(), processInstance.getState().toString(), "update"); return result; @@ -548,7 +561,8 @@ public Map updateProcessInstance(User loginUser, long projectCod } int saveTaskResult = processService.saveTaskDefine(loginUser, projectCode, taskDefinitionLogs, syncDefine); if (saveTaskResult == Constants.DEFINITION_FAILURE) { - logger.error("Update task definition error, projectCode:{}, processInstanceId:{}", projectCode, processInstanceId); + logger.error("Update task definition error, projectCode:{}, processInstanceId:{}", projectCode, + processInstanceId); putMsg(result, Status.UPDATE_TASK_DEFINITION_ERROR); throw new ServiceException(Status.UPDATE_TASK_DEFINITION_ERROR); } @@ -576,20 +590,24 @@ public Map updateProcessInstance(User loginUser, long projectCod processDefinition.setUpdateTime(new Date()); int insertVersion = processService.saveProcessDefine(loginUser, processDefinition, syncDefine, Boolean.FALSE); if (insertVersion == 0) { - logger.error("Update process definition error, projectCode:{}, processDefinitionName:{}.", projectCode, processDefinition.getName()); + logger.error("Update process definition error, projectCode:{}, processDefinitionName:{}.", projectCode, + processDefinition.getName()); putMsg(result, Status.UPDATE_PROCESS_DEFINITION_ERROR); throw new ServiceException(Status.UPDATE_PROCESS_DEFINITION_ERROR); } else - logger.info("Update process definition complete, projectCode:{}, processDefinitionName:{}.", projectCode, processDefinition.getName()); + logger.info("Update process definition complete, projectCode:{}, processDefinitionName:{}.", projectCode, + processDefinition.getName()); int insertResult = processService.saveTaskRelation(loginUser, processDefinition.getProjectCode(), processDefinition.getCode(), insertVersion, taskRelationList, taskDefinitionLogs, syncDefine); if (insertResult == Constants.EXIT_CODE_SUCCESS) { - logger.info("Update task relations complete, projectCode:{}, processDefinitionCode:{}, processDefinitionVersion:{}.", + logger.info( + "Update task relations complete, projectCode:{}, processDefinitionCode:{}, processDefinitionVersion:{}.", projectCode, processDefinition.getCode(), insertVersion); putMsg(result, Status.SUCCESS); result.put(Constants.DATA_LIST, processDefinition); } else { - logger.info("Update task relations error, projectCode:{}, processDefinitionCode:{}, processDefinitionVersion:{}.", + logger.info( + "Update task relations error, projectCode:{}, processDefinitionCode:{}, processDefinitionVersion:{}.", projectCode, processDefinition.getCode(), insertVersion); putMsg(result, Status.UPDATE_PROCESS_DEFINITION_ERROR); throw new ServiceException(Status.UPDATE_PROCESS_DEFINITION_ERROR); @@ -597,12 +615,14 @@ public Map updateProcessInstance(User loginUser, long projectCod processInstance.setProcessDefinitionVersion(insertVersion); int update = processInstanceDao.updateProcessInstance(processInstance); if (update == 0) { - logger.error("Update process instance version error, projectCode:{}, processDefinitionCode:{}, processDefinitionVersion:{}", + logger.error( + "Update process instance version error, projectCode:{}, processDefinitionCode:{}, processDefinitionVersion:{}", projectCode, processDefinition.getCode(), insertVersion); putMsg(result, Status.UPDATE_PROCESS_INSTANCE_ERROR); throw new ServiceException(Status.UPDATE_PROCESS_INSTANCE_ERROR); } - logger.info("Update process instance complete, projectCode:{}, processDefinitionCode:{}, processDefinitionVersion:{}, processInstanceId:{}", + logger.info( + "Update process instance complete, projectCode:{}, processDefinitionCode:{}, processDefinitionVersion:{}, processInstanceId:{}", projectCode, processDefinition.getCode(), insertVersion, processInstanceId); putMsg(result, Status.SUCCESS); return result; @@ -649,14 +669,17 @@ public Map queryParentInstanceBySubId(User loginUser, long proje ProcessInstance subInstance = processService.findProcessInstanceDetailById(subId) .orElseThrow(() -> new ServiceException(PROCESS_INSTANCE_NOT_EXIST, subId)); if (subInstance.getIsSubProcess() == Flag.NO) { - logger.warn("Process instance is not sub process instance type, processInstanceId:{}, processInstanceName:{}.", subId, subInstance.getName()); + logger.warn( + "Process instance is not sub process instance type, processInstanceId:{}, processInstanceName:{}.", + subId, subInstance.getName()); putMsg(result, Status.PROCESS_INSTANCE_NOT_SUB_PROCESS_INSTANCE, subInstance.getName()); return result; } ProcessInstance parentWorkflowInstance = processService.findParentProcessInstance(subId); if (parentWorkflowInstance == null) { - logger.error("Parent process instance does not exist, projectCode:{}, subProcessInstanceId:{}.", projectCode, subId); + logger.error("Parent process instance does not exist, projectCode:{}, subProcessInstanceId:{}.", + projectCode, subId); putMsg(result, Status.SUB_PROCESS_INSTANCE_NOT_EXIST); return result; } @@ -691,7 +714,8 @@ public Map deleteProcessInstanceById(User loginUser, long projec if (!processInstance.getState().isFinished()) { logger.warn("Process Instance state is {} so can not delete process instance, processInstanceId:{}.", processInstance.getState().getDesc(), processInstanceId); - throw new ServiceException(PROCESS_INSTANCE_STATE_OPERATION_ERROR, processInstance.getName(), processInstance.getState(), "delete"); + throw new ServiceException(PROCESS_INSTANCE_STATE_OPERATION_ERROR, processInstance.getName(), + processInstance.getState(), "delete"); } ProcessDefinition processDefinition = @@ -706,7 +730,8 @@ public Map deleteProcessInstanceById(User loginUser, long projec processService.removeTaskLogFile(processInstanceId); } catch (Exception ignore) { // ignore - logger.warn("Remove task log file exception, projectCode:{}, ProcessDefinitionCode{}, processInstanceId:{}.", + logger.warn( + "Remove task log file exception, projectCode:{}, ProcessDefinitionCode{}, processInstanceId:{}.", projectCode, processInstance.getProcessDefinitionCode(), processInstanceId); } @@ -718,11 +743,13 @@ public Map deleteProcessInstanceById(User loginUser, long projec processService.deleteWorkTaskInstanceByProcessInstanceId(processInstanceId); if (delete > 0) { - logger.info("Delete process instance complete, projectCode:{}, ProcessDefinitionCode{}, processInstanceId:{}.", + logger.info( + "Delete process instance complete, projectCode:{}, ProcessDefinitionCode{}, processInstanceId:{}.", projectCode, processInstance.getProcessDefinitionCode(), processInstanceId); putMsg(result, Status.SUCCESS); } else { - logger.error("Delete process instance error, projectCode:{}, ProcessDefinitionCode{}, processInstanceId:{}.", + logger.error( + "Delete process instance error, projectCode:{}, ProcessDefinitionCode{}, processInstanceId:{}.", projectCode, processInstance.getProcessDefinitionCode(), processInstanceId); putMsg(result, Status.DELETE_PROCESS_INSTANCE_BY_ID_ERROR); throw new ServiceException(Status.DELETE_PROCESS_INSTANCE_BY_ID_ERROR); @@ -745,7 +772,8 @@ public Map viewVariables(long projectCode, Integer processInstan ProcessInstance processInstance = processInstanceMapper.queryDetailById(processInstanceId); if (processInstance == null) { - logger.error("Process instance does not exist, projectCode:{}, processInstanceId:{}.", projectCode, processInstanceId); + logger.error("Process instance does not exist, projectCode:{}, processInstanceId:{}.", projectCode, + processInstanceId); putMsg(result, Status.PROCESS_INSTANCE_NOT_EXIST, processInstanceId); return result; } @@ -753,7 +781,8 @@ public Map viewVariables(long projectCode, Integer processInstan ProcessDefinition processDefinition = processDefineMapper.queryByCode(processInstance.getProcessDefinitionCode()); if (processDefinition != null && projectCode != processDefinition.getProjectCode()) { - logger.error("Process definition does not exist, projectCode:{}, processDefinitionCode:{}.", projectCode, processInstance.getProcessDefinitionCode()); + logger.error("Process definition does not exist, projectCode:{}, processDefinitionCode:{}.", projectCode, + processInstance.getProcessDefinitionCode()); putMsg(result, PROCESS_INSTANCE_NOT_EXIST, processInstanceId); return result; } @@ -801,7 +830,8 @@ private Map> getLocalParams(ProcessInstance processI Map timeParams) { Map> localUserDefParams = new HashMap<>(); List taskInstanceList = - taskInstanceMapper.findValidTaskListByProcessId(processInstance.getId(), Flag.YES, processInstance.getTestFlag()); + taskInstanceMapper.findValidTaskListByProcessId(processInstance.getId(), Flag.YES, + processInstance.getTestFlag()); for (TaskInstance taskInstance : taskInstanceList) { TaskDefinitionLog taskDefinitionLog = taskDefinitionLogMapper.queryByDefinitionCodeAndVersion( taskInstance.getTaskCode(), taskInstance.getTaskDefinitionVersion()); @@ -836,7 +866,8 @@ public Map viewGantt(long projectCode, Integer processInstanceId ProcessInstance processInstance = processInstanceMapper.queryDetailById(processInstanceId); if (processInstance == null) { - logger.error("Process instance does not exist, projectCode:{}, processInstanceId:{}.", projectCode, processInstanceId); + logger.error("Process instance does not exist, projectCode:{}, processInstanceId:{}.", projectCode, + processInstanceId); putMsg(result, Status.PROCESS_INSTANCE_NOT_EXIST, processInstanceId); return result; } @@ -845,7 +876,8 @@ public Map viewGantt(long projectCode, Integer processInstanceId processInstance.getProcessDefinitionCode(), processInstance.getProcessDefinitionVersion()); if (processDefinition == null || projectCode != processDefinition.getProjectCode()) { - logger.error("Process definition does not exist, projectCode:{}, processDefinitionCode:{}.", projectCode, processInstance.getProcessDefinitionCode()); + logger.error("Process definition does not exist, projectCode:{}, processDefinitionCode:{}.", projectCode, + processInstance.getProcessDefinitionCode()); putMsg(result, PROCESS_INSTANCE_NOT_EXIST, processInstanceId); return result; } diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ProjectServiceImpl.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ProjectServiceImpl.java index bd43e549b4af..401ec6abd239 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ProjectServiceImpl.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ProjectServiceImpl.java @@ -189,7 +189,7 @@ public Result queryByCode(User loginUser, long projectCode) { public Map queryByName(User loginUser, String projectName) { Map result = new HashMap<>(); Project project = projectMapper.queryByName(projectName); - boolean hasProjectAndPerm = hasProjectAndPerm(loginUser, project, result,PROJECT); + boolean hasProjectAndPerm = hasProjectAndPerm(loginUser, project, result, PROJECT); if (!hasProjectAndPerm) { return result; } @@ -218,7 +218,8 @@ public Map checkProjectAndAuth(User loginUser, Project project, } else if (!canOperatorPermissions(loginUser, new Object[]{project.getId()}, AuthorizationType.PROJECTS, permission)) { // check read permission - logger.error("User does not have {} permission to operate project, userName:{}, projectCode:{}.", permission, loginUser.getUserName(), projectCode); + logger.error("User does not have {} permission to operate project, userName:{}, projectCode:{}.", + permission, loginUser.getUserName(), projectCode); putMsg(result, Status.USER_NO_OPERATION_PROJECT_PERM, loginUser.getUserName(), projectCode); } else { putMsg(result, Status.SUCCESS); @@ -244,9 +245,10 @@ public boolean hasProjectAndPerm(User loginUser, Project project, Map getCheckResult(User loginUser, Project project,String perm) { - Map checkResult = checkProjectAndAuth(loginUser, project, project == null ? 0L : project.getCode(),perm); + private Map getCheckResult(User loginUser, Project project, String perm) { + Map checkResult = + checkProjectAndAuth(loginUser, project, project == null ? 0L : project.getCode(), perm); Status status = (Status) checkResult.get(Constants.STATUS); if (status != Status.SUCCESS) { return checkResult; @@ -413,7 +417,6 @@ public Result update(User loginUser, Long projectCode, String projectName, Strin return result; } - /** * query unauthorized project * @@ -425,13 +428,16 @@ public Result update(User loginUser, Long projectCode, String projectName, Strin public Result queryUnauthorizedProject(User loginUser, Integer userId) { Result result = new Result(); - Set projectIds = resourcePermissionCheckService.userOwnedResourceIdsAcquisition(AuthorizationType.PROJECTS, loginUser.getId(), logger); + Set projectIds = resourcePermissionCheckService + .userOwnedResourceIdsAcquisition(AuthorizationType.PROJECTS, loginUser.getId(), logger); if (projectIds.isEmpty()) { result.setData(Collections.emptyList()); putMsg(result, Status.SUCCESS); return result; } - List projectList = projectMapper.listAuthorizedProjects(loginUser.getUserType().equals(UserType.ADMIN_USER) ? 0 : loginUser.getId(), new ArrayList<>(projectIds)); + List projectList = projectMapper.listAuthorizedProjects( + loginUser.getUserType().equals(UserType.ADMIN_USER) ? 0 : loginUser.getId(), + new ArrayList<>(projectIds)); List resultList = new ArrayList<>(); Set projectSet; @@ -535,7 +541,8 @@ public Map queryProjectCreatedByUser(User loginUser) { public Result queryProjectCreatedAndAuthorizedByUser(User loginUser) { Result result = new Result(); - Set projectIds = resourcePermissionCheckService.userOwnedResourceIdsAcquisition(AuthorizationType.PROJECTS, loginUser.getId(), logger); + Set projectIds = resourcePermissionCheckService + .userOwnedResourceIdsAcquisition(AuthorizationType.PROJECTS, loginUser.getId(), logger); if (projectIds.isEmpty()) { result.setData(Collections.emptyList()); putMsg(result, Status.SUCCESS); @@ -595,7 +602,8 @@ private int queryPermission(User user, Project project) { @Override public Result queryAllProjectList(User user) { Result result = new Result(); - List projects = projectMapper.queryAllProject(user.getUserType() == UserType.ADMIN_USER ? 0 : user.getId()); + List projects = + projectMapper.queryAllProject(user.getUserType() == UserType.ADMIN_USER ? 0 : user.getId()); result.setData(projects); putMsg(result, Status.SUCCESS); diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/QueueServiceImpl.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/QueueServiceImpl.java index 0bb8bd428439..bfba4d08d739 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/QueueServiceImpl.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/QueueServiceImpl.java @@ -37,10 +37,8 @@ import java.util.ArrayList; import java.util.Collections; -import java.util.HashMap; import java.util.HashSet; import java.util.List; -import java.util.Map; import java.util.Objects; import java.util.Set; @@ -91,7 +89,8 @@ private void createQueueValid(Queue queue) throws ServiceException { * @param updateQueue The queue object want to update */ private void updateQueueValid(Queue existsQueue, Queue updateQueue) throws ServiceException { - // Check the exists queue and the necessary of update operation, in not exist checker have to use updateQueue to avoid NPE + // Check the exists queue and the necessary of update operation, in not exist checker have to use updateQueue to + // avoid NPE if (Objects.isNull(existsQueue)) { throw new ServiceException(Status.QUEUE_NOT_EXIST, updateQueue.getQueue()); } else if (Objects.equals(existsQueue, updateQueue)) { @@ -102,9 +101,11 @@ else if (StringUtils.isEmpty(updateQueue.getQueue())) { throw new ServiceException(Status.REQUEST_PARAMS_NOT_VALID_ERROR, Constants.QUEUE); } else if (StringUtils.isEmpty(updateQueue.getQueueName())) { throw new ServiceException(Status.REQUEST_PARAMS_NOT_VALID_ERROR, Constants.QUEUE_NAME); - } else if (!Objects.equals(updateQueue.getQueue(), existsQueue.getQueue()) && checkQueueExist(updateQueue.getQueue())) { + } else if (!Objects.equals(updateQueue.getQueue(), existsQueue.getQueue()) + && checkQueueExist(updateQueue.getQueue())) { throw new ServiceException(Status.QUEUE_VALUE_EXIST, updateQueue.getQueue()); - } else if (!Objects.equals(updateQueue.getQueueName(), existsQueue.getQueueName()) && checkQueueNameExist(updateQueue.getQueueName())) { + } else if (!Objects.equals(updateQueue.getQueueName(), existsQueue.getQueueName()) + && checkQueueNameExist(updateQueue.getQueueName())) { throw new ServiceException(Status.QUEUE_NAME_EXIST, updateQueue.getQueueName()); } } @@ -118,7 +119,8 @@ else if (StringUtils.isEmpty(updateQueue.getQueue())) { @Override public Result queryList(User loginUser) { Result result = new Result(); - Set ids = resourcePermissionCheckService.userOwnedResourceIdsAcquisition(AuthorizationType.QUEUE, loginUser.getId(), logger); + Set ids = resourcePermissionCheckService.userOwnedResourceIdsAcquisition(AuthorizationType.QUEUE, + loginUser.getId(), logger); if (loginUser.getUserType().equals(UserType.GENERAL_USER)) { ids = ids.isEmpty() ? new HashSet<>() : ids; ids.add(Constants.DEFAULT_QUEUE_ID); @@ -142,7 +144,8 @@ public Result queryList(User loginUser) { public Result queryList(User loginUser, String searchVal, Integer pageNo, Integer pageSize) { Result result = new Result(); PageInfo pageInfo = new PageInfo<>(pageNo, pageSize); - Set ids = resourcePermissionCheckService.userOwnedResourceIdsAcquisition(AuthorizationType.QUEUE, loginUser.getId(), logger); + Set ids = resourcePermissionCheckService.userOwnedResourceIdsAcquisition(AuthorizationType.QUEUE, + loginUser.getId(), logger); if (ids.isEmpty()) { result.setData(pageInfo); putMsg(result, Status.SUCCESS); @@ -171,7 +174,7 @@ public Result queryList(User loginUser, String searchVal, Integer pageNo, Intege @Transactional public Result createQueue(User loginUser, String queue, String queueName) { Result result = new Result(); - if (!canOperatorPermissions(loginUser,null, AuthorizationType.QUEUE,YARN_QUEUE_CREATE)) { + if (!canOperatorPermissions(loginUser, null, AuthorizationType.QUEUE, YARN_QUEUE_CREATE)) { throw new ServiceException(Status.USER_NO_OPERATION_PERM); } @@ -182,7 +185,8 @@ public Result createQueue(User loginUser, String queue, String queueName) { result.setData(queueObj); logger.info("Queue create complete, queueName:{}.", queueObj.getQueueName()); putMsg(result, Status.SUCCESS); - permissionPostHandle(AuthorizationType.QUEUE, loginUser.getId(), Collections.singletonList(queueObj.getId()), logger); + permissionPostHandle(AuthorizationType.QUEUE, loginUser.getId(), Collections.singletonList(queueObj.getId()), + logger); return result; } @@ -198,7 +202,7 @@ public Result createQueue(User loginUser, String queue, String queueName) { @Override public Result updateQueue(User loginUser, int id, String queue, String queueName) { Result result = new Result(); - if (!canOperatorPermissions(loginUser,new Object[]{id}, AuthorizationType.QUEUE,YARN_QUEUE_UPDATE)) { + if (!canOperatorPermissions(loginUser, new Object[]{id}, AuthorizationType.QUEUE, YARN_QUEUE_UPDATE)) { throw new ServiceException(Status.USER_NO_OPERATION_PERM); } @@ -208,8 +212,9 @@ public Result updateQueue(User loginUser, int id, String queue, String queueName // check old queue using by any user if (checkIfQueueIsInUsing(existsQueue.getQueueName(), updateQueue.getQueueName())) { - //update user related old queue - Integer relatedUserNums = userMapper.updateUserQueue(existsQueue.getQueueName(), updateQueue.getQueueName()); + // update user related old queue + Integer relatedUserNums = + userMapper.updateUserQueue(existsQueue.getQueueName(), updateQueue.getQueueName()); logger.info("Old queue have related {} users, exec update user success.", relatedUserNums); } diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ResourcesServiceImpl.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ResourcesServiceImpl.java index abb8135ad4dc..55e7eaaab171 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ResourcesServiceImpl.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ResourcesServiceImpl.java @@ -279,7 +279,8 @@ public Result createResource(User loginUser, return result; } if (fullName.length() > Constants.RESOURCE_FULL_NAME_MAX_LENGTH) { - logger.warn("Resource file's name is longer than max full name length, fullName:{}, fullNameSize:{}, maxFullNameSize:{}", + logger.warn( + "Resource file's name is longer than max full name length, fullName:{}, fullNameSize:{}, maxFullNameSize:{}", RegexUtils.escapeNRT(name), fullName.length(), Constants.RESOURCE_FULL_NAME_MAX_LENGTH); putMsg(result, Status.RESOURCE_FULL_NAME_TOO_LONG_ERROR); return result; @@ -311,7 +312,8 @@ public Result createResource(User loginUser, logger.error("Upload resource file failed, resourceName:{}, fileName:{}.", RegexUtils.escapeNRT(name), RegexUtils.escapeNRT(file.getOriginalFilename())); putMsg(result, Status.STORE_OPERATE_CREATE_ERROR); - throw new ServiceException(String.format("upload resource: %s file: %s failed.", name, file.getOriginalFilename())); + throw new ServiceException( + String.format("upload resource: %s file: %s failed.", name, file.getOriginalFilename())); } else logger.info("Upload resource file complete, resourceName:{}, fileName:{}.", RegexUtils.escapeNRT(name), RegexUtils.escapeNRT(file.getOriginalFilename())); @@ -340,7 +342,8 @@ private void updateParentResourceSize(Resource resource, long size) { parentResource.setSize(0L); } resourcesMapper.updateById(parentResource); - logger.info("Resource size update complete, resourceFullName:{}, newSize:{}.", parentResource.getFullName(), parentResource.getSize()); + logger.info("Resource size update complete, resourceFullName:{}, newSize:{}.", + parentResource.getFullName(), parentResource.getSize()); } } } @@ -405,7 +408,8 @@ public Result updateResource(User loginUser, } if (!PropertyUtils.getResUploadStartupState()) { - logger.error("Storage does not start up, resource upload startup state: {}.", PropertyUtils.getResUploadStartupState()); + logger.error("Storage does not start up, resource upload startup state: {}.", + PropertyUtils.getResUploadStartupState()); putMsg(result, Status.STORAGE_NOT_STARTUP); return result; } @@ -452,7 +456,8 @@ public Result updateResource(User loginUser, String originFileName = storageOperate.getFileName(resource.getType(), tenantCode, originFullName); try { if (!storageOperate.exists(tenantCode, originFileName)) { - logger.error("Resource file does not exist in {} storage, tenantCode:{}, resourceId:{}, originFileName:{}.", + logger.error( + "Resource file does not exist in {} storage, tenantCode:{}, resourceId:{}, originFileName:{}.", resource.getType(), tenantCode, resourceId, originFileName); putMsg(result, Status.RESOURCE_NOT_EXIST); return result; @@ -613,7 +618,8 @@ private Result verifyFile(String name, ResourceType type, MultipartFile } if (file != null && FileUtils.directoryTraversal(Objects.requireNonNull(file.getOriginalFilename()))) { - logger.warn("File original name verify failed, fileOriginalName:{}.", RegexUtils.escapeNRT(file.getOriginalFilename())); + logger.warn("File original name verify failed, fileOriginalName:{}.", + RegexUtils.escapeNRT(file.getOriginalFilename())); putMsg(result, Status.VERIFY_PARAMETER_NAME_FAILED); return result; } @@ -621,7 +627,8 @@ private Result verifyFile(String name, ResourceType type, MultipartFile if (file != null) { // file is empty if (file.isEmpty()) { - logger.warn("Parameter file is empty, fileOriginalName:{}.", RegexUtils.escapeNRT(file.getOriginalFilename())); + logger.warn("Parameter file is empty, fileOriginalName:{}.", + RegexUtils.escapeNRT(file.getOriginalFilename())); putMsg(result, Status.RESOURCE_FILE_IS_EMPTY); return result; } @@ -646,7 +653,8 @@ private Result verifyFile(String name, ResourceType type, MultipartFile return result; } if (file.getSize() > Constants.MAX_FILE_SIZE) { - logger.warn("Resource file size is larger than max file size, fileOriginalName:{}, fileSize:{}, maxFileSize:{}.", + logger.warn( + "Resource file size is larger than max file size, fileOriginalName:{}, fileSize:{}, maxFileSize:{}.", RegexUtils.escapeNRT(file.getOriginalFilename()), file.getSize(), Constants.MAX_FILE_SIZE); putMsg(result, Status.RESOURCE_SIZE_EXCEED_LIMIT); return result; @@ -876,7 +884,8 @@ public Result delete(User loginUser, int resourceId) throws IOException Integer[] needDeleteResourceIdArray = allChildren.toArray(new Integer[allChildren.size()]); if (needDeleteResourceIdArray.length >= 2) { - logger.warn("Resource can not be deleted because there are files or folders {} in the current directory.", resource.getFileName()); + logger.warn("Resource can not be deleted because there are files or folders {} in the current directory.", + resource.getFileName()); putMsg(result, Status.RESOURCE_HAS_FOLDER, resource.getFileName()); return result; } @@ -886,14 +895,16 @@ public Result delete(User loginUser, int resourceId) throws IOException List udfFuncs = udfFunctionMapper.listUdfByResourceId(needDeleteResourceIdArray); List udfFuncIds = udfFuncs.stream().map(UdfFunc::getId).collect(Collectors.toList()); if (CollectionUtils.isNotEmpty(udfFuncs)) { - logger.warn("Resource can not be deleted because it is bound by UDF functions, udfFuncIds:{}", udfFuncIds); + logger.warn("Resource can not be deleted because it is bound by UDF functions, udfFuncIds:{}", + udfFuncIds); putMsg(result, Status.UDF_RESOURCE_IS_BOUND, udfFuncs.get(0).getFuncName()); return result; } } if (resourceIdSet.contains(resource.getPid())) { - logger.warn("Resource can not be deleted because it is used by process definition, resourceId:{}, processDefinitionCode:{}.", + logger.warn( + "Resource can not be deleted because it is used by process definition, resourceId:{}, processDefinitionCode:{}.", resource.getId(), resource.getPid()); putMsg(result, Status.RESOURCE_IS_USED); return result; @@ -901,7 +912,8 @@ public Result delete(User loginUser, int resourceId) throws IOException resourceIdSet.retainAll(allChildren); if (CollectionUtils.isNotEmpty(resourceIdSet)) { for (Integer resId : resourceIdSet) { - logger.warn("Resource can not be deleted because it is used by process definition, resourceId:{}, processDefinitionCode:{}.", + logger.warn( + "Resource can not be deleted because it is used by process definition, resourceId:{}, processDefinitionCode:{}.", resId, resourceProcessMap.get(resId)); } putMsg(result, Status.RESOURCE_IS_USED); @@ -959,7 +971,8 @@ public Result verifyResourceName(String fullName, ResourceType type, Use try { String filename = storageOperate.getFileName(type, tenantCode, fullName); if (storageOperate.exists(tenantCode, filename)) { - logger.warn("Resource file with same name exists so can not create again, tenantCode:{}, resourceName:{}.", + logger.warn( + "Resource file with same name exists so can not create again, tenantCode:{}, resourceName:{}.", tenantCode, RegexUtils.escapeNRT(filename)); putMsg(result, Status.RESOURCE_FILE_EXIST, filename); } @@ -969,7 +982,9 @@ public Result verifyResourceName(String fullName, ResourceType type, Use putMsg(result, Status.STORE_OPERATE_CREATE_ERROR); } } else { - logger.error("Tenant does not exist or tenant of current login user does not specified, loginUserName:{}.", loginUser.getUserName()); + logger.error( + "Tenant does not exist or tenant of current login user does not specified, loginUserName:{}.", + loginUser.getUserName()); putMsg(result, Status.CURRENT_LOGIN_USER_TENANT_NOT_EXIST); } } @@ -1114,7 +1129,8 @@ public Result readResource(User loginUser, int resourceId, int skipLineN if (storageOperate.exists(tenantCode, resourceFileName)) { List content = storageOperate.vimFile(tenantCode, resourceFileName, skipLineNum, limit); - logger.info("Vim file content in path {} success, tenantCode:{}, fileName:{}, skipLineNum:{}, limit:{}.", + logger.info( + "Vim file content in path {} success, tenantCode:{}, fileName:{}, skipLineNum:{}, limit:{}.", resourceFileName, tenantCode, resourceFileName, skipLineNum, limit); putMsg(result, Status.SUCCESS); Map map = new HashMap<>(); @@ -1325,7 +1341,8 @@ private Result checkResourceUploadStartupState() { putMsg(result, Status.SUCCESS); // if resource upload startup if (!PropertyUtils.getResUploadStartupState()) { - logger.error("Storage does not start up, resource upload startup state: {}.", PropertyUtils.getResUploadStartupState()); + logger.error("Storage does not start up, resource upload startup state: {}.", + PropertyUtils.getResUploadStartupState()); putMsg(result, Status.STORAGE_NOT_STARTUP); return result; } @@ -1437,7 +1454,8 @@ private Result uploadContentToStorage(User loginUser, String resourceNam if (!FileUtils.writeContent2File(content, localFilename)) { // write file fail - logger.error("Write file error, fileName:{}, content:{}.", localFilename, RegexUtils.escapeNRT(content)); + logger.error("Write file error, fileName:{}, content:{}.", localFilename, + RegexUtils.escapeNRT(content)); putMsg(result, Status.RESOURCE_NOT_EXIST); return result; } @@ -1450,7 +1468,8 @@ private Result uploadContentToStorage(User loginUser, String resourceNam if (!storageOperate.exists(tenantCode, resourcePath)) { // create if tenant dir not exists storageOperate.createTenantDirIfNotExists(tenantCode); - logger.info("Create tenant dir because path {} does not exist, tenantCode:{}.", resourcePath, tenantCode); + logger.info("Create tenant dir because path {} does not exist, tenantCode:{}.", resourcePath, + tenantCode); } if (storageOperate.exists(tenantCode, storageFileName)) { storageOperate.delete(tenantCode, storageFileName, false); @@ -1458,7 +1477,8 @@ private Result uploadContentToStorage(User loginUser, String resourceNam storageOperate.upload(tenantCode, localFilename, storageFileName, true, true); } catch (Exception e) { - logger.error("Upload content to storage error, tenantCode:{}, destFileName:{}.", tenantCode, storageFileName, e); + logger.error("Upload content to storage error, tenantCode:{}, destFileName:{}.", tenantCode, + storageFileName, e); result.setCode(Status.HDFS_OPERATION_ERROR.getCode()); result.setMsg(String.format("copy %s to hdfs %s fail", localFilename, storageFileName)); return result; @@ -1479,7 +1499,8 @@ private Result uploadContentToStorage(User loginUser, String resourceNam public org.springframework.core.io.Resource downloadResource(User loginUser, int resourceId) throws IOException { // if resource upload startup if (!PropertyUtils.getResUploadStartupState()) { - logger.warn("Storage does not start up, resource upload startup state: {}.", PropertyUtils.getResUploadStartupState()); + logger.warn("Storage does not start up, resource upload startup state: {}.", + PropertyUtils.getResUploadStartupState()); throw new ServiceException("hdfs not startup"); } @@ -1514,7 +1535,8 @@ public org.springframework.core.io.Resource downloadResource(User loginUser, int Tenant tenant = tenantMapper.queryById(user.getTenantId()); if (tenant == null) { logger.error("Tenant does not exists, tenantId:{}.", user.getTenantId()); - throw new ServiceException(String.format("The tenant id %d of resource owner does not exist", user.getTenantId())); + throw new ServiceException( + String.format("The tenant id %d of resource owner does not exist", user.getTenantId())); } String tenantCode = tenant.getTenantCode(); @@ -1526,7 +1548,8 @@ public org.springframework.core.io.Resource downloadResource(User loginUser, int try { storageOperate.download(tenantCode, fileName, localFileName, false, true); - org.springframework.core.io.Resource file2Resource = org.apache.dolphinscheduler.api.utils.FileUtils.file2Resource(localFileName); + org.springframework.core.io.Resource file2Resource = + org.apache.dolphinscheduler.api.utils.FileUtils.file2Resource(localFileName); if (file2Resource != null) { logger.info("Download resource complete, path:{}, localFileName:{}.", fileName, localFileName); } @@ -1578,7 +1601,8 @@ public Resource queryResourcesFileInfo(String userName, String fullName) { User user = userMapper.queryByUserNameAccurately(userName); Result resourceResponse = this.queryResource(user, fullName, null, ResourceType.FILE); if (resourceResponse.getCode() != Status.SUCCESS.getCode()) { - String msg = String.format("Query resource by fullName failed, userName:%s, fullName:%s", userName, fullName); + String msg = + String.format("Query resource by fullName failed, userName:%s, fullName:%s", userName, fullName); logger.error(msg); throw new IllegalArgumentException(msg); } diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/SessionServiceImpl.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/SessionServiceImpl.java index 6734a219901f..c8439ca6b221 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/SessionServiceImpl.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/SessionServiceImpl.java @@ -153,7 +153,7 @@ public void signOut(String ip, User loginUser) { */ Session session = sessionMapper.queryByUserIdAndIp(loginUser.getId(), ip); - //delete session + // delete session sessionMapper.deleteById(session.getId()); } catch (Exception e) { logger.warn("userId : {} , ip : {} , find more one session", loginUser.getId(), ip, e); diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/TaskGroupQueueServiceImpl.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/TaskGroupQueueServiceImpl.java index c9c1eba5afb2..c96996e3bc7b 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/TaskGroupQueueServiceImpl.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/TaskGroupQueueServiceImpl.java @@ -65,20 +65,21 @@ public class TaskGroupQueueServiceImpl extends BaseServiceImpl implements TaskGr * @return tasks list */ @Override - public Map queryTasksByGroupId(User loginUser, String taskName - , String processName, Integer status, int groupId, int pageNo, int pageSize) { + public Map queryTasksByGroupId(User loginUser, String taskName, String processName, Integer status, + int groupId, int pageNo, int pageSize) { Map result = new HashMap<>(); Page page = new Page<>(pageNo, pageSize); PageInfo pageInfo = new PageInfo<>(pageNo, pageSize); - Set projectIds = resourcePermissionCheckService.userOwnedResourceIdsAcquisition(AuthorizationType.PROJECTS, loginUser.getId(), logger); + Set projectIds = resourcePermissionCheckService + .userOwnedResourceIdsAcquisition(AuthorizationType.PROJECTS, loginUser.getId(), logger); if (projectIds.isEmpty()) { result.put(Constants.DATA_LIST, pageInfo); putMsg(result, Status.SUCCESS); return result; } List projects = projectMapper.selectBatchIds(projectIds); - IPage taskGroupQueue = taskGroupQueueMapper.queryTaskGroupQueueByTaskGroupIdPaging(page, taskName - ,processName,status,groupId,projects); + IPage taskGroupQueue = taskGroupQueueMapper.queryTaskGroupQueueByTaskGroupIdPaging(page, + taskName, processName, status, groupId, projects); pageInfo.setTotal((int) taskGroupQueue.getTotal()); pageInfo.setTotalList(taskGroupQueue.getRecords()); @@ -99,7 +100,7 @@ public Map queryTasksByGroupId(User loginUser, String taskName */ @Override public Map queryTasksByProcessId(User loginUser, int pageNo, int pageSize, int processId) { - return this.doQuery(loginUser, pageNo, pageSize, processId); + return this.doQuery(loginUser, pageNo, pageSize, processId); } /** @@ -112,7 +113,7 @@ public Map queryTasksByProcessId(User loginUser, int pageNo, int */ @Override public Map queryAllTasks(User loginUser, int pageNo, int pageSize) { - return this.doQuery(loginUser, pageNo, pageSize, 0); + return this.doQuery(loginUser, pageNo, pageSize, 0); } public Map doQuery(User loginUser, int pageNo, int pageSize, @@ -145,12 +146,12 @@ public boolean deleteByTaskId(int taskId) { } @Override - public void forceStartTask(int queueId,int forceStart) { - taskGroupQueueMapper.updateForceStart(queueId,forceStart); + public void forceStartTask(int queueId, int forceStart) { + taskGroupQueueMapper.updateForceStart(queueId, forceStart); } @Override public void modifyPriority(Integer queueId, Integer priority) { - taskGroupQueueMapper.modifyPriority(queueId,priority); + taskGroupQueueMapper.modifyPriority(queueId, priority); } } diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/TaskGroupServiceImpl.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/TaskGroupServiceImpl.java index d954b944136e..6f22af8662ab 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/TaskGroupServiceImpl.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/TaskGroupServiceImpl.java @@ -117,7 +117,8 @@ public Map createTaskGroup(User loginUser, Long projectCode, Str taskGroup.setCreateTime(new Date()); taskGroup.setUpdateTime(new Date()); if (taskGroupMapper.insert(taskGroup) > 0) { - permissionPostHandle(AuthorizationType.TASK_GROUP, loginUser.getId(), Collections.singletonList(taskGroup.getId()),logger); + permissionPostHandle(AuthorizationType.TASK_GROUP, loginUser.getId(), + Collections.singletonList(taskGroup.getId()), logger); logger.info("Create task group complete, taskGroupName:{}.", taskGroup.getName()); putMsg(result, Status.SUCCESS); } else { diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/TaskInstanceServiceImpl.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/TaskInstanceServiceImpl.java index 1aabfd44c782..7c6b724c9054 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/TaskInstanceServiceImpl.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/TaskInstanceServiceImpl.java @@ -38,11 +38,11 @@ import org.apache.dolphinscheduler.dao.mapper.ProjectMapper; import org.apache.dolphinscheduler.dao.mapper.TaskDefinitionMapper; import org.apache.dolphinscheduler.dao.mapper.TaskInstanceMapper; +import org.apache.dolphinscheduler.plugin.task.api.enums.TaskExecutionStatus; import org.apache.dolphinscheduler.remote.command.TaskKillRequestCommand; import org.apache.dolphinscheduler.remote.command.TaskSavePointRequestCommand; import org.apache.dolphinscheduler.remote.processor.StateEventCallbackService; import org.apache.dolphinscheduler.remote.utils.Host; -import org.apache.dolphinscheduler.plugin.task.api.enums.TaskExecutionStatus; import org.apache.dolphinscheduler.service.process.ProcessService; import java.util.Date; @@ -154,12 +154,12 @@ public Result queryTaskListPaging(User loginUser, if (taskExecuteType == TaskExecuteType.STREAM) { // stream task without process instance taskInstanceIPage = taskInstanceMapper.queryStreamTaskInstanceListPaging( - page, project.getCode(), processDefinitionName, searchVal, taskName, executorId, statusArray, host, taskExecuteType, start, end - ); + page, project.getCode(), processDefinitionName, searchVal, taskName, executorId, statusArray, host, + taskExecuteType, start, end); } else { taskInstanceIPage = taskInstanceMapper.queryTaskInstanceListPaging( - page, project.getCode(), processInstanceId, processInstanceName, searchVal, taskName, executorId, statusArray, host, taskExecuteType, start, end - ); + page, project.getCode(), processInstanceId, processInstanceName, searchVal, taskName, executorId, + statusArray, host, taskExecuteType, start, end); } Set exclusionSet = new HashSet<>(); exclusionSet.add(Constants.CLASS); @@ -205,21 +205,24 @@ public Map forceTaskSuccess(User loginUser, long projectCode, In // check whether the task instance can be found TaskInstance task = taskInstanceMapper.selectById(taskInstanceId); if (task == null) { - logger.error("Task instance can not be found, projectCode:{}, taskInstanceId:{}.", projectCode, taskInstanceId); + logger.error("Task instance can not be found, projectCode:{}, taskInstanceId:{}.", projectCode, + taskInstanceId); putMsg(result, Status.TASK_INSTANCE_NOT_FOUND); return result; } TaskDefinition taskDefinition = taskDefinitionMapper.queryByCode(task.getTaskCode()); if (taskDefinition != null && projectCode != taskDefinition.getProjectCode()) { - logger.error("Task definition can not be found, projectCode:{}, taskDefinitionCode:{}.", projectCode, task.getTaskCode()); + logger.error("Task definition can not be found, projectCode:{}, taskDefinitionCode:{}.", projectCode, + task.getTaskCode()); putMsg(result, Status.TASK_INSTANCE_NOT_FOUND, taskInstanceId); return result; } // check whether the task instance state type is failure or cancel if (!task.getState().isFailure() && !task.getState().isKill()) { - logger.warn("{} type task instance can not perform force success, projectCode:{}, taskInstanceId:{}.", task.getState().getDesc(), projectCode, taskInstanceId); + logger.warn("{} type task instance can not perform force success, projectCode:{}, taskInstanceId:{}.", + task.getState().getDesc(), projectCode, taskInstanceId); putMsg(result, Status.TASK_INSTANCE_STATE_OPERATION_ERROR, taskInstanceId, task.getState().toString()); return result; } @@ -229,10 +232,12 @@ public Map forceTaskSuccess(User loginUser, long projectCode, In int changedNum = taskInstanceMapper.updateById(task); if (changedNum > 0) { processService.forceProcessInstanceSuccessByTaskInstanceId(taskInstanceId); - logger.info("Task instance performs force success complete, projectCode:{}, taskInstanceId:{}", projectCode, taskInstanceId); + logger.info("Task instance performs force success complete, projectCode:{}, taskInstanceId:{}", projectCode, + taskInstanceId); putMsg(result, Status.SUCCESS); } else { - logger.error("Task instance performs force success complete, projectCode:{}, taskInstanceId:{}", projectCode, taskInstanceId); + logger.error("Task instance performs force success complete, projectCode:{}, taskInstanceId:{}", + projectCode, taskInstanceId); putMsg(result, Status.FORCE_TASK_SUCCESS_ERROR); } return result; @@ -243,17 +248,19 @@ public Result taskSavePoint(User loginUser, long projectCode, Integer taskInstan Result result = new Result(); Project project = projectMapper.queryByCode(projectCode); - //check user access for project - Map checkResult = projectService.checkProjectAndAuth(loginUser, project, projectCode, FORCED_SUCCESS); + // check user access for project + Map checkResult = + projectService.checkProjectAndAuth(loginUser, project, projectCode, FORCED_SUCCESS); Status status = (Status) checkResult.get(Constants.STATUS); if (status != Status.SUCCESS) { - putMsg(result,status); + putMsg(result, status); return result; } TaskInstance taskInstance = taskInstanceMapper.selectById(taskInstanceId); if (taskInstance == null) { - logger.error("Task definition can not be found, projectCode:{}, taskInstanceId:{}.", projectCode, taskInstanceId); + logger.error("Task definition can not be found, projectCode:{}, taskInstanceId:{}.", projectCode, + taskInstanceId); putMsg(result, Status.TASK_INSTANCE_NOT_FOUND); return result; } @@ -272,17 +279,19 @@ public Result stopTask(User loginUser, long projectCode, Integer taskInstanceId) Result result = new Result(); Project project = projectMapper.queryByCode(projectCode); - //check user access for project - Map checkResult = projectService.checkProjectAndAuth(loginUser, project, projectCode,FORCED_SUCCESS); + // check user access for project + Map checkResult = + projectService.checkProjectAndAuth(loginUser, project, projectCode, FORCED_SUCCESS); Status status = (Status) checkResult.get(Constants.STATUS); if (status != Status.SUCCESS) { - putMsg(result,status); + putMsg(result, status); return result; } TaskInstance taskInstance = taskInstanceMapper.selectById(taskInstanceId); if (taskInstance == null) { - logger.error("Task definition can not be found, projectCode:{}, taskInstanceId:{}.", projectCode, taskInstanceId); + logger.error("Task definition can not be found, projectCode:{}, taskInstanceId:{}.", projectCode, + taskInstanceId); putMsg(result, Status.TASK_INSTANCE_NOT_FOUND); return result; } diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/TenantServiceImpl.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/TenantServiceImpl.java index 169b0a634832..682f70c296e6 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/TenantServiceImpl.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/TenantServiceImpl.java @@ -17,10 +17,11 @@ package org.apache.dolphinscheduler.api.service.impl; -import com.baomidou.mybatisplus.core.metadata.IPage; -import com.baomidou.mybatisplus.extension.plugins.pagination.Page; -import org.apache.commons.collections.CollectionUtils; -import org.apache.commons.lang3.StringUtils; +import static org.apache.dolphinscheduler.api.constants.ApiFuncIdentificationConstant.TENANT_CREATE; +import static org.apache.dolphinscheduler.api.constants.ApiFuncIdentificationConstant.TENANT_DELETE; +import static org.apache.dolphinscheduler.api.constants.ApiFuncIdentificationConstant.TENANT_UPDATE; +import static org.apache.dolphinscheduler.common.Constants.TENANT_FULL_NAME_MAX_LENGTH; + import org.apache.dolphinscheduler.api.enums.Status; import org.apache.dolphinscheduler.api.exceptions.ServiceException; import org.apache.dolphinscheduler.api.service.QueueService; @@ -41,23 +42,26 @@ import org.apache.dolphinscheduler.dao.mapper.TenantMapper; import org.apache.dolphinscheduler.dao.mapper.UserMapper; import org.apache.dolphinscheduler.service.storage.StorageOperate; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.stereotype.Service; -import org.springframework.transaction.annotation.Transactional; + +import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.lang3.StringUtils; import java.util.ArrayList; import java.util.Collections; -import java.util.Date; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Objects; import java.util.Set; -import static org.apache.dolphinscheduler.api.constants.ApiFuncIdentificationConstant.*; -import static org.apache.dolphinscheduler.common.Constants.TENANT_FULL_NAME_MAX_LENGTH; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; +import org.springframework.transaction.annotation.Transactional; + +import com.baomidou.mybatisplus.core.metadata.IPage; +import com.baomidou.mybatisplus.extension.plugins.pagination.Page; /** * tenant service impl @@ -121,7 +125,8 @@ else if (StringUtils.isEmpty(updateTenant.getTenantCode())) { throw new ServiceException(Status.TENANT_FULL_NAME_TOO_LONG_ERROR); } else if (!RegexUtils.isValidLinuxUserName(updateTenant.getTenantCode())) { throw new ServiceException(Status.CHECK_OS_TENANT_CODE_ERROR); - } else if (!Objects.equals(existsTenant.getTenantCode(), updateTenant.getTenantCode()) && checkTenantExists(updateTenant.getTenantCode())) { + } else if (!Objects.equals(existsTenant.getTenantCode(), updateTenant.getTenantCode()) + && checkTenantExists(updateTenant.getTenantCode())) { throw new ServiceException(Status.OS_TENANT_CODE_EXIST, updateTenant.getTenantCode()); } } @@ -144,10 +149,10 @@ public Map createTenant(User loginUser, String desc) throws Exception { Map result = new HashMap<>(); result.put(Constants.STATUS, false); - if (!canOperatorPermissions(loginUser,null, AuthorizationType.TENANT, TENANT_CREATE)) { + if (!canOperatorPermissions(loginUser, null, AuthorizationType.TENANT, TENANT_CREATE)) { throw new ServiceException(Status.USER_NO_OPERATION_PERM); } - if(checkDescriptionLength(desc)){ + if (checkDescriptionLength(desc)) { logger.warn("Parameter description is too long."); putMsg(result, Status.DESCRIPTION_TOO_LONG_ERROR); return result; @@ -160,7 +165,8 @@ public Map createTenant(User loginUser, if (PropertyUtils.getResUploadStartupState()) { storageOperate.createTenantDirIfNotExists(tenantCode); } - permissionPostHandle(AuthorizationType.TENANT, loginUser.getId(), Collections.singletonList(tenant.getId()), logger); + permissionPostHandle(AuthorizationType.TENANT, loginUser.getId(), Collections.singletonList(tenant.getId()), + logger); result.put(Constants.DATA_LIST, tenant); putMsg(result, Status.SUCCESS); return result; @@ -180,7 +186,8 @@ public Result queryTenantList(User loginUser, String searchVal, Integer Result result = new Result<>(); PageInfo pageInfo = new PageInfo<>(pageNo, pageSize); - Set ids = resourcePermissionCheckService.userOwnedResourceIdsAcquisition(AuthorizationType.TENANT, loginUser.getId(), logger); + Set ids = resourcePermissionCheckService.userOwnedResourceIdsAcquisition(AuthorizationType.TENANT, + loginUser.getId(), logger); if (ids.isEmpty()) { result.setData(pageInfo); putMsg(result, Status.SUCCESS); @@ -213,10 +220,10 @@ public Map updateTenant(User loginUser, int id, String tenantCod Map result = new HashMap<>(); - if (!canOperatorPermissions(loginUser,null, AuthorizationType.TENANT,TENANT_UPDATE)) { + if (!canOperatorPermissions(loginUser, null, AuthorizationType.TENANT, TENANT_UPDATE)) { throw new ServiceException(Status.USER_NO_OPERATION_PERM); } - if(checkDescriptionLength(desc)){ + if (checkDescriptionLength(desc)) { logger.warn("Parameter description is too long."); putMsg(result, Status.DESCRIPTION_TOO_LONG_ERROR); return result; @@ -229,7 +236,8 @@ public Map updateTenant(User loginUser, int id, String tenantCod /** * if the tenant code is modified, the original resource needs to be copied to the new tenant. */ - if (!Objects.equals(existsTenant.getTenantCode(), updateTenant.getTenantCode()) && PropertyUtils.getResUploadStartupState()) { + if (!Objects.equals(existsTenant.getTenantCode(), updateTenant.getTenantCode()) + && PropertyUtils.getResUploadStartupState()) { storageOperate.createTenantDirIfNotExists(tenantCode); } int update = tenantMapper.updateById(updateTenant); @@ -256,7 +264,7 @@ public Map updateTenant(User loginUser, int id, String tenantCod public Map deleteTenantById(User loginUser, int id) throws Exception { Map result = new HashMap<>(); - if (!canOperatorPermissions(loginUser,null, AuthorizationType.TENANT,TENANT_DELETE)) { + if (!canOperatorPermissions(loginUser, null, AuthorizationType.TENANT, TENANT_DELETE)) { throw new ServiceException(Status.USER_NO_OPERATION_PERM); } @@ -268,14 +276,16 @@ public Map deleteTenantById(User loginUser, int id) throws Excep List processInstances = getProcessInstancesByTenant(tenant); if (CollectionUtils.isNotEmpty(processInstances)) { - logger.warn("Delete tenant failed, because there are {} executing process instances using it.", processInstances.size()); + logger.warn("Delete tenant failed, because there are {} executing process instances using it.", + processInstances.size()); throw new ServiceException(Status.DELETE_TENANT_BY_ID_FAIL, processInstances.size()); } List processDefinitions = processDefinitionMapper.queryDefinitionListByTenant(tenant.getId()); if (CollectionUtils.isNotEmpty(processDefinitions)) { - logger.warn("Delete tenant failed, because there are {} process definitions using it.", processDefinitions.size()); + logger.warn("Delete tenant failed, because there are {} process definitions using it.", + processDefinitions.size()); throw new ServiceException(Status.DELETE_TENANT_BY_ID_FAIL_DEFINES, processDefinitions.size()); } @@ -287,7 +297,7 @@ public Map deleteTenantById(User loginUser, int id) throws Excep // if resource upload startup if (PropertyUtils.getResUploadStartupState()) { - storageOperate.deleteTenant(tenant.getTenantCode()); + storageOperate.deleteTenant(tenant.getTenantCode()); } int delete = tenantMapper.deleteById(id); @@ -304,7 +314,8 @@ public Map deleteTenantById(User loginUser, int id) throws Excep } private List getProcessInstancesByTenant(Tenant tenant) { - return processInstanceMapper.queryByTenantIdAndStatus(tenant.getId(), org.apache.dolphinscheduler.service.utils.Constants.NOT_TERMINATED_STATES); + return processInstanceMapper.queryByTenantIdAndStatus(tenant.getId(), + org.apache.dolphinscheduler.service.utils.Constants.NOT_TERMINATED_STATES); } /** @@ -317,7 +328,8 @@ private List getProcessInstancesByTenant(Tenant tenant) { public Map queryTenantList(User loginUser) { Map result = new HashMap<>(); - Set ids = resourcePermissionCheckService.userOwnedResourceIdsAcquisition(AuthorizationType.TENANT, loginUser.getId(), logger); + Set ids = resourcePermissionCheckService.userOwnedResourceIdsAcquisition(AuthorizationType.TENANT, + loginUser.getId(), logger); if (ids.isEmpty()) { result.put(Constants.DATA_LIST, Collections.emptyList()); putMsg(result, Status.SUCCESS); diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/UdfFuncServiceImpl.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/UdfFuncServiceImpl.java index 578668eea6c6..c5e5da007c60 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/UdfFuncServiceImpl.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/UdfFuncServiceImpl.java @@ -17,9 +17,6 @@ package org.apache.dolphinscheduler.api.service.impl; -import com.baomidou.mybatisplus.core.metadata.IPage; -import com.baomidou.mybatisplus.extension.plugins.pagination.Page; -import org.apache.commons.lang3.StringUtils; import org.apache.dolphinscheduler.api.constants.ApiFuncIdentificationConstant; import org.apache.dolphinscheduler.api.enums.Status; import org.apache.dolphinscheduler.api.service.UdfFuncService; @@ -34,11 +31,8 @@ import org.apache.dolphinscheduler.dao.mapper.ResourceMapper; import org.apache.dolphinscheduler.dao.mapper.UDFUserMapper; import org.apache.dolphinscheduler.dao.mapper.UdfFuncMapper; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.stereotype.Service; -import org.springframework.transaction.annotation.Transactional; + +import org.apache.commons.lang3.StringUtils; import java.util.ArrayList; import java.util.Collections; @@ -46,6 +40,15 @@ import java.util.List; import java.util.Set; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; +import org.springframework.transaction.annotation.Transactional; + +import com.baomidou.mybatisplus.core.metadata.IPage; +import com.baomidou.mybatisplus.extension.plugins.pagination.Page; + /** * udf func service impl */ @@ -88,19 +91,21 @@ public Result createUdfFunction(User loginUser, int resourceId) { Result result = new Result<>(); - boolean canOperatorPermissions = canOperatorPermissions(loginUser, null, AuthorizationType.UDF, ApiFuncIdentificationConstant.UDF_FUNCTION_CREATE); - if (!canOperatorPermissions){ + boolean canOperatorPermissions = canOperatorPermissions(loginUser, null, AuthorizationType.UDF, + ApiFuncIdentificationConstant.UDF_FUNCTION_CREATE); + if (!canOperatorPermissions) { putMsg(result, Status.NO_CURRENT_OPERATING_PERMISSION); return result; } - if(checkDescriptionLength(desc)){ + if (checkDescriptionLength(desc)) { logger.warn("Parameter description is too long."); putMsg(result, Status.DESCRIPTION_TOO_LONG_ERROR); return result; } // if resource upload startup if (!PropertyUtils.getResUploadStartupState()) { - logger.error("Storage does not start up, resource upload startup state: {}.", PropertyUtils.getResUploadStartupState()); + logger.error("Storage does not start up, resource upload startup state: {}.", + PropertyUtils.getResUploadStartupState()); putMsg(result, Status.HDFS_NOT_STARTUP); return result; } @@ -119,7 +124,7 @@ public Result createUdfFunction(User loginUser, return result; } - //save data + // save data UdfFunc udf = new UdfFunc(); Date now = new Date(); udf.setUserId(loginUser.getId()); @@ -165,8 +170,9 @@ private boolean checkUdfFuncNameExists(String name) { @Override public Result queryUdfFuncDetail(User loginUser, int id) { Result result = new Result<>(); - boolean canOperatorPermissions = canOperatorPermissions(loginUser, new Object[]{id}, AuthorizationType.UDF, ApiFuncIdentificationConstant.UDF_FUNCTION_VIEW); - if (!canOperatorPermissions){ + boolean canOperatorPermissions = canOperatorPermissions(loginUser, new Object[]{id}, AuthorizationType.UDF, + ApiFuncIdentificationConstant.UDF_FUNCTION_VIEW); + if (!canOperatorPermissions) { putMsg(result, Status.NO_CURRENT_OPERATING_PERMISSION); return result; } @@ -206,12 +212,13 @@ public Result updateUdfFunc(User loginUser, int resourceId) { Result result = new Result<>(); - boolean canOperatorPermissions = canOperatorPermissions(loginUser, new Object[]{udfFuncId}, AuthorizationType.UDF, ApiFuncIdentificationConstant.UDF_FUNCTION_UPDATE); - if (!canOperatorPermissions){ + boolean canOperatorPermissions = canOperatorPermissions(loginUser, new Object[]{udfFuncId}, + AuthorizationType.UDF, ApiFuncIdentificationConstant.UDF_FUNCTION_UPDATE); + if (!canOperatorPermissions) { putMsg(result, Status.NO_CURRENT_OPERATING_PERMISSION); return result; } - if(checkDescriptionLength(desc)){ + if (checkDescriptionLength(desc)) { logger.warn("Parameter description is too long."); putMsg(result, Status.DESCRIPTION_TOO_LONG_ERROR); return result; @@ -228,7 +235,8 @@ public Result updateUdfFunc(User loginUser, // if resource upload startup if (!PropertyUtils.getResUploadStartupState()) { - logger.error("Storage does not start up, resource upload startup state: {}.", PropertyUtils.getResUploadStartupState()); + logger.error("Storage does not start up, resource upload startup state: {}.", + PropertyUtils.getResUploadStartupState()); putMsg(result, Status.HDFS_NOT_STARTUP); return result; } @@ -282,14 +290,15 @@ public Result updateUdfFunc(User loginUser, @Override public Result queryUdfFuncListPaging(User loginUser, String searchVal, Integer pageNo, Integer pageSize) { Result result = new Result(); - boolean canOperatorPermissions = canOperatorPermissions(loginUser, null, AuthorizationType.UDF, ApiFuncIdentificationConstant.UDF_FUNCTION_VIEW); - if (!canOperatorPermissions){ + boolean canOperatorPermissions = canOperatorPermissions(loginUser, null, AuthorizationType.UDF, + ApiFuncIdentificationConstant.UDF_FUNCTION_VIEW); + if (!canOperatorPermissions) { putMsg(result, Status.NO_CURRENT_OPERATING_PERMISSION); return result; } PageInfo pageInfo = new PageInfo<>(pageNo, pageSize); IPage udfFuncList = getUdfFuncsPage(loginUser, searchVal, pageSize, pageNo); - pageInfo.setTotal((int)udfFuncList.getTotal()); + pageInfo.setTotal((int) udfFuncList.getTotal()); pageInfo.setTotalList(udfFuncList.getRecords()); result.setData(pageInfo); putMsg(result, Status.SUCCESS); @@ -306,7 +315,8 @@ public Result queryUdfFuncListPaging(User loginUser, String searchVal, I * @return udf function list page */ private IPage getUdfFuncsPage(User loginUser, String searchVal, Integer pageSize, int pageNo) { - Set udfFuncIds = resourcePermissionCheckService.userOwnedResourceIdsAcquisition(AuthorizationType.UDF, loginUser.getId(), logger); + Set udfFuncIds = resourcePermissionCheckService.userOwnedResourceIdsAcquisition(AuthorizationType.UDF, + loginUser.getId(), logger); Page page = new Page<>(pageNo, pageSize); if (udfFuncIds.isEmpty()) { return page; @@ -325,13 +335,15 @@ private IPage getUdfFuncsPage(User loginUser, String searchVal, Integer public Result queryUdfFuncList(User loginUser, Integer type) { Result result = new Result<>(); - boolean canOperatorPermissions = canOperatorPermissions(loginUser, null, AuthorizationType.UDF, ApiFuncIdentificationConstant.UDF_FUNCTION_VIEW); - if (!canOperatorPermissions){ + boolean canOperatorPermissions = canOperatorPermissions(loginUser, null, AuthorizationType.UDF, + ApiFuncIdentificationConstant.UDF_FUNCTION_VIEW); + if (!canOperatorPermissions) { putMsg(result, Status.NO_CURRENT_OPERATING_PERMISSION); return result; } - Set udfFuncIds = resourcePermissionCheckService.userOwnedResourceIdsAcquisition(AuthorizationType.UDF, loginUser.getId(), logger); - if (udfFuncIds.isEmpty()){ + Set udfFuncIds = resourcePermissionCheckService.userOwnedResourceIdsAcquisition(AuthorizationType.UDF, + loginUser.getId(), logger); + if (udfFuncIds.isEmpty()) { result.setData(Collections.emptyList()); putMsg(result, Status.SUCCESS); return result; @@ -354,8 +366,9 @@ public Result queryUdfFuncList(User loginUser, Integer type) { public Result delete(User loginUser, int id) { Result result = new Result<>(); - boolean canOperatorPermissions = canOperatorPermissions(loginUser, new Object[]{id}, AuthorizationType.UDF, ApiFuncIdentificationConstant.UDF_FUNCTION_DELETE); - if (!canOperatorPermissions){ + boolean canOperatorPermissions = canOperatorPermissions(loginUser, new Object[]{id}, AuthorizationType.UDF, + ApiFuncIdentificationConstant.UDF_FUNCTION_DELETE); + if (!canOperatorPermissions) { putMsg(result, Status.NO_CURRENT_OPERATING_PERMISSION); return result; } @@ -375,8 +388,9 @@ public Result delete(User loginUser, int id) { @Override public Result verifyUdfFuncByName(User loginUser, String name) { Result result = new Result<>(); - boolean canOperatorPermissions = canOperatorPermissions(loginUser, null, AuthorizationType.UDF, ApiFuncIdentificationConstant.UDF_FUNCTION_VIEW); - if (!canOperatorPermissions){ + boolean canOperatorPermissions = canOperatorPermissions(loginUser, null, AuthorizationType.UDF, + ApiFuncIdentificationConstant.UDF_FUNCTION_VIEW); + if (!canOperatorPermissions) { putMsg(result, Status.NO_CURRENT_OPERATING_PERMISSION); return result; } diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/UsersServiceImpl.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/UsersServiceImpl.java index 7ed97daada9d..47d35bac72bc 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/UsersServiceImpl.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/UsersServiceImpl.java @@ -56,6 +56,7 @@ import org.apache.dolphinscheduler.dao.mapper.UserMapper; import org.apache.dolphinscheduler.dao.utils.ResourceProcessDefinitionUtils; import org.apache.dolphinscheduler.service.storage.StorageOperate; + import org.apache.commons.collections.CollectionUtils; import org.apache.commons.lang3.StringUtils; @@ -339,7 +340,8 @@ public Result queryUserList(User loginUser, String searchVal, Integer pa return result; } if (!isAdmin(loginUser)) { - logger.warn("User does not have permission for this feature, userId:{}, userName:{}.", loginUser.getId(), loginUser.getUserName()); + logger.warn("User does not have permission for this feature, userId:{}, userName:{}.", loginUser.getId(), + loginUser.getUserName()); putMsg(result, Status.USER_NO_OPERATION_PERM); return result; } @@ -390,7 +392,8 @@ public Map updateUser(User loginUser, int userId, return result; } if (check(result, !canOperator(loginUser, userId), Status.USER_NO_OPERATION_PERM)) { - logger.warn("User does not have permission for this feature, userId:{}, userName:{}.", loginUser.getId(), loginUser.getUserName()); + logger.warn("User does not have permission for this feature, userId:{}, userName:{}.", loginUser.getId(), + loginUser.getUserName()); return result; } User user = userMapper.selectById(userId); @@ -441,7 +444,8 @@ public Map updateUser(User loginUser, int userId, } if (state == 0 && user.getState() != state && Objects.equals(loginUser.getId(), user.getId())) { - logger.warn("Not allow to disable your own account, userId:{}, userName:{}.", user.getId(), user.getUserName()); + logger.warn("Not allow to disable your own account, userId:{}, userName:{}.", user.getId(), + user.getUserName()); putMsg(result, Status.NOT_ALLOW_TO_DISABLE_OWN_ACCOUNT); return result; } @@ -492,7 +496,8 @@ public Map deleteUserById(User loginUser, int id) throws IOExcep } // only admin can operate if (!isAdmin(loginUser)) { - logger.warn("User does not have permission for this feature, userId:{}, userName:{}.", loginUser.getId(), loginUser.getUserName()); + logger.warn("User does not have permission for this feature, userId:{}, userName:{}.", loginUser.getId(), + loginUser.getUserName()); putMsg(result, Status.USER_NO_OPERATION_PERM, id); return result; } @@ -508,7 +513,8 @@ public Map deleteUserById(User loginUser, int id) throws IOExcep if (CollectionUtils.isNotEmpty(projects)) { String projectNames = projects.stream().map(Project::getName).collect(Collectors.joining(",")); putMsg(result, Status.TRANSFORM_PROJECT_OWNERSHIP, projectNames); - logger.warn("Please transfer the project ownership before deleting the user, userId:{}, projects:{}.", id, projectNames); + logger.warn("Please transfer the project ownership before deleting the user, userId:{}, projects:{}.", id, + projectNames); return result; } // delete user @@ -610,7 +616,8 @@ public Map grantProjectByCode(final User loginUser, final int us // 3. only project owner can operate if (!this.canOperator(loginUser, project.getUserId())) { - logger.warn("User does not have permission for project, userId:{}, userName:{}, projectCode:{}.", loginUser.getId(), loginUser.getUserName(), projectCode); + logger.warn("User does not have permission for project, userId:{}, userName:{}, projectCode:{}.", + loginUser.getId(), loginUser.getUserName(), projectCode); this.putMsg(result, Status.USER_NO_OPERATION_PERM); return result; } @@ -777,7 +784,8 @@ public Map grantResources(User loginUser, int userId, String res } - logger.info("User is granted permission for resources, userId:{}, resourceIds:{}.", user.getId(), needAuthorizeResIds); + logger.info("User is granted permission for resources, userId:{}, resourceIds:{}.", user.getId(), + needAuthorizeResIds); putMsg(result, Status.SUCCESS); diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/WorkFlowLineageServiceImpl.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/WorkFlowLineageServiceImpl.java index 791d4acc2294..91893f602705 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/WorkFlowLineageServiceImpl.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/WorkFlowLineageServiceImpl.java @@ -86,7 +86,8 @@ public Map queryWorkFlowLineageByName(long projectCode, String w putMsg(result, Status.PROJECT_NOT_FOUND, projectCode); return result; } - List workFlowLineageList = workFlowLineageMapper.queryWorkFlowLineageByName(projectCode, workFlowName); + List workFlowLineageList = + workFlowLineageMapper.queryWorkFlowLineageByName(projectCode, workFlowName); result.put(Constants.DATA_LIST, workFlowLineageList); putMsg(result, Status.SUCCESS); return result; @@ -116,23 +117,26 @@ private void recursiveWorkFlow(long projectCode, long sourceWorkFlowCode, List workFlowLineages, Set workFlowRelations) { - workFlowLineages.add(workFlowLineageMapper.queryWorkFlowLineageByCode(projectCode,sourceWorkFlowCode)); + workFlowLineages.add(workFlowLineageMapper.queryWorkFlowLineageByCode(projectCode, sourceWorkFlowCode)); List downStreamWorkFlowLineages = workFlowLineageMapper.queryDownstreamLineageByProcessDefinitionCode(sourceWorkFlowCode, "DEPENDENT"); workFlowLineages.addAll(downStreamWorkFlowLineages); - downStreamWorkFlowLineages.forEach(workFlowLineage -> workFlowRelations.add(new WorkFlowRelation(sourceWorkFlowCode, workFlowLineage.getWorkFlowCode()))); + downStreamWorkFlowLineages.forEach(workFlowLineage -> workFlowRelations + .add(new WorkFlowRelation(sourceWorkFlowCode, workFlowLineage.getWorkFlowCode()))); List upstreamWorkFlowLineages = new ArrayList<>(); getUpstreamLineages(sourceWorkFlowCode, upstreamWorkFlowLineages); workFlowLineages.addAll(upstreamWorkFlowLineages); - upstreamWorkFlowLineages.forEach(workFlowLineage -> workFlowRelations.add(new WorkFlowRelation(workFlowLineage.getWorkFlowCode(), sourceWorkFlowCode))); + upstreamWorkFlowLineages.forEach(workFlowLineage -> workFlowRelations + .add(new WorkFlowRelation(workFlowLineage.getWorkFlowCode(), sourceWorkFlowCode))); } private void getUpstreamLineages(long sourceWorkFlowCode, List upstreamWorkFlowLineages) { List workFlowDependentDefinitionList = - workFlowLineageMapper.queryUpstreamDependentParamsByProcessDefinitionCode(sourceWorkFlowCode, "DEPENDENT"); + workFlowLineageMapper.queryUpstreamDependentParamsByProcessDefinitionCode(sourceWorkFlowCode, + "DEPENDENT"); List upstreamProcessDefinitionCodes = new ArrayList<>(); @@ -151,7 +155,8 @@ private void getUpstreamLineages(long sourceWorkFlowCode, private void getProcessDefinitionCodeByDependentDefinitionList(List dependentDefinitionList, List processDefinitionCodes) { for (DependentProcessDefinition dependentProcessDefinition : dependentDefinitionList) { - for (DependentTaskModel dependentTaskModel : dependentProcessDefinition.getDependentParameters().getDependTaskList()) { + for (DependentTaskModel dependentTaskModel : dependentProcessDefinition.getDependentParameters() + .getDependTaskList()) { for (DependentItem dependentItem : dependentTaskModel.getDependItemList()) { if (!processDefinitionCodes.contains(dependentItem.getDefinitionCode())) { processDefinitionCodes.add(dependentItem.getDefinitionCode()); @@ -174,8 +179,10 @@ public Map queryWorkFlowLineage(long projectCode) { Map workFlowLineagesMap = new HashMap<>(); Set workFlowRelations = new HashSet<>(); if (!processLineages.isEmpty()) { - List workFlowLineages = workFlowLineageMapper.queryWorkFlowLineageByLineage(processLineages); - workFlowLineagesMap = workFlowLineages.stream().collect(Collectors.toMap(WorkFlowLineage::getWorkFlowCode, workFlowLineage -> workFlowLineage)); + List workFlowLineages = + workFlowLineageMapper.queryWorkFlowLineageByLineage(processLineages); + workFlowLineagesMap = workFlowLineages.stream() + .collect(Collectors.toMap(WorkFlowLineage::getWorkFlowCode, workFlowLineage -> workFlowLineage)); Map> workFlowMap = new HashMap<>(); for (ProcessLineage processLineage : processLineages) { workFlowMap.compute(processLineage.getProcessDefinitionCode(), (k, v) -> { @@ -186,18 +193,22 @@ public Map queryWorkFlowLineage(long projectCode) { v.add(new TaskDefinition(processLineage.getPreTaskCode(), processLineage.getPreTaskVersion())); } if (processLineage.getPostTaskCode() > 0) { - v.add(new TaskDefinition(processLineage.getPostTaskCode(), processLineage.getPostTaskVersion())); + v.add(new TaskDefinition(processLineage.getPostTaskCode(), + processLineage.getPostTaskVersion())); } return v; }); } for (Entry> workFlow : workFlowMap.entrySet()) { - Set sourceWorkFlowCodes = querySourceWorkFlowCodes(projectCode, workFlow.getKey(), workFlow.getValue()); + Set sourceWorkFlowCodes = + querySourceWorkFlowCodes(projectCode, workFlow.getKey(), workFlow.getValue()); if (sourceWorkFlowCodes.isEmpty()) { workFlowRelations.add(new WorkFlowRelation(0L, workFlow.getKey())); } else { - workFlowLineagesMap.get(workFlow.getKey()).setSourceWorkFlowCode(StringUtils.join(sourceWorkFlowCodes, Constants.COMMA)); - sourceWorkFlowCodes.forEach(code -> workFlowRelations.add(new WorkFlowRelation(code, workFlow.getKey()))); + workFlowLineagesMap.get(workFlow.getKey()) + .setSourceWorkFlowCode(StringUtils.join(sourceWorkFlowCodes, Constants.COMMA)); + sourceWorkFlowCodes + .forEach(code -> workFlowRelations.add(new WorkFlowRelation(code, workFlow.getKey()))); } } } @@ -209,7 +220,8 @@ public Map queryWorkFlowLineage(long projectCode) { return result; } - private Set querySourceWorkFlowCodes(long projectCode, long workFlowCode, List taskDefinitionList) { + private Set querySourceWorkFlowCodes(long projectCode, long workFlowCode, + List taskDefinitionList) { Set sourceWorkFlowCodes = new HashSet<>(); if (taskDefinitionList == null || taskDefinitionList.isEmpty()) { return sourceWorkFlowCodes; @@ -218,14 +230,16 @@ private Set querySourceWorkFlowCodes(long projectCode, long workFlowCode, for (TaskDefinitionLog taskDefinitionLog : taskDefinitionLogs) { if (taskDefinitionLog.getProjectCode() == projectCode) { if (taskDefinitionLog.getTaskType().equals(TASK_TYPE_DEPENDENT)) { - DependentParameters dependentParameters = JSONUtils.parseObject(taskDefinitionLog.getDependence(), DependentParameters.class); + DependentParameters dependentParameters = + JSONUtils.parseObject(taskDefinitionLog.getDependence(), DependentParameters.class); if (dependentParameters != null) { List dependTaskList = dependentParameters.getDependTaskList(); if (!CollectionUtils.isEmpty(dependTaskList)) { for (DependentTaskModel taskModel : dependTaskList) { List dependItemList = taskModel.getDependItemList(); for (DependentItem dependentItem : dependItemList) { - if (dependentItem.getProjectCode() == projectCode && dependentItem.getDefinitionCode() != workFlowCode) { + if (dependentItem.getProjectCode() == projectCode + && dependentItem.getDefinitionCode() != workFlowCode) { sourceWorkFlowCodes.add(dependentItem.getDefinitionCode()); } } @@ -248,14 +262,18 @@ private Set querySourceWorkFlowCodes(long projectCode, long workFlowCode, */ @Override public Optional taskDepOnTaskMsg(long projectCode, long processDefinitionCode, long taskCode) { - List tasksDep = workFlowLineageMapper.queryTaskDepOnTask(projectCode, processDefinitionCode, taskCode); + List tasksDep = + workFlowLineageMapper.queryTaskDepOnTask(projectCode, processDefinitionCode, taskCode); if (CollectionUtils.isEmpty(tasksDep)) { return Optional.empty(); } - String taskDepStr = tasksDep.stream().map(task -> String.format(Constants.FORMAT_S_S_COLON, task.getProcessDefinitionName(), task.getTaskName())).collect(Collectors.joining(Constants.COMMA)); + String taskDepStr = tasksDep.stream().map( + task -> String.format(Constants.FORMAT_S_S_COLON, task.getProcessDefinitionName(), task.getTaskName())) + .collect(Collectors.joining(Constants.COMMA)); TaskDefinition taskDefinition = taskDefinitionMapper.queryByCode(taskCode); - return Optional.of(MessageFormat.format(Status.DELETE_TASK_USE_BY_OTHER_FAIL.getMsg(), taskDefinition.getName(), taskDepStr)); + return Optional.of(MessageFormat.format(Status.DELETE_TASK_USE_BY_OTHER_FAIL.getMsg(), taskDefinition.getName(), + taskDepStr)); } /** @@ -268,8 +286,10 @@ public Optional taskDepOnTaskMsg(long projectCode, long processDefinitio @Override public Set queryTaskDepOnProcess(long projectCode, long processDefinitionCode) { Set taskMainInfos = new HashSet<>(); - List taskDependents = workFlowLineageMapper.queryTaskDependentDepOnProcess(projectCode, processDefinitionCode); - List taskSubProcess = workFlowLineageMapper.queryTaskSubProcessDepOnProcess(projectCode, processDefinitionCode); + List taskDependents = + workFlowLineageMapper.queryTaskDependentDepOnProcess(projectCode, processDefinitionCode); + List taskSubProcess = + workFlowLineageMapper.queryTaskSubProcessDepOnProcess(projectCode, processDefinitionCode); taskMainInfos.addAll(taskDependents); taskMainInfos.addAll(taskSubProcess); return taskMainInfos; diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/WorkerGroupServiceImpl.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/WorkerGroupServiceImpl.java index d95a2017f3f7..41f4cf8bfadd 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/WorkerGroupServiceImpl.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/WorkerGroupServiceImpl.java @@ -322,7 +322,8 @@ public Map deleteWorkerGroupById(User loginUser, Integer id) { return result; } List processInstances = processInstanceMapper - .queryByWorkerGroupNameAndStatus(workerGroup.getName(), org.apache.dolphinscheduler.service.utils.Constants.NOT_TERMINATED_STATES); + .queryByWorkerGroupNameAndStatus(workerGroup.getName(), + org.apache.dolphinscheduler.service.utils.Constants.NOT_TERMINATED_STATES); if (CollectionUtils.isNotEmpty(processInstances)) { List processInstanceIds = processInstances.stream().map(ProcessInstance::getId).collect(Collectors.toList()); diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/utils/CheckUtils.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/utils/CheckUtils.java index 33dd23113f9f..b3e5c95844fd 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/utils/CheckUtils.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/utils/CheckUtils.java @@ -17,6 +17,9 @@ package org.apache.dolphinscheduler.api.utils; +import static org.apache.dolphinscheduler.common.Constants.USER_PASSWORD_MAX_LENGTH; +import static org.apache.dolphinscheduler.common.Constants.USER_PASSWORD_MIN_LENGTH; + import org.apache.dolphinscheduler.api.enums.Status; import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.utils.JSONUtils; @@ -31,9 +34,6 @@ import org.hibernate.validator.internal.constraintvalidators.bv.EmailValidator; -import static org.apache.dolphinscheduler.common.Constants.USER_PASSWORD_MAX_LENGTH; -import static org.apache.dolphinscheduler.common.Constants.USER_PASSWORD_MIN_LENGTH; - /** * check utils */ @@ -67,7 +67,7 @@ public static boolean checkEmail(String email) { if (!emailValidator.isValid(email, null)) { return false; } - //Email is at least a second-level domain name + // Email is at least a second-level domain name int indexDomain = email.lastIndexOf("@"); String domainString = email.substring(indexDomain); return domainString.contains("."); diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/utils/FileUtils.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/utils/FileUtils.java index f85a7b111c1d..94771a41911c 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/utils/FileUtils.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/utils/FileUtils.java @@ -36,6 +36,7 @@ * file utils */ public class FileUtils { + private static final Logger logger = LoggerFactory.getLogger(FileUtils.class); /** diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/utils/PageInfo.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/utils/PageInfo.java index b9513b2cc400..ba5735f4023c 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/utils/PageInfo.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/utils/PageInfo.java @@ -53,7 +53,7 @@ public class PageInfo { private Integer pageNo; public PageInfo() { - + } public PageInfo(Integer currentPage, Integer pageSize) { @@ -128,4 +128,4 @@ public Integer getCurrentPage() { public void setCurrentPage(Integer currentPage) { this.currentPage = currentPage; } -} \ No newline at end of file +} diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/WorkerGroupControllerTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/WorkerGroupControllerTest.java index 297fc988b996..d3236786e26d 100644 --- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/WorkerGroupControllerTest.java +++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/WorkerGroupControllerTest.java @@ -24,7 +24,6 @@ import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; import org.apache.dolphinscheduler.api.utils.Result; -import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.enums.NodeType; import org.apache.dolphinscheduler.common.utils.JSONUtils; import org.apache.dolphinscheduler.dao.entity.WorkerGroup; @@ -134,7 +133,8 @@ public void testDeleteById() throws Exception { workerGroup.setId(12); workerGroup.setName("测试"); Mockito.when(workerGroupMapper.selectById(12)).thenReturn(workerGroup); - Mockito.when(processInstanceMapper.queryByWorkerGroupNameAndStatus("测试", org.apache.dolphinscheduler.service.utils.Constants.NOT_TERMINATED_STATES)) + Mockito.when(processInstanceMapper.queryByWorkerGroupNameAndStatus("测试", + org.apache.dolphinscheduler.service.utils.Constants.NOT_TERMINATED_STATES)) .thenReturn(null); Mockito.when(workerGroupMapper.deleteById(12)).thenReturn(1); Mockito.when(processInstanceMapper.updateProcessInstanceByWorkerGroupName("测试", "")).thenReturn(1); diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/BaseServiceTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/BaseServiceTest.java index 325b49984155..bc01e1ba93a1 100644 --- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/BaseServiceTest.java +++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/BaseServiceTest.java @@ -24,6 +24,7 @@ import org.apache.dolphinscheduler.common.enums.UserType; import org.apache.dolphinscheduler.dao.entity.User; import org.apache.dolphinscheduler.service.storage.impl.HadoopUtils; + import java.util.HashMap; import java.util.Map; diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ProcessInstanceServiceTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ProcessInstanceServiceTest.java index 08b06b720fc4..7bfaf3ca1ed4 100644 --- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ProcessInstanceServiceTest.java +++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ProcessInstanceServiceTest.java @@ -284,7 +284,7 @@ public void testQueryTopNLongestRunningProcessInstance() { when(projectMapper.queryByCode(projectCode)).thenReturn(project); when(projectService.checkProjectAndAuth(loginUser, project, projectCode, WORKFLOW_INSTANCE)).thenReturn(result); projectAuthFailRes = processInstanceService - .queryTopNLongestRunningProcessInstance(loginUser, projectCode, -1, startTime, endTime); + .queryTopNLongestRunningProcessInstance(loginUser, projectCode, -1, startTime, endTime); Assertions.assertEquals(Status.NEGTIVE_SIZE_NUMBER_ERROR, projectAuthFailRes.get(Constants.STATUS)); putMsg(result, Status.SUCCESS, projectCode); @@ -294,7 +294,7 @@ public void testQueryTopNLongestRunningProcessInstance() { when(usersService.getUserIdByName(loginUser.getUserName())).thenReturn(loginUser.getId()); when(usersService.queryUser(processInstance.getExecutorId())).thenReturn(loginUser); Map successRes = processInstanceService.queryTopNLongestRunningProcessInstance(loginUser, - projectCode, size, startTime, endTime); + projectCode, size, startTime, endTime); Assertions.assertEquals(Status.SUCCESS, successRes.get(Constants.STATUS)); } @@ -314,21 +314,22 @@ public void testTopNLongestRunningProcessInstanceFailure() { when(projectMapper.queryByCode(projectCode)).thenReturn(project); when(projectService.checkProjectAndAuth(loginUser, project, projectCode, WORKFLOW_INSTANCE)).thenReturn(result); Map startTimeBiggerFailRes = processInstanceService - .queryTopNLongestRunningProcessInstance(loginUser, projectCode, size, endTime, startTime); - Assertions.assertEquals(Status.START_TIME_BIGGER_THAN_END_TIME_ERROR, startTimeBiggerFailRes.get(Constants.STATUS)); + .queryTopNLongestRunningProcessInstance(loginUser, projectCode, size, endTime, startTime); + Assertions.assertEquals(Status.START_TIME_BIGGER_THAN_END_TIME_ERROR, + startTimeBiggerFailRes.get(Constants.STATUS)); putMsg(result, Status.SUCCESS, projectCode); when(projectMapper.queryByCode(projectCode)).thenReturn(project); when(projectService.checkProjectAndAuth(loginUser, project, projectCode, WORKFLOW_INSTANCE)).thenReturn(result); Map dataNullFailRes = processInstanceService - .queryTopNLongestRunningProcessInstance(loginUser, projectCode, size, null, endTime); + .queryTopNLongestRunningProcessInstance(loginUser, projectCode, size, null, endTime); Assertions.assertEquals(Status.DATA_IS_NULL, dataNullFailRes.get(Constants.STATUS)); putMsg(result, Status.SUCCESS, projectCode); when(projectMapper.queryByCode(projectCode)).thenReturn(project); when(projectService.checkProjectAndAuth(loginUser, project, projectCode, WORKFLOW_INSTANCE)).thenReturn(result); dataNullFailRes = processInstanceService - .queryTopNLongestRunningProcessInstance(loginUser, projectCode, size, startTime, null); + .queryTopNLongestRunningProcessInstance(loginUser, projectCode, size, startTime, null); Assertions.assertEquals(Status.DATA_IS_NULL, dataNullFailRes.get(Constants.STATUS)); } @@ -344,7 +345,7 @@ public void testQueryProcessInstanceById() { when(projectMapper.queryByCode(projectCode)).thenReturn(project); when(projectService.checkProjectAndAuth(loginUser, project, projectCode, WORKFLOW_INSTANCE)).thenReturn(result); Map projectAuthFailRes = - processInstanceService.queryProcessInstanceById(loginUser, projectCode, 1); + processInstanceService.queryProcessInstanceById(loginUser, projectCode, 1); Assertions.assertEquals(Status.PROJECT_NOT_FOUND, projectAuthFailRes.get(Constants.STATUS)); // project auth success @@ -371,7 +372,7 @@ public void testQueryProcessInstanceById() { Assertions.assertEquals(Status.SUCCESS, workerExistRes.get(Constants.STATUS)); when(processService.findProcessDefinition(processInstance.getProcessDefinitionCode(), - processInstance.getProcessDefinitionVersion())).thenReturn(null);; + processInstance.getProcessDefinitionVersion())).thenReturn(null);; workerExistRes = processInstanceService.queryProcessInstanceById(loginUser, projectCode, 1); Assertions.assertEquals(Status.PROCESS_DEFINE_NOT_EXIST, workerExistRes.get(Constants.STATUS)); } @@ -388,7 +389,7 @@ public void testQueryTaskListByProcessId() throws IOException { when(projectMapper.queryByCode(projectCode)).thenReturn(project); when(projectService.checkProjectAndAuth(loginUser, project, projectCode, WORKFLOW_INSTANCE)).thenReturn(result); Map projectAuthFailRes = - processInstanceService.queryTaskListByProcessId(loginUser, projectCode, 1); + processInstanceService.queryTaskListByProcessId(loginUser, projectCode, 1); Assertions.assertEquals(Status.PROJECT_NOT_FOUND, projectAuthFailRes.get(Constants.STATUS)); // project auth success @@ -638,7 +639,7 @@ public void testDeleteProcessInstanceById() { when(projectMapper.queryByCode(projectCode)).thenReturn(project); when(projectService.checkProjectAndAuth(loginUser, project, projectCode, INSTANCE_DELETE)).thenReturn(result); Map projectAuthFailRes = - processInstanceService.deleteProcessInstanceById(loginUser, projectCode, 1); + processInstanceService.deleteProcessInstanceById(loginUser, projectCode, 1); Assertions.assertEquals(Status.PROJECT_NOT_FOUND, projectAuthFailRes.get(Constants.STATUS)); // not sub process @@ -675,7 +676,7 @@ public void testDeleteProcessInstanceById() { processDefinition.setProjectCode(projectCode); when(processService.deleteWorkProcessInstanceById(1)).thenReturn(1); Map successRes = - processInstanceService.deleteProcessInstanceById(loginUser, projectCode, 1); + processInstanceService.deleteProcessInstanceById(loginUser, projectCode, 1); Assertions.assertEquals(Status.SUCCESS, successRes.get(Constants.STATUS)); when(processService.deleteWorkProcessInstanceById(1)).thenReturn(0); diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ProcessTaskRelationServiceTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ProcessTaskRelationServiceTest.java index b1ae0537a050..b5b99de84331 100644 --- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ProcessTaskRelationServiceTest.java +++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ProcessTaskRelationServiceTest.java @@ -52,6 +52,7 @@ import java.util.Objects; import java.util.Set; import java.util.stream.Collectors; + import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/TenantServiceTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/TenantServiceTest.java index 8b38bd101c27..3509a288cb9d 100644 --- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/TenantServiceTest.java +++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/TenantServiceTest.java @@ -205,7 +205,8 @@ public void testDeleteTenantById() throws Exception { Mockito.when(resourcePermissionCheckService.resourcePermissionCheck(AuthorizationType.TENANT, null, 0, baseServiceLogger)).thenReturn(true); Mockito.when(tenantMapper.queryById(1)).thenReturn(getTenant()); - Mockito.when(processInstanceMapper.queryByTenantIdAndStatus(1, org.apache.dolphinscheduler.service.utils.Constants.NOT_TERMINATED_STATES)) + Mockito.when(processInstanceMapper.queryByTenantIdAndStatus(1, + org.apache.dolphinscheduler.service.utils.Constants.NOT_TERMINATED_STATES)) .thenReturn(getInstanceList()); Mockito.when(processDefinitionMapper.queryDefinitionListByTenant(2)).thenReturn(getDefinitionsList()); Mockito.when(userMapper.queryUserListByTenant(3)).thenReturn(getUserList()); diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/WorkerGroupServiceTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/WorkerGroupServiceTest.java index 5929725cfd49..e80233c63336 100644 --- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/WorkerGroupServiceTest.java +++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/WorkerGroupServiceTest.java @@ -86,7 +86,9 @@ public void testDeleteWorkerGroupById() { user.setUserType(UserType.ADMIN_USER); WorkerGroup wg2 = getWorkerGroup(2); Mockito.when(workerGroupMapper.selectById(2)).thenReturn(wg2); - Mockito.when(processInstanceMapper.queryByWorkerGroupNameAndStatus(wg2.getName(), org.apache.dolphinscheduler.service.utils.Constants.NOT_TERMINATED_STATES)).thenReturn(getProcessInstanceList()); + Mockito.when(processInstanceMapper.queryByWorkerGroupNameAndStatus(wg2.getName(), + org.apache.dolphinscheduler.service.utils.Constants.NOT_TERMINATED_STATES)) + .thenReturn(getProcessInstanceList()); Map result = workerGroupService.deleteWorkerGroupById(user, 1); Assertions.assertEquals(Status.DELETE_WORKER_GROUP_NOT_EXIST.getCode(), ((Status) result.get(Constants.STATUS)).getCode()); @@ -96,7 +98,9 @@ public void testDeleteWorkerGroupById() { // correct WorkerGroup wg3 = getWorkerGroup(3); Mockito.when(workerGroupMapper.selectById(3)).thenReturn(wg3); - Mockito.when(processInstanceMapper.queryByWorkerGroupNameAndStatus(wg3.getName(), org.apache.dolphinscheduler.service.utils.Constants.NOT_TERMINATED_STATES)).thenReturn(new ArrayList<>()); + Mockito.when(processInstanceMapper.queryByWorkerGroupNameAndStatus(wg3.getName(), + org.apache.dolphinscheduler.service.utils.Constants.NOT_TERMINATED_STATES)) + .thenReturn(new ArrayList<>()); result = workerGroupService.deleteWorkerGroupById(user, 3); Assertions.assertEquals(Status.SUCCESS.getMsg(), result.get(Constants.MSG)); } diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/config/YamlPropertySourceFactory.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/config/YamlPropertySourceFactory.java index 8c9bd2fe742b..57467d0a143a 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/config/YamlPropertySourceFactory.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/config/YamlPropertySourceFactory.java @@ -17,15 +17,15 @@ package org.apache.dolphinscheduler.common.config; +import java.io.IOException; +import java.util.Properties; + import org.springframework.beans.factory.config.YamlPropertiesFactoryBean; import org.springframework.core.env.PropertiesPropertySource; import org.springframework.core.env.PropertySource; import org.springframework.core.io.support.EncodedResource; import org.springframework.core.io.support.PropertySourceFactory; -import java.io.IOException; -import java.util.Properties; - public class YamlPropertySourceFactory implements PropertySourceFactory { @Override diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/AlertEvent.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/AlertEvent.java index 0c8ed89fd779..b46a9ed8f77b 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/AlertEvent.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/AlertEvent.java @@ -19,5 +19,5 @@ public enum AlertEvent { - SERVER_DOWN,TIME_OUT + SERVER_DOWN, TIME_OUT } diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/AlertStatus.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/AlertStatus.java index 14ed47ea2cb4..de5a87c8ae9c 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/AlertStatus.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/AlertStatus.java @@ -23,6 +23,7 @@ * alert sending(execution) status */ public enum AlertStatus { + /** * 0 waiting executed; 1 execute successfully，2 execute failed */ diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/AlertType.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/AlertType.java index d5713a775aa5..33c1a360a810 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/AlertType.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/AlertType.java @@ -23,6 +23,7 @@ * describe the reason why alert generates */ public enum AlertType { + /** * 0 process instance failure, 1 process instance success, 2 process instance blocked, 3 process instance timeout, 4 fault tolerance warning, * 5 task failure, 6 task success, 7 task timeout, 8 close alert @@ -36,8 +37,7 @@ public enum AlertType { TASK_SUCCESS(6, "task success"), TASK_TIMEOUT(7, "task timeout"), - CLOSE_ALERT(8, "the process instance success, can close the before alert") - ; + CLOSE_ALERT(8, "the process instance success, can close the before alert"); AlertType(int code, String descp) { this.code = code; diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/AlertWarnLevel.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/AlertWarnLevel.java index 71579a9611fa..34a15e6d9b66 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/AlertWarnLevel.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/AlertWarnLevel.java @@ -19,5 +19,5 @@ public enum AlertWarnLevel { - MIDDLE,SERIOUS + MIDDLE, SERIOUS } diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/AuditResourceType.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/AuditResourceType.java index cb6b6ca0f0f0..d422cd7de3e0 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/AuditResourceType.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/AuditResourceType.java @@ -23,6 +23,7 @@ * Audit Module type */ public enum AuditResourceType { + // TODO: add other audit resource enums USER_MODULE(0, "USER"), PROJECT_MODULE(1, "PROJECT"); diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/AuthorizationType.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/AuthorizationType.java index f8654f4d1a23..49905df3eb2d 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/AuthorizationType.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/AuthorizationType.java @@ -23,6 +23,7 @@ * Authorization type */ public enum AuthorizationType { + /** * 0 RESOURCE_FILE_ID; * 1 RESOURCE_FILE_NAME; @@ -53,14 +54,14 @@ public enum AuthorizationType { ALERT_GROUP(7, "alert group"), ENVIRONMENT(8, "environment"), ACCESS_TOKEN(9, "access token"), - QUEUE(10,"queue"), - DATA_ANALYSIS(11,"data analysis"), - K8S_NAMESPACE(12,"k8s namespace"), - MONITOR(13,"monitor"), - ALERT_PLUGIN_INSTANCE(14,"alert plugin instance"), - TENANT(15,"tenant"), - DATA_QUALITY(16,"data quality"), - TASK_GROUP(17,"task group"), + QUEUE(10, "queue"), + DATA_ANALYSIS(11, "data analysis"), + K8S_NAMESPACE(12, "k8s namespace"), + MONITOR(13, "monitor"), + ALERT_PLUGIN_INSTANCE(14, "alert plugin instance"), + TENANT(15, "tenant"), + DATA_QUALITY(16, "data quality"), + TASK_GROUP(17, "task group"), ; AuthorizationType(int code, String descp) { diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/BlockingOpportunity.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/BlockingOpportunity.java index c2d103e5d8c6..5fc7c5073fb1 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/BlockingOpportunity.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/BlockingOpportunity.java @@ -24,7 +24,7 @@ public enum BlockingOpportunity { private final String desc; - BlockingOpportunity(String desc){ + BlockingOpportunity(String desc) { this.desc = desc; } diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/CacheType.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/CacheType.java index f845b2083f13..b288a9831abb 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/CacheType.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/CacheType.java @@ -18,6 +18,7 @@ package org.apache.dolphinscheduler.common.enums; public enum CacheType { + TENANT("tenant"), USER("user"), QUEUE("queue"), diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/CommandType.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/CommandType.java index 35b64821aff0..d87bc39b7949 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/CommandType.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/CommandType.java @@ -76,7 +76,7 @@ public String getDescp() { static { for (CommandType commandType : CommandType.values()) { - COMMAND_TYPE_MAP.put(commandType.code,commandType); + COMMAND_TYPE_MAP.put(commandType.code, commandType); } } diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/ComplementDependentMode.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/ComplementDependentMode.java index 68f8e573d22c..04c03edf1b8a 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/ComplementDependentMode.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/ComplementDependentMode.java @@ -23,12 +23,13 @@ * task node depend type */ public enum ComplementDependentMode { + /** * 0 off mode * 1 run complement data with all dependent process */ - OFF_MODE(0,"off mode"), - ALL_DEPENDENT(1,"all dependent"); + OFF_MODE(0, "off mode"), + ALL_DEPENDENT(1, "all dependent"); ComplementDependentMode(int code, String desc) { this.code = code; diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/ConditionType.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/ConditionType.java index a80f1d38b33e..8f690bc8f887 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/ConditionType.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/ConditionType.java @@ -26,6 +26,7 @@ * condition type */ public enum ConditionType { + /** * 0 none * 1 judge diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/Flag.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/Flag.java index 83a7069142f7..80c9cae99914 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/Flag.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/Flag.java @@ -28,6 +28,7 @@ * have_alert */ public enum Flag { + /** * 0 no * 1 yes diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/HttpCheckCondition.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/HttpCheckCondition.java index 278e71e738ad..91a9c14285be 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/HttpCheckCondition.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/HttpCheckCondition.java @@ -27,5 +27,5 @@ public enum HttpCheckCondition { * 2 body_contains * 3 body_not_contains */ - STATUS_CODE_DEFAULT,STATUS_CODE_CUSTOM, BODY_CONTAINS, BODY_NOT_CONTAINS -} \ No newline at end of file + STATUS_CODE_DEFAULT, STATUS_CODE_CUSTOM, BODY_CONTAINS, BODY_NOT_CONTAINS +} diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/HttpMethod.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/HttpMethod.java index 509fd7c14747..4c225ff2cddb 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/HttpMethod.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/HttpMethod.java @@ -29,4 +29,4 @@ public enum HttpMethod { * 4 delete */ GET, POST, HEAD, PUT, DELETE -} \ No newline at end of file +} diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/HttpParametersType.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/HttpParametersType.java index 0f7299ebe509..487ae651e949 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/HttpParametersType.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/HttpParametersType.java @@ -26,5 +26,5 @@ public enum HttpParametersType { * 1 body; * 2 headers; */ - PARAMETER,BODY,HEADERS + PARAMETER, BODY, HEADERS } diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/PluginType.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/PluginType.java index b5d5f7d002e4..08e26b35228a 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/PluginType.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/PluginType.java @@ -28,7 +28,7 @@ public enum PluginType { ALERT(1, "alert", true), REGISTER(2, "register", false), - TASK(3,"task",true); + TASK(3, "task", true); PluginType(int code, String desc, boolean hasUi) { this.code = code; @@ -53,7 +53,6 @@ public boolean getHasUi() { return hasUi; } - private static HashMap PLUGIN_TYPE_MAP = new HashMap<>(); static { diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/Priority.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/Priority.java index 1ccfb6fe4e33..85fd66534528 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/Priority.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/Priority.java @@ -23,6 +23,7 @@ * define process and task priority */ public enum Priority { + /** * 0 highest priority * 1 higher priority diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/ReleaseState.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/ReleaseState.java index 87952becd47b..e11375aa5f36 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/ReleaseState.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/ReleaseState.java @@ -41,12 +41,12 @@ public enum ReleaseState { private final String descp; public static ReleaseState getEnum(int value) { - for (ReleaseState e:ReleaseState.values()) { + for (ReleaseState e : ReleaseState.values()) { if (e.ordinal() == value) { return e; } } - //For values out of enum scope + // For values out of enum scope return null; } diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/RunMode.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/RunMode.java index 9d231cce0096..f55905e74506 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/RunMode.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/RunMode.java @@ -22,7 +22,8 @@ /** * complement data run mode */ -public enum RunMode { +public enum RunMode { + /** * 0 serial run * 1 parallel run diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/SlotCheckState.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/SlotCheckState.java index 731911bc3b7d..b37a3338d143 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/SlotCheckState.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/SlotCheckState.java @@ -19,5 +19,5 @@ public enum SlotCheckState { - PASS,INJECT,CHANGE + PASS, INJECT, CHANGE } diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/SqoopJobType.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/SqoopJobType.java index 2f69f060cc11..17b244c74596 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/SqoopJobType.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/SqoopJobType.java @@ -19,7 +19,8 @@ import com.baomidou.mybatisplus.annotation.EnumValue; -public enum SqoopJobType { +public enum SqoopJobType { + CUSTOM(0, "CUSTOM"), TEMPLATE(1, "TEMPLATE"); diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/TaskDependType.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/TaskDependType.java index 030194756466..7bc8b9d41035 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/TaskDependType.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/TaskDependType.java @@ -23,6 +23,7 @@ * task node depend type */ public enum TaskDependType { + /** * 0 run current tasks only * 1 run current tasks and previous tasks diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/TaskExecuteType.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/TaskExecuteType.java index 4694cec26d8d..c2c7f3a16525 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/TaskExecuteType.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/TaskExecuteType.java @@ -23,6 +23,7 @@ * task execute type */ public enum TaskExecuteType { + /** * 0 batch * 1 stream diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/TaskRecordStatus.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/TaskRecordStatus.java index faa9d0ceabd9..a57ad3fe4706 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/TaskRecordStatus.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/TaskRecordStatus.java @@ -29,7 +29,6 @@ public enum TaskRecordStatus { * 1 failure * 2 exception */ - SUCCESS,FAILURE,EXCEPTION - + SUCCESS, FAILURE, EXCEPTION } diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/TimeoutFlag.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/TimeoutFlag.java index f5ef952b028c..9088575812c7 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/TimeoutFlag.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/TimeoutFlag.java @@ -23,6 +23,7 @@ * timeout flag */ public enum TimeoutFlag { + /** * 0 close * 1 open diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/UdfType.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/UdfType.java index 211061681bc5..9c4266bf7a30 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/UdfType.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/UdfType.java @@ -23,13 +23,14 @@ * UDF type */ public enum UdfType { + /** * 0 hive; 1 spark */ - HIVE(0, "hive"), - SPARK(1, "spark"); + HIVE(0, "hive"), + SPARK(1, "spark"); - UdfType(int code, String descp){ + UdfType(int code, String descp) { this.code = code; this.descp = descp; } @@ -46,14 +47,13 @@ public String getDescp() { return descp; } - public static UdfType of(int type){ - for(UdfType ut : values()){ - if(ut.getCode() == type){ + public static UdfType of(int type) { + for (UdfType ut : values()) { + if (ut.getCode() == type) { return ut; } } throw new IllegalArgumentException("invalid type : " + type); } - } diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/UserType.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/UserType.java index ebab5da74239..effc2cfc1e83 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/UserType.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/UserType.java @@ -23,6 +23,7 @@ * user type */ public enum UserType { + /** * 0 admin user; 1 general user */ @@ -46,4 +47,3 @@ public String getDescp() { return descp; } } - diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/WarningType.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/WarningType.java index 65f31cc91172..2ba69e944e15 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/WarningType.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/WarningType.java @@ -29,6 +29,7 @@ * types for whether to send warning when process ends; */ public enum WarningType { + /** * 0 do not send warning; * 1 send if process success; @@ -58,7 +59,7 @@ public String getDescp() { } private static final Map WARNING_TYPE_MAP = - Arrays.stream(WarningType.values()).collect(toMap(WarningType::getDescp, Functions.identity())); + Arrays.stream(WarningType.values()).collect(toMap(WarningType::getDescp, Functions.identity())); public static WarningType of(String descp) { if (WARNING_TYPE_MAP.containsKey(descp)) { diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/WorkflowExecutionStatus.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/WorkflowExecutionStatus.java index 65e2b8d946f7..4fced361106f 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/WorkflowExecutionStatus.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/WorkflowExecutionStatus.java @@ -17,13 +17,15 @@ package org.apache.dolphinscheduler.common.enums; -import com.baomidou.mybatisplus.annotation.EnumValue; -import lombok.NonNull; - import java.util.HashMap; import java.util.Map; +import lombok.NonNull; + +import com.baomidou.mybatisplus.annotation.EnumValue; + public enum WorkflowExecutionStatus { + // This class is split from ExecutionStatus #11339. // In order to compatible with the old value, the code is not consecutive SUBMITTED_SUCCESS(0, "submit success"), diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/exception/StorageOperateNoConfiguredException.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/exception/StorageOperateNoConfiguredException.java index 3aaf7980d012..b5ab837a8415 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/exception/StorageOperateNoConfiguredException.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/exception/StorageOperateNoConfiguredException.java @@ -37,7 +37,8 @@ public StorageOperateNoConfiguredException(Throwable cause) { super(cause); } - public StorageOperateNoConfiguredException(String message, Throwable cause, boolean enableSuppression, boolean writableStackTrace) { + public StorageOperateNoConfiguredException(String message, Throwable cause, boolean enableSuppression, + boolean writableStackTrace) { super(message, cause, enableSuppression, writableStackTrace); } } diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/graph/DAG.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/graph/DAG.java index afa780163a25..fb5a564cf66c 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/graph/DAG.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/graph/DAG.java @@ -42,6 +42,7 @@ * EdgeInfo: edge description information */ public class DAG { + private static final Logger logger = LoggerFactory.getLogger(DAG.class); private final ReadWriteLock lock = new ReentrantReadWriteLock(); @@ -466,8 +467,7 @@ private Map.Entry> topologicalSortImpl() { } /* - * After scanning, there is no node with 0 degree of entry, - * indicating that there is a ring, and return directly + * After scanning, there is no node with 0 degree of entry, indicating that there is a ring, and return directly */ if (zeroIndegreeNodeQueue.isEmpty()) { return new AbstractMap.SimpleEntry<>(false, topoResultList); @@ -501,13 +501,12 @@ private Map.Entry> topologicalSortImpl() { @Override public String toString() { return "DAG{" - + "nodesMap=" - + nodesMap - + ", edgesMap=" - + edgesMap - + ", reverseEdgesMap=" - + reverseEdgesMap - + '}'; + + "nodesMap=" + + nodesMap + + ", edgesMap=" + + edgesMap + + ", reverseEdgesMap=" + + reverseEdgesMap + + '}'; } } - diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/model/BaseHeartBeatTask.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/model/BaseHeartBeatTask.java index 557f7ec86b51..e703ce0f17ef 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/model/BaseHeartBeatTask.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/model/BaseHeartBeatTask.java @@ -17,10 +17,11 @@ package org.apache.dolphinscheduler.common.model; -import lombok.extern.slf4j.Slf4j; import org.apache.dolphinscheduler.common.lifecycle.ServerLifeCycleManager; import org.apache.dolphinscheduler.common.thread.BaseDaemonThread; +import lombok.extern.slf4j.Slf4j; + @Slf4j public abstract class BaseHeartBeatTask extends BaseDaemonThread { @@ -48,7 +49,8 @@ public void run() { while (runningFlag) { try { if (!ServerLifeCycleManager.isRunning()) { - log.info("The current server status is {}, will not write heartBeatInfo into registry", ServerLifeCycleManager.getServerStatus()); + log.info("The current server status is {}, will not write heartBeatInfo into registry", + ServerLifeCycleManager.getServerStatus()); continue; } T heartBeat = getHeartBeat(); diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/model/MasterHeartBeat.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/model/MasterHeartBeat.java index 6ca51a4c29e9..e9552f507df6 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/model/MasterHeartBeat.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/model/MasterHeartBeat.java @@ -27,6 +27,7 @@ @NoArgsConstructor @AllArgsConstructor public class MasterHeartBeat implements HeartBeat { + private long startupTime; private long reportTime; private double cpuUsage; diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/model/PreviousTaskNode.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/model/PreviousTaskNode.java index 020357578e17..f439184982c5 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/model/PreviousTaskNode.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/model/PreviousTaskNode.java @@ -37,7 +37,7 @@ public class PreviousTaskNode { public PreviousTaskNode() { } - + public PreviousTaskNode(long code, String name, int version) { this.code = code; this.name = name; diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/model/Server.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/model/Server.java index 743979eed763..873a69bcafbc 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/model/Server.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/model/Server.java @@ -17,10 +17,10 @@ package org.apache.dolphinscheduler.common.model; -import lombok.Data; - import java.util.Date; +import lombok.Data; + @Data public class Server { diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/model/TaskNodeRelation.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/model/TaskNodeRelation.java index 7ee83226013b..6abedd39a3e8 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/model/TaskNodeRelation.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/model/TaskNodeRelation.java @@ -60,7 +60,7 @@ public boolean equals(Object o) { if (!(o instanceof TaskNodeRelation)) { return false; } - TaskNodeRelation relation = (TaskNodeRelation)o; + TaskNodeRelation relation = (TaskNodeRelation) o; return (relation.getStartNode().equals(this.startNode) && relation.getEndNode().equals(this.endNode)); } diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/model/WorkerHeartBeat.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/model/WorkerHeartBeat.java index 7980e9bbed6f..4bc5c044bc60 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/model/WorkerHeartBeat.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/model/WorkerHeartBeat.java @@ -44,5 +44,4 @@ public class WorkerHeartBeat implements HeartBeat { private int workerWaitingTaskCount; // worker waiting task count private int workerExecThreadCount; // worker thread pool thread count - } diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/process/HttpProperty.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/process/HttpProperty.java index ab259f4181af..11786fd5a3d6 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/process/HttpProperty.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/process/HttpProperty.java @@ -22,6 +22,7 @@ import java.util.Objects; public class HttpProperty { + /** * key */ @@ -115,9 +116,9 @@ public int hashCode() { @Override public String toString() { return "HttpProperty{" - + "prop='" + prop + '\'' - + ", httpParametersType=" + httpParametersType - + ", value='" + value + '\'' - + '}'; + + "prop='" + prop + '\'' + + ", httpParametersType=" + httpParametersType + + ", value='" + value + '\'' + + '}'; } } diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/shell/AbstractShell.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/shell/AbstractShell.java index c2f63cce149f..913f4500e693 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/shell/AbstractShell.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/shell/AbstractShell.java @@ -39,10 +39,8 @@ * time-intervals. */ public abstract class AbstractShell { - - private static final Logger logger = LoggerFactory.getLogger(AbstractShell.class); - + private static final Logger logger = LoggerFactory.getLogger(AbstractShell.class); /** * Time after which the executing script would be timedout @@ -79,11 +77,11 @@ public abstract class AbstractShell { * If or not script finished executing */ private AtomicBoolean completed; - + public AbstractShell() { this(0L); } - + /** * @param interval the minimum duration to wait before re-executing the * command. @@ -145,7 +143,7 @@ private void runCommand() throws IOException { if (timeOutInterval > 0) { timeOutTimer = new Timer(); timeoutTimerTask = new ShellTimeoutTimerTask(this); - //One time scheduling. + // One time scheduling. timeOutTimer.schedule(timeoutTimerTask, timeOutInterval); } final BufferedReader errReader = @@ -159,6 +157,7 @@ private void runCommand() throws IOException { // read error and input streams as this would free up the buffers // free the error stream buffer Thread errThread = new Thread() { + @Override public void run() { try { @@ -174,6 +173,7 @@ public void run() { } }; Thread inThread = new Thread() { + @Override public void run() { try { @@ -200,9 +200,9 @@ public void run() { } catch (InterruptedException ie) { logger.warn("Interrupted while reading the error and in stream", ie); } - completed.compareAndSet(false,true); - //the timeout thread handling - //taken care in finally block + completed.compareAndSet(false, true); + // the timeout thread handling + // taken care in finally block if (exitCode != 0 || errMsg.length() > 0) { throw new ExitCodeException(exitCode, errMsg.toString()); } @@ -237,7 +237,7 @@ public void run() { * @return an array containing the command name and its parameters */ protected abstract String[] getExecString(); - + /** * Parse the execution result * @param lines lines @@ -285,9 +285,9 @@ public void run() { try { p.exitValue(); } catch (Exception e) { - //Process has not terminated. - //So check if it has completed - //if not just destroy it. + // Process has not terminated. + // So check if it has completed + // if not just destroy it. if (p != null && !shell.completed.get()) { shell.setTimedOut(); p.destroy(); @@ -300,6 +300,7 @@ public void run() { * This is an IOException with exit code added. */ public static class ExitCodeException extends IOException { + int exitCode; public ExitCodeException(int exitCode, String message) { @@ -317,6 +318,7 @@ public int getExitCode() { * */ public static class ProcessContainer extends ConcurrentHashMap { + private static final ProcessContainer container = new ProcessContainer(); private ProcessContainer() { diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/shell/ShellExecutor.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/shell/ShellExecutor.java index ac1eae590a1b..1e12d0ed5fb7 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/shell/ShellExecutor.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/shell/ShellExecutor.java @@ -44,8 +44,8 @@ public ShellExecutor(String[] execString, File dir) { } public ShellExecutor(String[] execString, File dir, - Map env) { - this(execString, dir, env,0L); + Map env) { + this(execString, dir, env, 0L); } /** @@ -63,7 +63,7 @@ public ShellExecutor(String[] execString, File dir, * If 0, the command will not be timed out. */ public ShellExecutor(String[] execString, File dir, - Map env, long timeout) { + Map env, long timeout) { command = execString.clone(); if (dir != null) { setWorkingDirectory(dir); @@ -113,8 +113,7 @@ public static String execCommand(Map env, String[] cmd, * @return the output of the executed command. * @throws IOException errors */ - public static String execCommand(Map env, String... cmd) - throws IOException { + public static String execCommand(Map env, String... cmd) throws IOException { return execCommand(env, cmd, 0L); } @@ -138,7 +137,7 @@ protected void parseExecResult(BufferedReader lines) throws IOException { int nRead; String line = ""; while ((nRead = lines.read(buf, 0, buf.length)) > 0) { - line = new String(buf,0,nRead); + line = new String(buf, 0, nRead); output.append(line); } } @@ -151,7 +150,6 @@ public String getOutput() { return (output == null) ? "" : output.toString(); } - /** * Returns the commands of this instance. * Arguments with spaces in are presented with quotes round; other diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/thread/ThreadUtils.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/thread/ThreadUtils.java index f4f2a17bc719..6eca0e783f49 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/thread/ThreadUtils.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/thread/ThreadUtils.java @@ -21,13 +21,13 @@ import java.util.concurrent.Executors; import java.util.concurrent.ThreadFactory; +import lombok.experimental.UtilityClass; + import org.slf4j.Logger; import org.slf4j.LoggerFactory; import com.google.common.util.concurrent.ThreadFactoryBuilder; -import lombok.experimental.UtilityClass; - @UtilityClass public class ThreadUtils { diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/CollectionUtils.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/CollectionUtils.java index 0ac3b5db012b..ae24894c337e 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/CollectionUtils.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/CollectionUtils.java @@ -75,7 +75,8 @@ public static boolean equalLists(Collection a, Collection b) { * @param T * @return removes certain attributes of each object in the list */ - public static List> getListByExclusion(List originList, Set exclusionSet) { + public static List> getListByExclusion(List originList, + Set exclusionSet) { List> instanceList = new ArrayList<>(); if (originList == null) { return instanceList; diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/DateUtils.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/DateUtils.java index 08890e86fb98..be503dd5b0c8 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/DateUtils.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/DateUtils.java @@ -22,8 +22,6 @@ import org.apache.commons.lang3.StringUtils; -import javax.annotation.Nonnull; -import javax.annotation.Nullable; import java.time.Instant; import java.time.LocalDateTime; import java.time.ZoneId; @@ -33,6 +31,9 @@ import java.util.Date; import java.util.TimeZone; +import javax.annotation.Nonnull; +import javax.annotation.Nullable; + import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -48,7 +49,7 @@ public final class DateUtils { private static final Logger logger = LoggerFactory.getLogger(DateUtils.class); private static final DateTimeFormatter YYYY_MM_DD_HH_MM_SS = - DateTimeFormatter.ofPattern(Constants.YYYY_MM_DD_HH_MM_SS); + DateTimeFormatter.ofPattern(Constants.YYYY_MM_DD_HH_MM_SS); private DateUtils() { throw new UnsupportedOperationException("Construct DateUtils"); @@ -123,7 +124,8 @@ public static String format(Date date, String format, String timezone) { public static String format(Date date, DateTimeFormatter dateTimeFormatter, String timezone) { LocalDateTime localDateTime = - StringUtils.isEmpty(timezone) ? date2LocalDateTime(date) : date2LocalDateTime(date, ZoneId.of(timezone)); + StringUtils.isEmpty(timezone) ? date2LocalDateTime(date) + : date2LocalDateTime(date, ZoneId.of(timezone)); return format(localDateTime, dateTimeFormatter); } @@ -243,8 +245,8 @@ public static ZonedDateTime stringToZoneDateTime(@Nonnull String date) { Date d = stringToDate(date); if (d == null) { throw new IllegalArgumentException(String.format( - "data: %s should be a validate data string - yyyy-MM-dd HH:mm:ss ", - date)); + "data: %s should be a validate data string - yyyy-MM-dd HH:mm:ss ", + date)); } return ZonedDateTime.ofInstant(d.toInstant(), ZoneId.systemDefault()); } @@ -612,9 +614,9 @@ public static Date transformTimezoneDate(Date date, String sourceTimezoneId, Str } String dateToString = dateToString(date, sourceTimezoneId); LocalDateTime localDateTime = - LocalDateTime.parse(dateToString, DateTimeFormatter.ofPattern(Constants.YYYY_MM_DD_HH_MM_SS)); + LocalDateTime.parse(dateToString, DateTimeFormatter.ofPattern(Constants.YYYY_MM_DD_HH_MM_SS)); ZonedDateTime zonedDateTime = - ZonedDateTime.of(localDateTime, TimeZone.getTimeZone(targetTimezoneId).toZoneId()); + ZonedDateTime.of(localDateTime, TimeZone.getTimeZone(targetTimezoneId).toZoneId()); return Date.from(zonedDateTime.toInstant()); } @@ -632,6 +634,7 @@ public static TimeZone getTimezone(String timezoneId) { * Time unit representing one thousandth of a second */ public static class MILLISECONDS { + public static long toDays(long d) { return d / (C6 / C2); } diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/FileUtils.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/FileUtils.java index 23e4b74b7573..beb64167c664 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/FileUtils.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/FileUtils.java @@ -57,7 +57,8 @@ private FileUtils() { * @return download file name */ public static String getDownloadFilename(String filename) { - String fileName = String.format("%s/download/%s/%s", DATA_BASEDIR, DateUtils.getCurrentTime(YYYYMMDDHHMMSS), filename); + String fileName = + String.format("%s/download/%s/%s", DATA_BASEDIR, DateUtils.getCurrentTime(YYYYMMDDHHMMSS), filename); File file = new File(fileName); if (!file.getParentFile().exists()) { @@ -94,7 +95,8 @@ public static String getUploadFilename(String tenantCode, String filename) { * @param taskInstanceId task instance id * @return directory of process execution */ - public static String getProcessExecDir(long projectCode, long processDefineCode, int processDefineVersion, int processInstanceId, int taskInstanceId) { + public static String getProcessExecDir(long projectCode, long processDefineCode, int processDefineVersion, + int processInstanceId, int taskInstanceId) { String fileName = String.format("%s/exec/process/%d/%s/%d/%d", DATA_BASEDIR, projectCode, processDefineCode + "_" + processDefineVersion, processInstanceId, taskInstanceId); File file = new File(fileName); @@ -119,7 +121,7 @@ public static String getResourceViewSuffixes() { * @throws IOException errors */ public static void createWorkDirIfAbsent(String execLocalPath) throws IOException { - //if work dir exists, first delete + // if work dir exists, first delete File execLocalPathFile = new File(execLocalPath); if (execLocalPathFile.exists()) { @@ -134,7 +136,7 @@ public static void createWorkDirIfAbsent(String execLocalPath) throws IOExceptio } } - //create work dir + // create work dir org.apache.commons.io.FileUtils.forceMkdir(execLocalPathFile); String mkdirLog = "create dir success " + execLocalPath; logger.info(mkdirLog); @@ -230,7 +232,7 @@ public static String readFile2Str(InputStream inputStream) { * @param filename String type of filename * @return whether file path could be traversal or not */ - public static boolean directoryTraversal(String filename){ + public static boolean directoryTraversal(String filename) { if (filename.contains(FOLDER_SEPARATOR)) { return true; } diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/HttpUtils.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/HttpUtils.java index 100d60b13116..83557865ff93 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/HttpUtils.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/HttpUtils.java @@ -67,6 +67,7 @@ public static CloseableHttpClient getInstance() { } private static class HttpClientInstance { + private static final CloseableHttpClient httpClient = getHttpClientBuilder().build(); } @@ -85,6 +86,7 @@ public static HttpClientBuilder getHttpClientBuilder() { private static Registry socketFactoryRegistry; private static X509TrustManager xtm = new X509TrustManager() { + @Override public void checkClientTrusted(X509Certificate[] chain, String authType) { } @@ -102,7 +104,7 @@ public X509Certificate[] getAcceptedIssuers() { static { try { ctx = SSLContext.getInstance(SSLConnectionSocketFactory.TLS); - ctx.init(null, new TrustManager[] {xtm}, null); + ctx.init(null, new TrustManager[]{xtm}, null); } catch (NoSuchAlgorithmException e) { logger.error("SSLContext init with NoSuchAlgorithmException", e); } catch (KeyManagementException e) { @@ -111,14 +113,14 @@ public X509Certificate[] getAcceptedIssuers() { socketFactory = new SSLConnectionSocketFactory(ctx, NoopHostnameVerifier.INSTANCE); /** set timeout、request time、socket timeout */ requestConfig = RequestConfig.custom().setCookieSpec(CookieSpecs.IGNORE_COOKIES) - .setExpectContinueEnabled(Boolean.TRUE) - .setTargetPreferredAuthSchemes(Arrays.asList(AuthSchemes.NTLM, AuthSchemes.DIGEST, AuthSchemes.SPNEGO)) - .setProxyPreferredAuthSchemes(Arrays.asList(AuthSchemes.BASIC, AuthSchemes.SPNEGO)) - .setConnectTimeout(Constants.HTTP_CONNECT_TIMEOUT).setSocketTimeout(Constants.SOCKET_TIMEOUT) - .setConnectionRequestTimeout(Constants.HTTP_CONNECTION_REQUEST_TIMEOUT).setRedirectsEnabled(true) - .build(); + .setExpectContinueEnabled(Boolean.TRUE) + .setTargetPreferredAuthSchemes(Arrays.asList(AuthSchemes.NTLM, AuthSchemes.DIGEST, AuthSchemes.SPNEGO)) + .setProxyPreferredAuthSchemes(Arrays.asList(AuthSchemes.BASIC, AuthSchemes.SPNEGO)) + .setConnectTimeout(Constants.HTTP_CONNECT_TIMEOUT).setSocketTimeout(Constants.SOCKET_TIMEOUT) + .setConnectionRequestTimeout(Constants.HTTP_CONNECTION_REQUEST_TIMEOUT).setRedirectsEnabled(true) + .build(); socketFactoryRegistry = RegistryBuilder.create() - .register("http", PlainConnectionSocketFactory.INSTANCE).register("https", socketFactory).build(); + .register("http", PlainConnectionSocketFactory.INSTANCE).register("https", socketFactory).build(); cm = new PoolingHttpClientConnectionManager(socketFactoryRegistry); cm.setDefaultMaxPerRoute(60); cm.setMaxTotal(100); diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/KerberosHttpClient.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/KerberosHttpClient.java index c6d2508af9e7..9262dbd9b98b 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/KerberosHttpClient.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/KerberosHttpClient.java @@ -47,6 +47,7 @@ * kerberos http client */ public class KerberosHttpClient { + public static final Logger logger = LoggerFactory.getLogger(KerberosHttpClient.class); private String principal; @@ -75,6 +76,7 @@ private static CloseableHttpClient buildSpengoHttpClient() { HttpClientBuilder builder = HttpUtils.getHttpClientBuilder(); BasicCredentialsProvider credentialsProvider = new BasicCredentialsProvider(); credentialsProvider.setCredentials(new AuthScope(null, -1, null), new Credentials() { + @Override public Principal getUserPrincipal() { return null; @@ -92,6 +94,7 @@ public String getPassword() { public String get(final String url, final String userId) { logger.info("Calling KerberosHttpClient {} {} {}", this.principal, this.keyTabLocation, url); Configuration config = new Configuration() { + @SuppressWarnings("serial") @Override public AppConfigurationEntry[] getAppConfigurationEntry(String name) { @@ -105,9 +108,9 @@ public AppConfigurationEntry[] getAppConfigurationEntry(String name) { options.put("doNotPrompt", "true"); options.put("isInitiator", "true"); options.put("debug", "true"); - return new AppConfigurationEntry[] { - new AppConfigurationEntry("com.sun.security.auth.module.Krb5LoginModule", - AppConfigurationEntry.LoginModuleControlFlag.REQUIRED, options)}; + return new AppConfigurationEntry[]{ + new AppConfigurationEntry("com.sun.security.auth.module.Krb5LoginModule", + AppConfigurationEntry.LoginModuleControlFlag.REQUIRED, options)}; } }; Set princ = new HashSet<>(1); @@ -140,9 +143,9 @@ public static String get(String url) { String responseContent; KerberosHttpClient kerberosHttpClient = new KerberosHttpClient( - PropertyUtils.getString(Constants.LOGIN_USER_KEY_TAB_USERNAME), - PropertyUtils.getString(Constants.LOGIN_USER_KEY_TAB_PATH), - PropertyUtils.getString(Constants.JAVA_SECURITY_KRB5_CONF_PATH), true); + PropertyUtils.getString(Constants.LOGIN_USER_KEY_TAB_USERNAME), + PropertyUtils.getString(Constants.LOGIN_USER_KEY_TAB_PATH), + PropertyUtils.getString(Constants.JAVA_SECURITY_KRB5_CONF_PATH), true); responseContent = kerberosHttpClient.get(url, PropertyUtils.getString(Constants.LOGIN_USER_KEY_TAB_USERNAME)); return responseContent; diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/RetryerUtils.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/RetryerUtils.java index 138ff25f1677..30c03b0cb8e5 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/RetryerUtils.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/RetryerUtils.java @@ -30,6 +30,7 @@ import com.github.rholder.retry.WaitStrategies; public class RetryerUtils { + private static Retryer defaultRetryerResultCheck; private static Retryer defaultRetryerResultNoCheck; @@ -92,7 +93,8 @@ public static Retryer getDefaultRetryer() { * @throws ExecutionException the execution exception * @throws RetryException the retry exception */ - public static Boolean retryCall(final Callable callable, boolean checkResult) throws ExecutionException, RetryException { + public static Boolean retryCall(final Callable callable, + boolean checkResult) throws ExecutionException, RetryException { return getDefaultRetryer(checkResult).call(callable); } diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/placeholder/BusinessTimeUtils.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/placeholder/BusinessTimeUtils.java index ab92df8f9bf4..26be151941b3 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/placeholder/BusinessTimeUtils.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/placeholder/BusinessTimeUtils.java @@ -17,12 +17,11 @@ package org.apache.dolphinscheduler.common.utils.placeholder; +import static org.apache.commons.lang3.time.DateUtils.addDays; import static org.apache.dolphinscheduler.common.Constants.PARAMETER_FORMAT_DATE; import static org.apache.dolphinscheduler.common.Constants.PARAMETER_FORMAT_TIME; import static org.apache.dolphinscheduler.common.utils.DateUtils.format; -import static org.apache.commons.lang3.time.DateUtils.addDays; - import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.enums.CommandType; @@ -34,6 +33,7 @@ * business time utils */ public class BusinessTimeUtils { + private BusinessTimeUtils() { throw new IllegalStateException("BusinessTimeUtils class"); } diff --git a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/ConstantsTest.java b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/ConstantsTest.java index 96b438031dfe..7026b2a87df8 100644 --- a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/ConstantsTest.java +++ b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/ConstantsTest.java @@ -18,6 +18,7 @@ package org.apache.dolphinscheduler.common; import org.apache.commons.lang3.SystemUtils; + import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; diff --git a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/exception/ExceptionTest.java b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/exception/ExceptionTest.java index e49d8d9b296e..6d7d7fe799b1 100644 --- a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/exception/ExceptionTest.java +++ b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/exception/ExceptionTest.java @@ -17,14 +17,13 @@ package org.apache.dolphinscheduler.common.exception; - import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; public class ExceptionTest { @Test - public void testException(){ + public void testException() { final String message = "Test"; RuntimeException time = new RuntimeException(message); diff --git a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/graph/DAGTest.java b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/graph/DAGTest.java index deff8a2ea59f..5b3958dbf5f7 100644 --- a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/graph/DAGTest.java +++ b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/graph/DAGTest.java @@ -16,6 +16,10 @@ */ package org.apache.dolphinscheduler.common.graph; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.BeforeEach; @@ -23,341 +27,322 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; - public class DAGTest { - private DAG graph; - private static final Logger logger = LoggerFactory.getLogger(DAGTest.class); - - @BeforeEach - public void setup() { - graph = new DAG<>(); - } - - @AfterEach - public void tearDown() { - clear(); - } - - private void clear() { - graph = null; - graph = new DAG<>(); - Assertions.assertEquals(graph.getNodesCount(), 0); - } + private DAG graph; + private static final Logger logger = LoggerFactory.getLogger(DAGTest.class); - - private void makeGraph() { - clear(); - - // 1->2 - // 2->5 - // 3->5 - // 4->6 - // 5->6 - // 6->7 - - for (int i = 1; i <= 7; ++i) { - graph.addNode(i, "v(" + i + ")"); + @BeforeEach + public void setup() { + graph = new DAG<>(); } - // construction side - Assertions.assertTrue(graph.addEdge(1, 2)); - - Assertions.assertTrue(graph.addEdge(2, 5)); - - Assertions.assertTrue(graph.addEdge(3, 5)); - - Assertions.assertTrue(graph.addEdge(4, 6)); - - Assertions.assertTrue(graph.addEdge(5, 6)); - - Assertions.assertTrue(graph.addEdge(6, 7)); - - Assertions.assertEquals(graph.getNodesCount(), 7); - Assertions.assertEquals(graph.getEdgesCount(), 6); - - } - - - /** - * add node - */ - @Test - public void testAddNode() { - clear(); - - graph.addNode(1, "v(1)"); - graph.addNode(2, null); - graph.addNode(5, "v(5)"); - - Assertions.assertEquals(graph.getNodesCount(), 3); - - Assertions.assertEquals(graph.getNode(1), "v(1)"); - Assertions.assertTrue(graph.containsNode(1)); + @AfterEach + public void tearDown() { + clear(); + } - Assertions.assertFalse(graph.containsNode(10)); - } + private void clear() { + graph = null; + graph = new DAG<>(); + Assertions.assertEquals(graph.getNodesCount(), 0); + } - /** - * add edge - */ - @Test - public void testAddEdge() { - clear(); + private void makeGraph() { + clear(); - Assertions.assertFalse(graph.addEdge(1, 2, "edge(1 -> 2)", false)); + // 1->2 + // 2->5 + // 3->5 + // 4->6 + // 5->6 + // 6->7 - graph.addNode(1, "v(1)"); + for (int i = 1; i <= 7; ++i) { + graph.addNode(i, "v(" + i + ")"); + } - Assertions.assertTrue(graph.addEdge(1, 2, "edge(1 -> 2)",true)); + // construction side + Assertions.assertTrue(graph.addEdge(1, 2)); - graph.addNode(2, "v(2)"); + Assertions.assertTrue(graph.addEdge(2, 5)); - Assertions.assertTrue(graph.addEdge(1, 2, "edge(1 -> 2)",true)); + Assertions.assertTrue(graph.addEdge(3, 5)); - Assertions.assertFalse(graph.containsEdge(1, 3)); + Assertions.assertTrue(graph.addEdge(4, 6)); - Assertions.assertTrue(graph.containsEdge(1, 2)); - Assertions.assertEquals(graph.getEdgesCount(), 1); + Assertions.assertTrue(graph.addEdge(5, 6)); - int node = 3; - graph.addNode(node, "v(3)"); - Assertions.assertFalse(graph.addEdge(node, node)); + Assertions.assertTrue(graph.addEdge(6, 7)); - } + Assertions.assertEquals(graph.getNodesCount(), 7); + Assertions.assertEquals(graph.getEdgesCount(), 6); + } - /** - * add subsequent node - */ - @Test - public void testSubsequentNodes() { - makeGraph(); + /** + * add node + */ + @Test + public void testAddNode() { + clear(); - Assertions.assertEquals(graph.getSubsequentNodes(1).size(), 1); + graph.addNode(1, "v(1)"); + graph.addNode(2, null); + graph.addNode(5, "v(5)"); - } + Assertions.assertEquals(graph.getNodesCount(), 3); + Assertions.assertEquals(graph.getNode(1), "v(1)"); + Assertions.assertTrue(graph.containsNode(1)); - /** - * test indegree - */ - @Test - public void testIndegree() { - makeGraph(); + Assertions.assertFalse(graph.containsNode(10)); + } - Assertions.assertEquals(graph.getIndegree(1), 0); - Assertions.assertEquals(graph.getIndegree(2), 1); - Assertions.assertEquals(graph.getIndegree(3), 0); - Assertions.assertEquals(graph.getIndegree(4), 0); - } + /** + * add edge + */ + @Test + public void testAddEdge() { + clear(); + Assertions.assertFalse(graph.addEdge(1, 2, "edge(1 -> 2)", false)); - /** - * test begin node - */ - @Test - public void testBeginNode() { - makeGraph(); + graph.addNode(1, "v(1)"); - Assertions.assertEquals(graph.getBeginNode().size(), 3); + Assertions.assertTrue(graph.addEdge(1, 2, "edge(1 -> 2)", true)); - Assertions.assertTrue(graph.getBeginNode().contains(1)); - Assertions.assertTrue(graph.getBeginNode().contains(3)); - Assertions.assertTrue(graph.getBeginNode().contains(4)); - } + graph.addNode(2, "v(2)"); + Assertions.assertTrue(graph.addEdge(1, 2, "edge(1 -> 2)", true)); - /** - * test end node - */ - @Test - public void testEndNode() { - makeGraph(); + Assertions.assertFalse(graph.containsEdge(1, 3)); - Assertions.assertEquals(graph.getEndNode().size(), 1); + Assertions.assertTrue(graph.containsEdge(1, 2)); + Assertions.assertEquals(graph.getEdgesCount(), 1); - Assertions.assertTrue(graph.getEndNode().contains(7)); - } + int node = 3; + graph.addNode(node, "v(3)"); + Assertions.assertFalse(graph.addEdge(node, node)); + } - /** - * test cycle - */ - @Test - public void testCycle() { - clear(); + /** + * add subsequent node + */ + @Test + public void testSubsequentNodes() { + makeGraph(); + Assertions.assertEquals(graph.getSubsequentNodes(1).size(), 1); - for (int i = 1; i <= 5; ++i) { - graph.addNode(i, "v(" + i + ")"); } - // construction side - try { - graph.addEdge(1, 2); - graph.addEdge(2, 3); - graph.addEdge(3, 4); - - Assertions.assertFalse(graph.hasCycle()); - } catch (Exception e) { - e.printStackTrace(); - Assertions.fail(); + /** + * test indegree + */ + @Test + public void testIndegree() { + makeGraph(); + + Assertions.assertEquals(graph.getIndegree(1), 0); + Assertions.assertEquals(graph.getIndegree(2), 1); + Assertions.assertEquals(graph.getIndegree(3), 0); + Assertions.assertEquals(graph.getIndegree(4), 0); } + /** + * test begin node + */ + @Test + public void testBeginNode() { + makeGraph(); - try { - boolean addResult = graph.addEdge(4, 1); + Assertions.assertEquals(graph.getBeginNode().size(), 3); - if(!addResult){ - Assertions.assertTrue(true); - } - - graph.addEdge(5, 1); - - Assertions.assertFalse(graph.hasCycle()); - } catch (Exception e) { - e.printStackTrace(); - Assertions.fail(); + Assertions.assertTrue(graph.getBeginNode().contains(1)); + Assertions.assertTrue(graph.getBeginNode().contains(3)); + Assertions.assertTrue(graph.getBeginNode().contains(4)); } - clear(); + /** + * test end node + */ + @Test + public void testEndNode() { + makeGraph(); + + Assertions.assertEquals(graph.getEndNode().size(), 1); - // construction node - for (int i = 1; i <= 5; ++i) { - graph.addNode(i, "v(" + i +")"); + Assertions.assertTrue(graph.getEndNode().contains(7)); } - // construction side, 1->2, 2->3, 3->4 - try { - graph.addEdge(1, 2); - graph.addEdge(2, 3); - graph.addEdge(3, 4); - graph.addEdge(4, 5); - graph.addEdge(5, 2);//会失败，添加不进去，所以下一步无环 - - Assertions.assertFalse(graph.hasCycle()); - } catch (Exception e) { - e.printStackTrace(); - Assertions.fail(); + /** + * test cycle + */ + @Test + public void testCycle() { + clear(); + + for (int i = 1; i <= 5; ++i) { + graph.addNode(i, "v(" + i + ")"); + } + + // construction side + try { + graph.addEdge(1, 2); + graph.addEdge(2, 3); + graph.addEdge(3, 4); + + Assertions.assertFalse(graph.hasCycle()); + } catch (Exception e) { + e.printStackTrace(); + Assertions.fail(); + } + + try { + boolean addResult = graph.addEdge(4, 1); + + if (!addResult) { + Assertions.assertTrue(true); + } + + graph.addEdge(5, 1); + + Assertions.assertFalse(graph.hasCycle()); + } catch (Exception e) { + e.printStackTrace(); + Assertions.fail(); + } + + clear(); + + // construction node + for (int i = 1; i <= 5; ++i) { + graph.addNode(i, "v(" + i + ")"); + } + + // construction side, 1->2, 2->3, 3->4 + try { + graph.addEdge(1, 2); + graph.addEdge(2, 3); + graph.addEdge(3, 4); + graph.addEdge(4, 5); + graph.addEdge(5, 2);// 会失败，添加不进去，所以下一步无环 + + Assertions.assertFalse(graph.hasCycle()); + } catch (Exception e) { + e.printStackTrace(); + Assertions.fail(); + } } - } - - - @Test - public void testTopologicalSort(){ - makeGraph(); - - try { - // topological result is : 1 3 4 2 5 6 7 - List topoList = new ArrayList<>(); - topoList.add(1); - topoList.add(3); - topoList.add(4); - topoList.add(2); - topoList.add(5); - topoList.add(6); - topoList.add(7); - - Assertions.assertEquals(graph.topologicalSort(),topoList); - } catch (Exception e) { - e.printStackTrace(); - Assertions.fail(); + + @Test + public void testTopologicalSort() { + makeGraph(); + + try { + // topological result is : 1 3 4 2 5 6 7 + List topoList = new ArrayList<>(); + topoList.add(1); + topoList.add(3); + topoList.add(4); + topoList.add(2); + topoList.add(5); + topoList.add(6); + topoList.add(7); + + Assertions.assertEquals(graph.topologicalSort(), topoList); + } catch (Exception e) { + e.printStackTrace(); + Assertions.fail(); + } } - } + @Test + public void testTopologicalSort2() { + clear(); - @Test - public void testTopologicalSort2() { - clear(); + graph.addEdge(1, 2, null, true); + graph.addEdge(2, 3, null, true); + graph.addEdge(3, 4, null, true); + graph.addEdge(4, 5, null, true); + graph.addEdge(5, 1, null, false); // The loop will fail to add - graph.addEdge(1, 2, null, true); - graph.addEdge(2, 3, null, true); - graph.addEdge(3, 4, null, true); - graph.addEdge(4, 5, null, true); - graph.addEdge(5, 1, null, false); //The loop will fail to add + try { + List topoList = new ArrayList<>();// topological result is : 1 2 3 4 5 + topoList.add(1); + topoList.add(2); + topoList.add(3); + topoList.add(4); + topoList.add(5); - try { - List topoList = new ArrayList<>();// topological result is : 1 2 3 4 5 - topoList.add(1); - topoList.add(2); - topoList.add(3); - topoList.add(4); - topoList.add(5); + Assertions.assertEquals(graph.topologicalSort(), topoList); - Assertions.assertEquals(graph.topologicalSort(),topoList); + } catch (Exception e) { + e.printStackTrace(); + Assertions.fail(); + } - } catch (Exception e) { - e.printStackTrace(); - Assertions.fail(); } - } - + @Test + public void testTopologicalSort3() throws Exception { + clear(); - @Test - public void testTopologicalSort3() throws Exception { - clear(); + // 1->2 + // 1->3 + // 2->5 + // 3->4 + // 4->6 + // 5->6 + // 6->7 + // 6->8 - // 1->2 - // 1->3 - // 2->5 - // 3->4 - // 4->6 - // 5->6 - // 6->7 - // 6->8 + for (int i = 1; i <= 8; ++i) { + graph.addNode(i, "v(" + i + ")"); + } - for (int i = 1; i <= 8; ++i) { - graph.addNode(i, "v(" + i + ")"); - } - - // construction node - Assertions.assertTrue(graph.addEdge(1, 2)); - - Assertions.assertTrue(graph.addEdge(1, 3)); + // construction node + Assertions.assertTrue(graph.addEdge(1, 2)); - Assertions.assertTrue(graph.addEdge(2, 5)); - Assertions.assertTrue(graph.addEdge(3, 4)); + Assertions.assertTrue(graph.addEdge(1, 3)); - Assertions.assertTrue(graph.addEdge(4, 6)); + Assertions.assertTrue(graph.addEdge(2, 5)); + Assertions.assertTrue(graph.addEdge(3, 4)); - Assertions.assertTrue(graph.addEdge(5, 6)); + Assertions.assertTrue(graph.addEdge(4, 6)); - Assertions.assertTrue(graph.addEdge(6, 7)); - Assertions.assertTrue(graph.addEdge(6, 8)); + Assertions.assertTrue(graph.addEdge(5, 6)); + Assertions.assertTrue(graph.addEdge(6, 7)); + Assertions.assertTrue(graph.addEdge(6, 8)); + Assertions.assertEquals(graph.getNodesCount(), 8); + logger.info(Arrays.toString(graph.topologicalSort().toArray())); - Assertions.assertEquals(graph.getNodesCount(), 8); + List expectedList = new ArrayList<>(); - logger.info(Arrays.toString(graph.topologicalSort().toArray())); + for (int i = 1; i <= 8; ++i) { + expectedList.add(i); - List expectedList = new ArrayList<>(); + logger.info(i + " subsequentNodes : " + graph.getSubsequentNodes(i)); + } + logger.info(6 + " previousNodesb: " + graph.getPreviousNodes(6)); + Assertions.assertEquals(5, graph.getSubsequentNodes(2).toArray()[0]); - for (int i = 1; i <= 8; ++i) { - expectedList.add(i); - - logger.info(i + " subsequentNodes : " + graph.getSubsequentNodes(i)); } - logger.info(6 + " previousNodesb: " + graph.getPreviousNodes(6)); - Assertions.assertEquals(5, graph.getSubsequentNodes(2).toArray()[0]); - - } - - @Test - public void testTopologicalSort4() { - clear(); - try { - graph.topologicalSort(); - } catch (Exception e) { - Assertions.assertTrue(e.getMessage().contains("serious error: graph has cycle")); + + @Test + public void testTopologicalSort4() { + clear(); + try { + graph.topologicalSort(); + } catch (Exception e) { + Assertions.assertTrue(e.getMessage().contains("serious error: graph has cycle")); + } } - } } diff --git a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/os/OSUtilsTest.java b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/os/OSUtilsTest.java index 7525d1c76e0b..561f2a73f2f7 100644 --- a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/os/OSUtilsTest.java +++ b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/os/OSUtilsTest.java @@ -16,15 +16,17 @@ */ package org.apache.dolphinscheduler.common.os; -import org.apache.commons.lang3.SystemUtils; import org.apache.dolphinscheduler.common.utils.OSUtils; + +import org.apache.commons.lang3.SystemUtils; + +import java.util.List; + import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.util.List; - /** * OSUtilsTest */ diff --git a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/CodeGenerateUtilsTest.java b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/CodeGenerateUtilsTest.java index e484b61f8dd6..3871646c95c8 100644 --- a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/CodeGenerateUtilsTest.java +++ b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/CodeGenerateUtilsTest.java @@ -17,12 +17,13 @@ package org.apache.dolphinscheduler.common.utils; +import java.util.HashSet; + import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; -import java.util.HashSet; - public class CodeGenerateUtilsTest { + @Test public void testNoGenerateDuplicateCode() throws CodeGenerateUtils.CodeGenerateException { HashSet existsCode = new HashSet<>(); diff --git a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/DateUtilsTest.java b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/DateUtilsTest.java index 36ffe1134a98..dc07c1cdaa6a 100644 --- a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/DateUtilsTest.java +++ b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/DateUtilsTest.java @@ -18,12 +18,7 @@ package org.apache.dolphinscheduler.common.utils; import org.apache.dolphinscheduler.common.thread.ThreadLocalContext; -import org.junit.jupiter.api.AfterEach; -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import javax.management.timer.Timer; import java.text.ParseException; import java.text.SimpleDateFormat; import java.time.ZoneId; @@ -31,6 +26,13 @@ import java.util.Date; import java.util.TimeZone; +import javax.management.timer.Timer; + +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + public class DateUtilsTest { @BeforeEach @@ -226,7 +228,8 @@ public void testTransformToTimezone() { Assertions.assertEquals(DateUtils.dateToString(date), DateUtils.dateToString(defaultTimeZoneDate)); Date targetTimeZoneDate = DateUtils.transformTimezoneDate(date, TimeZone.getDefault().getID(), "Asia/Shanghai"); - Assertions.assertEquals(DateUtils.dateToString(date, TimeZone.getDefault().getID()), DateUtils.dateToString(targetTimeZoneDate, "Asia/Shanghai")); + Assertions.assertEquals(DateUtils.dateToString(date, TimeZone.getDefault().getID()), + DateUtils.dateToString(targetTimeZoneDate, "Asia/Shanghai")); } @Test @@ -286,7 +289,7 @@ public void testDateToTimeStamp() throws ParseException { @Test public void testTimeStampToDate() { long timeStamp = 1664456400000L; - SimpleDateFormat sdf=new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); + SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); sdf.setTimeZone(TimeZone.getTimeZone("Asia/Shanghai")); String sd = sdf.format(new Date(timeStamp)); Assertions.assertEquals("2022-09-29 21:00:00", sd); diff --git a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/EncryptionUtilsTest.java b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/EncryptionUtilsTest.java index 28db71b5a7c8..9f9710efd02c 100644 --- a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/EncryptionUtilsTest.java +++ b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/EncryptionUtilsTest.java @@ -24,7 +24,6 @@ */ public class EncryptionUtilsTest { - @Test public void testGetMd5() { Assertions.assertEquals(EncryptionUtils.getMd5(null), EncryptionUtils.getMd5("")); diff --git a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/FileUtilsTest.java b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/FileUtilsTest.java index c5e6e3354bbf..7fbd77d8bc8f 100644 --- a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/FileUtilsTest.java +++ b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/FileUtilsTest.java @@ -17,7 +17,13 @@ package org.apache.dolphinscheduler.common.utils; +import static org.apache.dolphinscheduler.common.Constants.YYYYMMDDHHMMSS; + import org.apache.dolphinscheduler.common.Constants; + +import java.io.FileInputStream; +import java.io.FileNotFoundException; + import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; @@ -25,11 +31,6 @@ import org.mockito.Mockito; import org.mockito.junit.jupiter.MockitoExtension; -import java.io.FileInputStream; -import java.io.FileNotFoundException; - -import static org.apache.dolphinscheduler.common.Constants.YYYYMMDDHHMMSS; - @ExtendWith(MockitoExtension.class) public class FileUtilsTest { diff --git a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/LocalJettyHttpServer.java b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/LocalJettyHttpServer.java index c4ff8bfc1336..1275784cdccc 100644 --- a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/LocalJettyHttpServer.java +++ b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/LocalJettyHttpServer.java @@ -19,19 +19,24 @@ import java.io.IOException; import java.io.OutputStream; -import java.net.BindException; + +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; + import junit.extensions.TestSetup; import junit.framework.Test; -import org.mortbay.jetty.*; + +import org.mortbay.jetty.HttpConnection; +import org.mortbay.jetty.Request; +import org.mortbay.jetty.Server; import org.mortbay.jetty.handler.AbstractHandler; import org.mortbay.jetty.handler.ContextHandler; import org.mortbay.util.ByteArrayISO8859Writer; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import javax.servlet.http.HttpServletRequest; -import javax.servlet.http.HttpServletResponse; public class LocalJettyHttpServer extends TestSetup { + protected static Server server; private static Logger logger = LoggerFactory.getLogger(LocalJettyHttpServer.class); private Integer serverPort = 0; @@ -44,13 +49,14 @@ public LocalJettyHttpServer(Test suite) { super(suite); } - protected void setUp() throws Exception { server = new Server(serverPort); ContextHandler context = new ContextHandler("/test.json"); context.setHandler(new AbstractHandler() { + @Override - public void handle(String s, HttpServletRequest request, HttpServletResponse response, int i) throws IOException { + public void handle(String s, HttpServletRequest request, HttpServletResponse response, + int i) throws IOException { ByteArrayISO8859Writer writer = new ByteArrayISO8859Writer(); writer.write("{\"name\":\"Github\"}"); writer.flush(); @@ -58,7 +64,8 @@ public void handle(String s, HttpServletRequest request, HttpServletResponse res OutputStream out = response.getOutputStream(); writer.writeTo(out); out.flush(); - Request baseRequest = request instanceof Request ? (Request) request : HttpConnection.getCurrentConnection().getRequest(); + Request baseRequest = request instanceof Request ? (Request) request + : HttpConnection.getCurrentConnection().getRequest(); baseRequest.setHandled(true); } }); diff --git a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/LocalServerHttpUtilsTest.java b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/LocalServerHttpUtilsTest.java index f03066ef49a2..ac1e75bab7b2 100644 --- a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/LocalServerHttpUtilsTest.java +++ b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/LocalServerHttpUtilsTest.java @@ -17,19 +17,23 @@ package org.apache.dolphinscheduler.common.utils; -import com.fasterxml.jackson.databind.node.ObjectNode; -import junit.framework.Test; -import junit.framework.TestCase; -import junit.framework.TestSuite; import org.apache.dolphinscheduler.common.Constants; + import org.apache.http.client.config.RequestConfig; import org.apache.http.client.methods.HttpGet; import org.apache.http.impl.client.CloseableHttpClient; import org.apache.http.impl.client.HttpClients; + +import junit.framework.Test; +import junit.framework.TestCase; +import junit.framework.TestSuite; + import org.junit.jupiter.api.Assertions; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import com.fasterxml.jackson.databind.node.ObjectNode; + public class LocalServerHttpUtilsTest extends TestCase { public static final Logger logger = LoggerFactory.getLogger(LocalServerHttpUtilsTest.class); @@ -58,8 +62,8 @@ public void testGetResponseContentString() { HttpGet httpget = new HttpGet("http://localhost:" + server.getServerPort() + "/test.json"); /** set timeout、request time、socket timeout */ RequestConfig requestConfig = RequestConfig.custom().setConnectTimeout(Constants.HTTP_CONNECT_TIMEOUT) - .setConnectionRequestTimeout(Constants.HTTP_CONNECTION_REQUEST_TIMEOUT) - .setSocketTimeout(Constants.SOCKET_TIMEOUT).setRedirectsEnabled(true).build(); + .setConnectionRequestTimeout(Constants.HTTP_CONNECTION_REQUEST_TIMEOUT) + .setSocketTimeout(Constants.SOCKET_TIMEOUT).setRedirectsEnabled(true).build(); httpget.setConfig(requestConfig); String responseContent = null; diff --git a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/NetUtilsTest.java b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/NetUtilsTest.java index 20625f29c2ee..8a17395784ce 100644 --- a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/NetUtilsTest.java +++ b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/NetUtilsTest.java @@ -45,7 +45,8 @@ public void testGetHostInKubernetesMode() { when(address.getCanonicalHostName()) .thenReturn("dolphinscheduler-worker-0.dolphinscheduler-worker-headless.default.svc.cluster.local"); when(address.getHostName()).thenReturn("dolphinscheduler-worker-0"); - Assertions.assertEquals("dolphinscheduler-worker-0.dolphinscheduler-worker-headless", NetUtils.getHost(address)); + Assertions.assertEquals("dolphinscheduler-worker-0.dolphinscheduler-worker-headless", + NetUtils.getHost(address)); address = mock(InetAddress.class); when(address.getCanonicalHostName()) diff --git a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/PropertyUtilsTest.java b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/PropertyUtilsTest.java index b4c6426372dd..49e1dd35ed92 100644 --- a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/PropertyUtilsTest.java +++ b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/PropertyUtilsTest.java @@ -18,6 +18,7 @@ package org.apache.dolphinscheduler.common.utils; import org.apache.dolphinscheduler.common.Constants; + import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; diff --git a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/RetryerUtilsTest.java b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/RetryerUtilsTest.java index 8fcbdea5d0fc..a1a157337b59 100644 --- a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/RetryerUtilsTest.java +++ b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/RetryerUtilsTest.java @@ -16,12 +16,13 @@ */ package org.apache.dolphinscheduler.common.utils; -import com.github.rholder.retry.RetryException; -import com.github.rholder.retry.Retryer; +import java.util.concurrent.ExecutionException; + import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; -import java.util.concurrent.ExecutionException; +import com.github.rholder.retry.RetryException; +import com.github.rholder.retry.Retryer; public class RetryerUtilsTest { diff --git a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/ScriptRunnerTest.java b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/ScriptRunnerTest.java index cb983ae450f1..7b1b96b56fd9 100644 --- a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/ScriptRunnerTest.java +++ b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/ScriptRunnerTest.java @@ -16,15 +16,16 @@ */ package org.apache.dolphinscheduler.common.utils; +import java.io.StringReader; + import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; -import java.io.StringReader; - public class ScriptRunnerTest { + @Test public void testRunScript() { - //connection is null + // connection is null Exception exception = null; ScriptRunner s = new ScriptRunner(null, true, true); try { diff --git a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/StringTest.java b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/StringTest.java index d818eef21a56..04bf267a51eb 100644 --- a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/StringTest.java +++ b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/StringTest.java @@ -16,24 +16,24 @@ */ package org.apache.dolphinscheduler.common.utils; -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.Test; - import java.util.ArrayList; import java.util.List; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + public class StringTest { @Test - public void stringCompareTest(){ + public void stringCompareTest() { - for(int j = 0; j < 5; j++) { + for (int j = 0; j < 5; j++) { long start = System.currentTimeMillis(); int size = 10000; List taskList = new ArrayList<>(size); - //init + // init for (int i = 0; i < size; i++) { taskList.add(String.format("%d_%010d_%010d", 1, i, i + 1)); } diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/AlertDao.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/AlertDao.java index dfdf4521a6e3..470dffb58866 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/AlertDao.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/AlertDao.java @@ -17,7 +17,6 @@ package org.apache.dolphinscheduler.dao; -import org.apache.commons.lang3.math.NumberUtils; import org.apache.dolphinscheduler.common.enums.AlertEvent; import org.apache.dolphinscheduler.common.enums.AlertStatus; import org.apache.dolphinscheduler.common.enums.AlertType; @@ -39,6 +38,7 @@ import org.apache.commons.codec.digest.DigestUtils; import org.apache.commons.collections4.CollectionUtils; +import org.apache.commons.lang3.math.NumberUtils; import java.time.LocalDateTime; import java.time.ZoneId; @@ -67,7 +67,7 @@ public class AlertDao { * logger of AlertDao */ private static final Logger logger = LoggerFactory.getLogger(AlertDao.class); - + private static final int QUERY_ALERT_THRESHOLD = 100; @Value("${alert.alarm-suppression.crash:60}") diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/CycleDependency.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/CycleDependency.java index 02f9dbd75d03..109a204f114e 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/CycleDependency.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/CycleDependency.java @@ -25,6 +25,7 @@ * cycle dependency */ public class CycleDependency { + /** * process define id */ diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/DagData.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/DagData.java index b6754a3b67de..f180627ac66f 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/DagData.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/DagData.java @@ -39,7 +39,8 @@ public class DagData { */ private List taskDefinitionList; - public DagData(ProcessDefinition processDefinition, List processTaskRelationList, List taskDefinitionList) { + public DagData(ProcessDefinition processDefinition, List processTaskRelationList, + List taskDefinitionList) { this.processDefinition = processDefinition; this.processTaskRelationList = processTaskRelationList; this.taskDefinitionList = taskDefinitionList; diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/DataSource.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/DataSource.java index a80a7e06d35e..441f1be8529e 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/DataSource.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/DataSource.java @@ -17,13 +17,13 @@ package org.apache.dolphinscheduler.dao.entity; -import com.baomidou.mybatisplus.annotation.*; import org.apache.dolphinscheduler.spi.enums.DbType; import java.util.Date; import lombok.Data; +import com.baomidou.mybatisplus.annotation.FieldFill; import com.baomidou.mybatisplus.annotation.IdType; import com.baomidou.mybatisplus.annotation.TableField; import com.baomidou.mybatisplus.annotation.TableId; diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/DependentProcessDefinition.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/DependentProcessDefinition.java index 87bb3d4234d9..17f60f928c99 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/DependentProcessDefinition.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/DependentProcessDefinition.java @@ -89,13 +89,13 @@ public CycleEnum cycle2CycleEnum(String cycle) { cycleEnum = CycleEnum.DAY; break; case "hour": - cycleEnum = CycleEnum.HOUR; + cycleEnum = CycleEnum.HOUR; break; case "week": - cycleEnum = CycleEnum.WEEK; + cycleEnum = CycleEnum.WEEK; break; case "month": - cycleEnum = CycleEnum.MONTH; + cycleEnum = CycleEnum.MONTH; break; default: break; diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/DqExecuteResultAlertContent.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/DqExecuteResultAlertContent.java index cd7f24b08175..7ec52868b269 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/DqExecuteResultAlertContent.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/DqExecuteResultAlertContent.java @@ -35,7 +35,7 @@ public class DqExecuteResultAlertContent implements Serializable { * process define name */ @JsonProperty("processDefinitionName") - private String processDefinitionName; + private String processDefinitionName; /** * process_instance_id */ @@ -141,8 +141,9 @@ public static Builder newBuilder() { } public static class Builder { + private long processDefinitionId; - private String processDefinitionName; + private String processDefinitionName; private long processInstanceId; private String processInstanceName; private long taskInstanceId; diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/ErrorCommand.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/ErrorCommand.java index d83a3dc624b6..efacc1c92b40 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/ErrorCommand.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/ErrorCommand.java @@ -17,7 +17,6 @@ package org.apache.dolphinscheduler.dao.entity; -import com.baomidou.mybatisplus.annotation.TableField; import org.apache.dolphinscheduler.common.enums.CommandType; import org.apache.dolphinscheduler.common.enums.FailureStrategy; import org.apache.dolphinscheduler.common.enums.Priority; @@ -29,6 +28,7 @@ import lombok.Data; import com.baomidou.mybatisplus.annotation.IdType; +import com.baomidou.mybatisplus.annotation.TableField; import com.baomidou.mybatisplus.annotation.TableId; import com.baomidou.mybatisplus.annotation.TableName; diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/ExecuteStatusCount.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/ExecuteStatusCount.java index ffd45e49faa1..453040204083 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/ExecuteStatusCount.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/ExecuteStatusCount.java @@ -17,10 +17,11 @@ package org.apache.dolphinscheduler.dao.entity; +import org.apache.dolphinscheduler.plugin.task.api.enums.TaskExecutionStatus; + import lombok.AllArgsConstructor; import lombok.Data; import lombok.NoArgsConstructor; -import org.apache.dolphinscheduler.plugin.task.api.enums.TaskExecutionStatus; @Data @NoArgsConstructor diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/FavTask.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/FavTask.java index 3f2f85896444..e9e3c8864221 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/FavTask.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/FavTask.java @@ -17,14 +17,15 @@ package org.apache.dolphinscheduler.dao.entity; -import com.baomidou.mybatisplus.annotation.IdType; -import com.baomidou.mybatisplus.annotation.TableId; -import com.baomidou.mybatisplus.annotation.TableName; import lombok.AllArgsConstructor; import lombok.Getter; import lombok.NoArgsConstructor; import lombok.Setter; +import com.baomidou.mybatisplus.annotation.IdType; +import com.baomidou.mybatisplus.annotation.TableId; +import com.baomidou.mybatisplus.annotation.TableName; + @NoArgsConstructor @Getter @Setter diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/ProcessAlertContent.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/ProcessAlertContent.java index 22dab95234aa..31f89d13488d 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/ProcessAlertContent.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/ProcessAlertContent.java @@ -17,18 +17,24 @@ package org.apache.dolphinscheduler.dao.entity; -import com.fasterxml.jackson.annotation.JsonInclude; -import com.fasterxml.jackson.annotation.JsonInclude.Include; -import com.fasterxml.jackson.annotation.JsonProperty; +import org.apache.dolphinscheduler.common.enums.AlertEvent; +import org.apache.dolphinscheduler.common.enums.AlertWarnLevel; +import org.apache.dolphinscheduler.common.enums.CommandType; +import org.apache.dolphinscheduler.common.enums.Flag; +import org.apache.dolphinscheduler.common.enums.WorkflowExecutionStatus; +import org.apache.dolphinscheduler.plugin.task.api.enums.TaskExecutionStatus; + +import java.io.Serializable; +import java.util.Date; + import lombok.AllArgsConstructor; import lombok.Builder; import lombok.Data; import lombok.NoArgsConstructor; -import org.apache.dolphinscheduler.common.enums.*; -import org.apache.dolphinscheduler.plugin.task.api.enums.TaskExecutionStatus; -import java.io.Serializable; -import java.util.Date; +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonInclude.Include; +import com.fasterxml.jackson.annotation.JsonProperty; @Data @AllArgsConstructor diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/ResponseTaskLog.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/ResponseTaskLog.java index 6876c060f852..3a0bf32947a9 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/ResponseTaskLog.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/ResponseTaskLog.java @@ -26,6 +26,7 @@ @Data @AllArgsConstructor public class ResponseTaskLog { + private int lineNum; private String message; } diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/ServerAlertContent.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/ServerAlertContent.java index 211863f73f4d..6ab298e296cb 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/ServerAlertContent.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/ServerAlertContent.java @@ -49,6 +49,7 @@ public static Builder newBuilder() { } public static class Builder { + private String type; private String host; diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/Session.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/Session.java index 39bf6143dd4d..7929b3de6ab4 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/Session.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/Session.java @@ -16,12 +16,12 @@ */ package org.apache.dolphinscheduler.dao.entity; +import java.util.Date; + import com.baomidou.mybatisplus.annotation.IdType; import com.baomidou.mybatisplus.annotation.TableId; import com.baomidou.mybatisplus.annotation.TableName; -import java.util.Date; - /** * session */ @@ -31,7 +31,7 @@ public class Session { /** * id */ - @TableId(value="id", type=IdType.INPUT) + @TableId(value = "id", type = IdType.INPUT) private String id; /** diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/TaskAlertContent.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/TaskAlertContent.java index fbd2f64a5983..5d2f1c8e08b1 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/TaskAlertContent.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/TaskAlertContent.java @@ -17,18 +17,19 @@ package org.apache.dolphinscheduler.dao.entity; +import org.apache.dolphinscheduler.plugin.task.api.enums.TaskExecutionStatus; + +import java.io.Serializable; +import java.util.Date; + import lombok.AllArgsConstructor; import lombok.Builder; import lombok.Data; import lombok.NoArgsConstructor; -import java.io.Serializable; -import java.util.Date; - import com.fasterxml.jackson.annotation.JsonInclude; import com.fasterxml.jackson.annotation.JsonInclude.Include; import com.fasterxml.jackson.annotation.JsonProperty; -import org.apache.dolphinscheduler.plugin.task.api.enums.TaskExecutionStatus; @Data @Builder diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/WorkFlowLineage.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/WorkFlowLineage.java index 68b63141c5c2..7eb4cb3820a8 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/WorkFlowLineage.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/WorkFlowLineage.java @@ -20,6 +20,7 @@ import java.util.Date; public class WorkFlowLineage { + private long workFlowCode; private String workFlowName; private String workFlowPublishStatus; diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/WorkFlowRelation.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/WorkFlowRelation.java index f89992ec13e9..72dbf8790d02 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/WorkFlowRelation.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/WorkFlowRelation.java @@ -20,6 +20,7 @@ import java.util.Objects; public class WorkFlowRelation { + private long sourceWorkFlowCode; private long targetWorkFlowCode; @@ -57,7 +58,7 @@ public boolean equals(Object o) { } WorkFlowRelation that = (WorkFlowRelation) o; return sourceWorkFlowCode == that.sourceWorkFlowCode - && targetWorkFlowCode == that.targetWorkFlowCode; + && targetWorkFlowCode == that.targetWorkFlowCode; } @Override diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/ZookeeperRecord.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/ZookeeperRecord.java index 7d45c66940ee..16d67b07aad9 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/ZookeeperRecord.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/ZookeeperRecord.java @@ -78,14 +78,13 @@ public class ZookeeperRecord { */ private Date date; - /** * is normal or not, 1:normal */ private int state; - - public ZookeeperRecord(String hostname,int connections, int watches, long sent, long received, String mode, float minLatency, float avgLatency, float maxLatency, int nodeCount, int state,Date date) { + public ZookeeperRecord(String hostname, int connections, int watches, long sent, long received, String mode, + float minLatency, float avgLatency, float maxLatency, int nodeCount, int state, Date date) { this.hostname = hostname; this.connections = connections; this.watches = watches; @@ -100,7 +99,6 @@ public ZookeeperRecord(String hostname,int connections, int watches, long sent, this.date = date; } - public String getHostname() { return hostname; } diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/AccessTokenMapper.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/AccessTokenMapper.java index b6fabb66c146..816576893283 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/AccessTokenMapper.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/AccessTokenMapper.java @@ -32,7 +32,6 @@ */ public interface AccessTokenMapper extends BaseMapper { - /** * access token page * @@ -43,8 +42,7 @@ public interface AccessTokenMapper extends BaseMapper { */ IPage selectAccessTokenPage(Page page, @Param("userName") String userName, - @Param("userId") int userId - ); + @Param("userId") int userId); /** * Query access token for specified user @@ -68,5 +66,6 @@ IPage selectAccessTokenPage(Page page, * @param accessTokensIds * @return access token for specified user */ - List listAuthorizedAccessToken(@Param("userId") int userId, @Param("accessTokensIds")List accessTokensIds); + List listAuthorizedAccessToken(@Param("userId") int userId, + @Param("accessTokensIds") List accessTokensIds); } diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/AlertGroupMapper.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/AlertGroupMapper.java index ceabc5b49b96..712b67e5f2d7 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/AlertGroupMapper.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/AlertGroupMapper.java @@ -19,10 +19,8 @@ import org.apache.dolphinscheduler.dao.entity.AlertGroup; -import org.apache.dolphinscheduler.dao.entity.User; import org.apache.ibatis.annotations.Param; -import java.util.ArrayList; import java.util.List; import com.baomidou.mybatisplus.core.mapper.BaseMapper; @@ -34,7 +32,6 @@ */ public interface AlertGroupMapper extends BaseMapper { - /** * alertgroup page * @param page page @@ -44,7 +41,6 @@ public interface AlertGroupMapper extends BaseMapper { IPage queryAlertGroupPage(Page page, @Param("groupName") String groupName); - /** * query by group name * @param groupName groupName @@ -91,7 +87,8 @@ IPage queryAlertGroupPage(Page page, * @param alertGroupsIds * @return */ - List listAuthorizedAlertGroupList (@Param("userId") int userId, @Param("alertGroupsIds")List alertGroupsIds); + List listAuthorizedAlertGroupList(@Param("userId") int userId, + @Param("alertGroupsIds") List alertGroupsIds); /** * queryAlertGroupPageByIds @@ -100,5 +97,6 @@ IPage queryAlertGroupPage(Page page, * @param searchVal * @return */ - IPage queryAlertGroupPageByIds(Page page, @Param("ids") List ids, @Param("searchVal") String searchVal); + IPage queryAlertGroupPageByIds(Page page, @Param("ids") List ids, + @Param("searchVal") String searchVal); } diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/AuditLogMapper.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/AuditLogMapper.java index bfd86aa23e0f..7d77b3e42928 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/AuditLogMapper.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/AuditLogMapper.java @@ -30,6 +30,7 @@ * auditlog mapper interface */ public interface AuditLogMapper extends BaseMapper { + IPage queryAuditLog(IPage page, @Param("resourceType") int[] resourceArray, @Param("operationType") int[] operationType, diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/CommandMapper.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/CommandMapper.java index de0ea222a1f7..877f7442e28b 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/CommandMapper.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/CommandMapper.java @@ -40,9 +40,9 @@ public interface CommandMapper extends BaseMapper { * @return CommandCount list */ List countCommandState( - @Param("startTime") Date startTime, - @Param("endTime") Date endTime, - @Param("projectCodeArray") Long[] projectCodeArray); + @Param("startTime") Date startTime, + @Param("endTime") Date endTime, + @Param("projectCodeArray") Long[] projectCodeArray); /** * query command page @@ -50,10 +50,11 @@ List countCommandState( */ List queryCommandPage(@Param("limit") int limit, @Param("offset") int offset); - /** * query command page by slot * @return command list */ - List queryCommandPageBySlot(@Param("limit") int limit, @Param("offset") int offset, @Param("masterCount") int masterCount, @Param("thisMasterSlot") int thisMasterSlot); + List queryCommandPageBySlot(@Param("limit") int limit, @Param("offset") int offset, + @Param("masterCount") int masterCount, + @Param("thisMasterSlot") int thisMasterSlot); } diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/DataSourceMapper.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/DataSourceMapper.java index b70c5f801c71..288e65089b14 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/DataSourceMapper.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/DataSourceMapper.java @@ -17,16 +17,15 @@ package org.apache.dolphinscheduler.dao.mapper; -import com.baomidou.mybatisplus.extension.plugins.pagination.Page; import org.apache.dolphinscheduler.dao.entity.DataSource; import org.apache.ibatis.annotations.Param; -import java.util.ArrayList; import java.util.List; import com.baomidou.mybatisplus.core.mapper.BaseMapper; import com.baomidou.mybatisplus.core.metadata.IPage; +import com.baomidou.mybatisplus.extension.plugins.pagination.Page; /** * datasource mapper interface @@ -39,7 +38,8 @@ public interface DataSourceMapper extends BaseMapper { * @param type type * @return datasource list */ - List queryDataSourceByType(@Param("userId") int userId, @Param("type") Integer type, @Param("testFlag") int testFlag); + List queryDataSourceByType(@Param("userId") int userId, @Param("type") Integer type, + @Param("testFlag") int testFlag); /** * datasource page @@ -59,7 +59,6 @@ IPage selectPaging(IPage page, */ List queryDataSourceByName(@Param("name") String name); - /** * query authed datasource * @param userId userId @@ -81,7 +80,6 @@ IPage selectPaging(IPage page, */ List listAllDataSourceByType(@Param("type") Integer type); - /** * list authorized UDF function * @@ -90,7 +88,8 @@ IPage selectPaging(IPage page, * @param T * @return UDF function list */ - List listAuthorizedDataSource(@Param("userId") int userId, @Param("dataSourceIds") T[] dataSourceIds); + List listAuthorizedDataSource(@Param("userId") int userId, + @Param("dataSourceIds") T[] dataSourceIds); /** * query datasource by name and user id @@ -108,7 +107,9 @@ IPage selectPaging(IPage page, * @param searchVal * @return */ - IPage selectPagingByIds(Page dataSourcePage, @Param("dataSourceIds") List dataSourceIds, @Param("name") String name); + IPage selectPagingByIds(Page dataSourcePage, + @Param("dataSourceIds") List dataSourceIds, + @Param("name") String name); /** * clearBindTestId diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/DataSourceUserMapper.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/DataSourceUserMapper.java index df51ebe33e58..209d65cfc909 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/DataSourceUserMapper.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/DataSourceUserMapper.java @@ -28,7 +28,6 @@ */ public interface DataSourceUserMapper extends BaseMapper { - /** * delete datasource by userId * @param userId userId diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/EnvironmentMapper.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/EnvironmentMapper.java index fbc1cb7a9abd..7688259c4fe4 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/EnvironmentMapper.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/EnvironmentMapper.java @@ -17,16 +17,15 @@ package org.apache.dolphinscheduler.dao.mapper; -import com.baomidou.mybatisplus.extension.plugins.pagination.Page; import org.apache.dolphinscheduler.dao.entity.Environment; import org.apache.ibatis.annotations.Param; import java.util.List; -import java.util.Set; import com.baomidou.mybatisplus.core.mapper.BaseMapper; import com.baomidou.mybatisplus.core.metadata.IPage; +import com.baomidou.mybatisplus.extension.plugins.pagination.Page; /** * environment mapper interface @@ -78,5 +77,6 @@ public interface EnvironmentMapper extends BaseMapper { * @param searchVal * @return */ - IPage queryEnvironmentListPagingByIds(Page page, @Param("ids")List ids, @Param("searchName")String searchVal); + IPage queryEnvironmentListPagingByIds(Page page, @Param("ids") List ids, + @Param("searchName") String searchVal); } diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/ErrorCommandMapper.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/ErrorCommandMapper.java index acd9ce79cbf5..5c04e217e162 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/ErrorCommandMapper.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/ErrorCommandMapper.java @@ -40,7 +40,7 @@ public interface ErrorCommandMapper extends BaseMapper { * @return CommandCount list */ List countCommandState( - @Param("startTime") Date startTime, - @Param("endTime") Date endTime, - @Param("projectCodeArray") Long[] projectCodeArray); + @Param("startTime") Date startTime, + @Param("endTime") Date endTime, + @Param("projectCodeArray") Long[] projectCodeArray); } diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/FavTaskMapper.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/FavTaskMapper.java index 0c102ce26324..8ea39dd7d8b8 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/FavTaskMapper.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/FavTaskMapper.java @@ -17,12 +17,14 @@ package org.apache.dolphinscheduler.dao.mapper; -import com.baomidou.mybatisplus.core.mapper.BaseMapper; import org.apache.dolphinscheduler.dao.entity.FavTask; + import org.apache.ibatis.annotations.Param; import java.util.Set; +import com.baomidou.mybatisplus.core.mapper.BaseMapper; + /** * fav mapper interface */ diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/K8sNamespaceMapper.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/K8sNamespaceMapper.java index f89e7579e4ec..ce0f2f87a440 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/K8sNamespaceMapper.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/K8sNamespaceMapper.java @@ -1,83 +1,84 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.dolphinscheduler.dao.mapper; - -import org.apache.dolphinscheduler.dao.entity.K8sNamespace; - -import org.apache.ibatis.annotations.Param; - -import java.util.List; - -import com.baomidou.mybatisplus.core.mapper.BaseMapper; -import com.baomidou.mybatisplus.core.metadata.IPage; - -/** - * namespace interface - */ -public interface K8sNamespaceMapper extends BaseMapper { - /** - * k8s namespace page - * - * @param page page - * @param searchVal searchVal - * @return k8s namespace IPage - */ - IPage queryK8sNamespacePaging(IPage page, - @Param("searchVal") String searchVal); - - /** - * check the target namespace exist - * - * @param namespace namespace - * @param clusterCode clusterCode - * @return true if exist else return null - */ - Boolean existNamespace(@Param("namespace") String namespace, @Param("clusterCode") Long clusterCode); - - /** - * query namespace except userId - * - * @param userId userId - * @return namespace list - */ - List queryNamespaceExceptUserId(@Param("userId") int userId); - - /** - * query authed namespace list by userId - * - * @param userId userId - * @return namespace list - */ - List queryAuthedNamespaceListByUserId(@Param("userId") int userId); - - /** - * query namespace can use - * - * @param userId userId - * @return namespace list - */ - List queryNamespaceAvailable(@Param("userId") Integer userId); - - /** - * check the target namespace - * - * @param namespaceCode namespaceCode - * @return true if exist else return null - */ - K8sNamespace queryByNamespaceCode(@Param("clusterCode") Long namespaceCode); -} \ No newline at end of file +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.dao.mapper; + +import org.apache.dolphinscheduler.dao.entity.K8sNamespace; + +import org.apache.ibatis.annotations.Param; + +import java.util.List; + +import com.baomidou.mybatisplus.core.mapper.BaseMapper; +import com.baomidou.mybatisplus.core.metadata.IPage; + +/** + * namespace interface + */ +public interface K8sNamespaceMapper extends BaseMapper { + + /** + * k8s namespace page + * + * @param page page + * @param searchVal searchVal + * @return k8s namespace IPage + */ + IPage queryK8sNamespacePaging(IPage page, + @Param("searchVal") String searchVal); + + /** + * check the target namespace exist + * + * @param namespace namespace + * @param clusterCode clusterCode + * @return true if exist else return null + */ + Boolean existNamespace(@Param("namespace") String namespace, @Param("clusterCode") Long clusterCode); + + /** + * query namespace except userId + * + * @param userId userId + * @return namespace list + */ + List queryNamespaceExceptUserId(@Param("userId") int userId); + + /** + * query authed namespace list by userId + * + * @param userId userId + * @return namespace list + */ + List queryAuthedNamespaceListByUserId(@Param("userId") int userId); + + /** + * query namespace can use + * + * @param userId userId + * @return namespace list + */ + List queryNamespaceAvailable(@Param("userId") Integer userId); + + /** + * check the target namespace + * + * @param namespaceCode namespaceCode + * @return true if exist else return null + */ + K8sNamespace queryByNamespaceCode(@Param("clusterCode") Long namespaceCode); +} diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/ProcessDefinitionLogMapper.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/ProcessDefinitionLogMapper.java index bd9297ec74e4..ba2d6cb9e345 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/ProcessDefinitionLogMapper.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/ProcessDefinitionLogMapper.java @@ -53,7 +53,8 @@ public interface ProcessDefinitionLogMapper extends BaseMapper queryByDefinitionName(@Param("projectCode") long projectCode, @Param("name") String name); + List queryByDefinitionName(@Param("projectCode") long projectCode, + @Param("name") String name); /** * query process definition log list @@ -81,7 +82,9 @@ public interface ProcessDefinitionLogMapper extends BaseMapper queryProcessDefinitionVersionsPaging(Page page, @Param("code") long code, @Param("projectCode") long projectCode); + IPage queryProcessDefinitionVersionsPaging(Page page, + @Param("code") long code, + @Param("projectCode") long projectCode); /** * delete the certain process definition version by process definition id and version number diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/ProcessInstanceMapMapper.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/ProcessInstanceMapMapper.java index 0397d3715432..fd3440e2d530 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/ProcessInstanceMapMapper.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/ProcessInstanceMapMapper.java @@ -17,11 +17,12 @@ package org.apache.dolphinscheduler.dao.mapper; +import org.apache.dolphinscheduler.dao.entity.ProcessInstanceMap; + import org.apache.ibatis.annotations.Param; import java.util.List; -import org.apache.dolphinscheduler.dao.entity.ProcessInstanceMap; import com.baomidou.mybatisplus.core.mapper.BaseMapper; /** @@ -38,7 +39,6 @@ public interface ProcessInstanceMapMapper extends BaseMapper ProcessInstanceMap queryByParentId(@Param("parentProcessId") int parentProcessId, @Param("parentTaskId") int parentTaskId); - /** * query by sub process id * @param subProcessId subProcessId diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/ProjectMapper.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/ProjectMapper.java index 3a3c314b3625..59c635f711a8 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/ProjectMapper.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/ProjectMapper.java @@ -32,6 +32,7 @@ * project mapper interface */ public interface ProjectMapper extends BaseMapper { + /** * query project detail by code * @param projectCode projectCode @@ -128,7 +129,6 @@ IPage queryProjectListPaging(IPage page, */ List queryAllProject(@Param("userId") int userId); - /** * list authorized Projects * @param userId @@ -136,7 +136,7 @@ IPage queryProjectListPaging(IPage page, * @param * @return */ - List listAuthorizedProjects(@Param("userId") int userId, @Param("projectsIds")List projectsIds); + List listAuthorizedProjects(@Param("userId") int userId, @Param("projectsIds") List projectsIds); /** * query all project for dependent node diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/QueueMapper.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/QueueMapper.java index 0b523e86199c..f37ed0f5a1e0 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/QueueMapper.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/QueueMapper.java @@ -17,12 +17,14 @@ package org.apache.dolphinscheduler.dao.mapper; import org.apache.dolphinscheduler.dao.entity.Queue; -import com.baomidou.mybatisplus.core.mapper.BaseMapper; -import com.baomidou.mybatisplus.core.metadata.IPage; + import org.apache.ibatis.annotations.Param; import java.util.List; +import com.baomidou.mybatisplus.core.mapper.BaseMapper; +import com.baomidou.mybatisplus.core.metadata.IPage; + /** * queue mapper interface */ @@ -34,7 +36,7 @@ public interface QueueMapper extends BaseMapper { * @param searchVal searchVal * @return queue IPage */ - IPage queryQueuePaging(IPage page, @Param("ids")List ids, + IPage queryQueuePaging(IPage page, @Param("ids") List ids, @Param("searchVal") String searchVal); /** @@ -44,7 +46,7 @@ IPage queryQueuePaging(IPage page, @Param("ids")List ids, * @return queue list */ List queryAllQueueList(@Param("queue") String queue, - @Param("queueName") String queueName); + @Param("queueName") String queueName); /** * check the target queue exist diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/ResourceMapper.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/ResourceMapper.java index 268e7bafb364..45c70cb72342 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/ResourceMapper.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/ResourceMapper.java @@ -97,14 +97,14 @@ IPage queryResourcePaging(IPage page, * @param T * @return resource list */ - List listAuthorizedResourceById(@Param("userId") int userId,@Param("resIds")T[] resIds); + List listAuthorizedResourceById(@Param("userId") int userId, @Param("resIds") T[] resIds); /** * delete resource by id array * @param resIds resource id array * @return delete num */ - int deleteIds(@Param("resIds")Integer[] resIds); + int deleteIds(@Param("resIds") Integer[] resIds); /** * list children @@ -119,14 +119,14 @@ IPage queryResourcePaging(IPage page, * @param type resource type * @return resource */ - List queryResource(@Param("fullName") String fullName,@Param("type") int type); + List queryResource(@Param("fullName") String fullName, @Param("type") int type); /** * list resource by id array * @param resIds resource id array * @return resource list */ - List listResourceByIds(@Param("resIds")Integer[] resIds); + List listResourceByIds(@Param("resIds") Integer[] resIds); /** * update resource @@ -143,8 +143,8 @@ IPage queryResourcePaging(IPage page, * @return true if exist else return null */ Boolean existResourceByUser(@Param("fullName") String fullName, - @Param("userId") int userId, - @Param("type") int type); + @Param("userId") int userId, + @Param("type") int type); /** * check resource exist diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/ResourceUserMapper.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/ResourceUserMapper.java index dbaca256cba9..6fc36ea425c4 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/ResourceUserMapper.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/ResourceUserMapper.java @@ -55,6 +55,6 @@ int deleteResourceUser(@Param("userId") int userId, * @return delete result */ int deleteResourceUserArray(@Param("userId") int userId, - @Param("resIds") Integer[] resIds); + @Param("resIds") Integer[] resIds); } diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/SessionMapper.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/SessionMapper.java index a72a6807105e..8f6a7597ee38 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/SessionMapper.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/SessionMapper.java @@ -17,11 +17,13 @@ package org.apache.dolphinscheduler.dao.mapper; import org.apache.dolphinscheduler.dao.entity.Session; -import com.baomidou.mybatisplus.core.mapper.BaseMapper; + import org.apache.ibatis.annotations.Param; import java.util.List; +import com.baomidou.mybatisplus.core.mapper.BaseMapper; + /** * session mapper interface */ @@ -40,6 +42,6 @@ public interface SessionMapper extends BaseMapper { * @param ip ip * @return session */ - Session queryByUserIdAndIp(@Param("userId") int userId,@Param("ip") String ip); + Session queryByUserIdAndIp(@Param("userId") int userId, @Param("ip") String ip); } diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/TaskDefinitionLogMapper.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/TaskDefinitionLogMapper.java index 851d8dac518d..98cd1175e189 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/TaskDefinitionLogMapper.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/TaskDefinitionLogMapper.java @@ -94,5 +94,6 @@ public interface TaskDefinitionLogMapper extends BaseMapper { * @param code process definition code * @return the paging task definition version list */ - IPage queryTaskDefinitionVersionsPaging(Page page, @Param("code") long code, @Param("projectCode") long projectCode); + IPage queryTaskDefinitionVersionsPaging(Page page, @Param("code") long code, + @Param("projectCode") long projectCode); } diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/TaskGroupMapper.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/TaskGroupMapper.java index 05c99194ce65..819e70dcfb4e 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/TaskGroupMapper.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/TaskGroupMapper.java @@ -21,12 +21,12 @@ import org.apache.ibatis.annotations.Param; +import java.util.List; + import com.baomidou.mybatisplus.core.mapper.BaseMapper; import com.baomidou.mybatisplus.core.metadata.IPage; import com.baomidou.mybatisplus.extension.plugins.pagination.Page; -import java.util.List; - /** * the Dao interfaces of task group * @@ -79,9 +79,10 @@ IPage queryTaskGroupPaging(IPage page, @Param("ids") List< */ int selectAvailableCountById(@Param("groupId") int groupId); - int selectCountByIdStatus(@Param("id") int id,@Param("status") int status); + int selectCountByIdStatus(@Param("id") int id, @Param("status") int status); - IPage queryTaskGroupPagingByProjectCode(Page page, @Param("ids") List ids, @Param("projectCode") Long projectCode); + IPage queryTaskGroupPagingByProjectCode(Page page, @Param("ids") List ids, + @Param("projectCode") Long projectCode); /** * listAuthorizedResource diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/TaskGroupQueueMapper.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/TaskGroupQueueMapper.java index 38f5ca60166d..9826b3e6d5ec 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/TaskGroupQueueMapper.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/TaskGroupQueueMapper.java @@ -44,8 +44,7 @@ public interface TaskGroupQueueMapper extends BaseMapper { * @return task group queue list */ IPage queryTaskGroupQueuePaging(IPage page, - @Param("groupId") int groupId - ); + @Param("groupId") int groupId); TaskGroupQueue queryByTaskId(@Param("taskId") int taskId); @@ -77,7 +76,8 @@ IPage queryTaskGroupQueuePaging(IPage page, /** * Query the {@link TaskGroupQueue}, who's priority > the given priority */ - List queryHighPriorityTasks(@Param("groupId") int groupId, @Param("priority") int priority, @Param("status") int status); + List queryHighPriorityTasks(@Param("groupId") int groupId, @Param("priority") int priority, + @Param("status") int status); TaskGroupQueue queryTheHighestPriorityTasks(@Param("groupId") int groupId, @Param("status") int status, @Param("forceStart") int forceStart, @Param("inQueue") int inQueue); @@ -86,14 +86,17 @@ TaskGroupQueue queryTheHighestPriorityTasks(@Param("groupId") int groupId, @Para void updateForceStart(@Param("queueId") int queueId, @Param("forceStart") int forceStart); - int updateInQueueLimit1(@Param("oldValue") int oldValue, @Param("newValue") int newValue - , @Param("groupId") int id, @Param("status") int status); + int updateInQueueLimit1(@Param("oldValue") int oldValue, @Param("newValue") int newValue, @Param("groupId") int id, + @Param("status") int status); int updateInQueueCAS(@Param("oldValue") int oldValue, @Param("newValue") int newValue, @Param("id") int id); void modifyPriority(@Param("queueId") int queueId, @Param("priority") int priority); - IPage queryTaskGroupQueueByTaskGroupIdPaging(Page page, @Param("taskName")String taskName - ,@Param("processName") String processName,@Param("status") Integer status,@Param("groupId") int groupId - ,@Param("projects") List projects); + IPage queryTaskGroupQueueByTaskGroupIdPaging(Page page, + @Param("taskName") String taskName, + @Param("processName") String processName, + @Param("status") Integer status, + @Param("groupId") int groupId, + @Param("projects") List projects); } diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/TaskInstanceMapper.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/TaskInstanceMapper.java index cdc26ee4e6b0..7b90ba24afa7 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/TaskInstanceMapper.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/TaskInstanceMapper.java @@ -17,18 +17,20 @@ package org.apache.dolphinscheduler.dao.mapper; -import com.baomidou.mybatisplus.core.mapper.BaseMapper; -import com.baomidou.mybatisplus.core.metadata.IPage; import org.apache.dolphinscheduler.common.enums.Flag; import org.apache.dolphinscheduler.common.enums.TaskExecuteType; import org.apache.dolphinscheduler.dao.entity.ExecuteStatusCount; import org.apache.dolphinscheduler.dao.entity.TaskInstance; import org.apache.dolphinscheduler.plugin.task.api.enums.TaskExecutionStatus; + import org.apache.ibatis.annotations.Param; import java.util.Date; import java.util.List; +import com.baomidou.mybatisplus.core.mapper.BaseMapper; +import com.baomidou.mybatisplus.core.metadata.IPage; + /** * task instance mapper interface */ @@ -55,7 +57,7 @@ TaskInstance queryByInstanceIdAndCode(@Param("processInstanceId") int processIns @Param("taskCode") Long taskCode); List queryByProcessInstanceIdsAndTaskCodes(@Param("processInstanceIds") List processInstanceIds, - @Param("taskCodes") List taskCodes); + @Param("taskCodes") List taskCodes); Integer countTask(@Param("projectCodes") Long[] projectCodes, @Param("taskIds") int[] taskIds); @@ -100,20 +102,19 @@ IPage queryTaskInstanceListPaging(IPage page, @Param("host") String host, @Param("taskExecuteType") TaskExecuteType taskExecuteType, @Param("startTime") Date startTime, - @Param("endTime") Date endTime - ); + @Param("endTime") Date endTime); IPage queryStreamTaskInstanceListPaging(IPage page, - @Param("projectCode") Long projectCode, - @Param("processDefinitionName") String processDefinitionName, - @Param("searchVal") String searchVal, - @Param("taskName") String taskName, - @Param("executorId") int executorId, - @Param("states") int[] statusArray, - @Param("host") String host, - @Param("taskExecuteType") TaskExecuteType taskExecuteType, - @Param("startTime") Date startTime, - @Param("endTime") Date endTime); + @Param("projectCode") Long projectCode, + @Param("processDefinitionName") String processDefinitionName, + @Param("searchVal") String searchVal, + @Param("taskName") String taskName, + @Param("executorId") int executorId, + @Param("states") int[] statusArray, + @Param("host") String host, + @Param("taskExecuteType") TaskExecuteType taskExecuteType, + @Param("startTime") Date startTime, + @Param("endTime") Date endTime); List loadAllInfosNoRelease(@Param("processInstanceId") int processInstanceId, @Param("status") int status); diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/TenantMapper.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/TenantMapper.java index 7da6741183c6..623c1f8a8db8 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/TenantMapper.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/TenantMapper.java @@ -17,20 +17,19 @@ package org.apache.dolphinscheduler.dao.mapper; -import com.baomidou.mybatisplus.extension.plugins.pagination.Page; import org.apache.dolphinscheduler.dao.entity.Tenant; import org.apache.ibatis.annotations.Param; +import java.util.List; + import org.springframework.cache.annotation.CacheConfig; import org.springframework.cache.annotation.CacheEvict; import org.springframework.cache.annotation.Cacheable; import com.baomidou.mybatisplus.core.mapper.BaseMapper; import com.baomidou.mybatisplus.core.metadata.IPage; - -import java.util.ArrayList; -import java.util.List; +import com.baomidou.mybatisplus.extension.plugins.pagination.Page; /** * tenant mapper interface @@ -74,7 +73,7 @@ public interface TenantMapper extends BaseMapper { * @param searchVal searchVal * @return tenant IPage */ - IPage queryTenantPaging(IPage page,@Param("ids") List ids, + IPage queryTenantPaging(IPage page, @Param("ids") List ids, @Param("searchVal") String searchVal); /** @@ -92,7 +91,8 @@ IPage queryTenantPaging(IPage page,@Param("ids") List i * @param searchVal * @return */ - IPage queryTenantPagingByIds(Page page, @Param("ids")List ids, @Param("searchVal")String searchVal); + IPage queryTenantPagingByIds(Page page, @Param("ids") List ids, + @Param("searchVal") String searchVal); /** * queryAll diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/UDFUserMapper.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/UDFUserMapper.java index 6c7026dadf7f..2a5bc6a417df 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/UDFUserMapper.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/UDFUserMapper.java @@ -17,9 +17,11 @@ package org.apache.dolphinscheduler.dao.mapper; import org.apache.dolphinscheduler.dao.entity.UDFUser; -import com.baomidou.mybatisplus.core.mapper.BaseMapper; + import org.apache.ibatis.annotations.Param; +import com.baomidou.mybatisplus.core.mapper.BaseMapper; + /** * udf user realtion mapper interface */ @@ -40,4 +42,3 @@ public interface UDFUserMapper extends BaseMapper { int deleteByUdfFuncId(@Param("udfFuncId") int udfFuncId); } - diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/UdfFuncMapper.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/UdfFuncMapper.java index 340c3e9ce1af..5573aedd8d98 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/UdfFuncMapper.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/UdfFuncMapper.java @@ -17,12 +17,14 @@ package org.apache.dolphinscheduler.dao.mapper; import org.apache.dolphinscheduler.dao.entity.UdfFunc; -import com.baomidou.mybatisplus.core.mapper.BaseMapper; -import com.baomidou.mybatisplus.core.metadata.IPage; + import org.apache.ibatis.annotations.Param; import java.util.List; +import com.baomidou.mybatisplus.core.mapper.BaseMapper; +import com.baomidou.mybatisplus.core.metadata.IPage; + /** * udf function mapper interface */ @@ -84,7 +86,7 @@ List getUdfFuncByType(@Param("ids") List ids, * @param udfIds UDF function id array * @return UDF function list */ - List listAuthorizedUdfFunc (@Param("userId") int userId,@Param("udfIds")T[] udfIds); + List listAuthorizedUdfFunc(@Param("userId") int userId, @Param("udfIds") T[] udfIds); /** * list UDF by resource id @@ -98,7 +100,7 @@ List getUdfFuncByType(@Param("ids") List ids, * @param resourceIds resource id array * @return UDF function list */ - List listAuthorizedUdfByResourceId(@Param("userId") int userId,@Param("resourceIds") int[] resourceIds); + List listAuthorizedUdfByResourceId(@Param("userId") int userId, @Param("resourceIds") int[] resourceIds); /** * batch update udf func diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/UserMapper.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/UserMapper.java index a58299806f43..249f0b9f4ad7 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/UserMapper.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/UserMapper.java @@ -80,7 +80,6 @@ public interface UserMapper extends BaseMapper { */ User queryUserByNamePassword(@Param("userName") String userName, @Param("password") String password); - /** * user page * diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/WorkFlowLineageMapper.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/WorkFlowLineageMapper.java index 951f12a14ad6..b47731c5915f 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/WorkFlowLineageMapper.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/WorkFlowLineageMapper.java @@ -35,7 +35,8 @@ public interface WorkFlowLineageMapper { * @param workFlowName workFlowName * @return WorkFlowLineage list */ - List queryWorkFlowLineageByName(@Param("projectCode") long projectCode, @Param("workFlowName") String workFlowName); + List queryWorkFlowLineageByName(@Param("projectCode") long projectCode, + @Param("workFlowName") String workFlowName); /** * queryWorkFlowLineageByCode @@ -44,7 +45,8 @@ public interface WorkFlowLineageMapper { * @param workFlowCode workFlowCode * @return WorkFlowLineage */ - WorkFlowLineage queryWorkFlowLineageByCode(@Param("projectCode") long projectCode, @Param("workFlowCode") long workFlowCode); + WorkFlowLineage queryWorkFlowLineageByCode(@Param("projectCode") long projectCode, + @Param("workFlowCode") long workFlowCode); /** * queryWorkFlowLineageByProcessDefinitionCodes @@ -95,7 +97,6 @@ List queryProcessLineageByCode(@Param("projectCode") long projec List queryDownstreamLineageByProcessDefinitionCode(@Param("code") long code, @Param("taskType") String taskType); - /** * query upstream work flow dependent task params by process definition code * diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/upgrade/JsonSplitDao.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/upgrade/JsonSplitDao.java index 46c7610fde79..f2cb65877f1b 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/upgrade/JsonSplitDao.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/upgrade/JsonSplitDao.java @@ -41,9 +41,11 @@ public class JsonSplitDao { * @param processDefinitionLogs processDefinitionLogs */ public void executeJsonSplitProcessDefinition(Connection conn, List processDefinitionLogs) { - String updateSql = "UPDATE t_ds_process_definition SET global_params=?,timeout=?,tenant_id=?,locations=?,update_time=? where id=?"; - String insertLogSql = "insert into t_ds_process_definition_log (code,name,version,description,project_code,release_state,user_id," - + "global_params,flag,locations,timeout,tenant_id,operator,operate_time,create_time,update_time) values (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)"; + String updateSql = + "UPDATE t_ds_process_definition SET global_params=?,timeout=?,tenant_id=?,locations=?,update_time=? where id=?"; + String insertLogSql = + "insert into t_ds_process_definition_log (code,name,version,description,project_code,release_state,user_id," + + "global_params,flag,locations,timeout,tenant_id,operator,operate_time,create_time,update_time) values (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)"; try { PreparedStatement processUpdate = conn.prepareStatement(updateSql); PreparedStatement insertLog = conn.prepareStatement(insertLogSql); @@ -101,12 +103,15 @@ public void executeJsonSplitProcessDefinition(Connection conn, List processTaskRelationLogs) { - String insertSql = "insert into t_ds_process_task_relation (project_code,process_definition_code,process_definition_version,pre_task_code,pre_task_version," - + "post_task_code,post_task_version,condition_type,condition_params,create_time,update_time) values (?,?,?,?,?,?,?,?,?,?,?)"; - String insertLogSql = "insert into t_ds_process_task_relation_log (project_code,process_definition_code,process_definition_version,pre_task_code," - + "pre_task_version,post_task_code,post_task_version,condition_type,condition_params,operator,operate_time,create_time,update_time) " - + "values (?,?,?,?,?,?,?,?,?,?,?,?,?)"; + public void executeJsonSplitProcessTaskRelation(Connection conn, + List processTaskRelationLogs) { + String insertSql = + "insert into t_ds_process_task_relation (project_code,process_definition_code,process_definition_version,pre_task_code,pre_task_version," + + "post_task_code,post_task_version,condition_type,condition_params,create_time,update_time) values (?,?,?,?,?,?,?,?,?,?,?)"; + String insertLogSql = + "insert into t_ds_process_task_relation_log (project_code,process_definition_code,process_definition_version,pre_task_code," + + "pre_task_version,post_task_code,post_task_version,condition_type,condition_params,operator,operate_time,create_time,update_time) " + + "values (?,?,?,?,?,?,?,?,?,?,?,?,?)"; try { PreparedStatement insert = conn.prepareStatement(insertSql); PreparedStatement insertLog = conn.prepareStatement(insertLogSql); @@ -167,12 +172,14 @@ public void executeJsonSplitProcessTaskRelation(Connection conn, List taskDefinitionLogs) { - String insertSql = "insert into t_ds_task_definition (code,name,version,description,project_code,user_id,task_type,task_params,flag,task_priority," - + "worker_group,environment_code,fail_retry_times,fail_retry_interval,timeout_flag,timeout_notify_strategy,timeout,delay_time,resource_ids," - + "create_time,update_time) values (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)"; - String insertLogSql = "insert into t_ds_task_definition_log (code,name,version,description,project_code,user_id,task_type,task_params,flag,task_priority," - + "worker_group,environment_code,fail_retry_times,fail_retry_interval,timeout_flag,timeout_notify_strategy,timeout,delay_time,resource_ids,operator," - + "operate_time,create_time,update_time) values (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)"; + String insertSql = + "insert into t_ds_task_definition (code,name,version,description,project_code,user_id,task_type,task_params,flag,task_priority," + + "worker_group,environment_code,fail_retry_times,fail_retry_interval,timeout_flag,timeout_notify_strategy,timeout,delay_time,resource_ids," + + "create_time,update_time) values (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)"; + String insertLogSql = + "insert into t_ds_task_definition_log (code,name,version,description,project_code,user_id,task_type,task_params,flag,task_priority," + + "worker_group,environment_code,fail_retry_times,fail_retry_interval,timeout_flag,timeout_notify_strategy,timeout,delay_time,resource_ids,operator," + + "operate_time,create_time,update_time) values (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)"; try { PreparedStatement insert = conn.prepareStatement(insertSql); PreparedStatement insertLog = conn.prepareStatement(insertLogSql); @@ -193,7 +200,8 @@ public void executeJsonSplitTaskDefinition(Connection conn, List pr public List queryProcessDefinition(Connection conn) { List processDefinitions = new ArrayList<>(); - String sql = "SELECT id,code,project_code,user_id,locations,name,description,release_state,flag,create_time FROM t_ds_process_definition"; + String sql = + "SELECT id,code,project_code,user_id,locations,name,description,release_state,flag,create_time FROM t_ds_process_definition"; ResultSet rs = null; PreparedStatement pstmt = null; try { @@ -140,7 +140,8 @@ public List queryProcessDefinition(Connection conn) { * @param processDefinitions processDefinitions * @param projectIdCodeMap projectIdCodeMap */ - public void updateProcessDefinitionCode(Connection conn, List processDefinitions, Map projectIdCodeMap) { + public void updateProcessDefinitionCode(Connection conn, List processDefinitions, + Map projectIdCodeMap) { String sql = "UPDATE t_ds_process_definition SET code=?, project_code=?, version=? where id=?"; try { for (ProcessDefinition processDefinition : processDefinitions) { diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/upgrade/ScheduleDao.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/upgrade/ScheduleDao.java index 80a49fd42ba6..9d486d00779e 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/upgrade/ScheduleDao.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/upgrade/ScheduleDao.java @@ -68,7 +68,8 @@ public Map queryAllSchedule(Connection conn) { * @param scheduleMap scheduleMap * @param processIdCodeMap processIdCodeMap */ - public void updateScheduleCode(Connection conn, Map scheduleMap, Map processIdCodeMap) { + public void updateScheduleCode(Connection conn, Map scheduleMap, + Map processIdCodeMap) { String sql = "UPDATE t_ds_schedules SET process_definition_code=?,timezone_id=?,environment_code=-1 where id=?"; try { Clock clock = Clock.systemDefaultZone(); diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/upgrade/SchemaUtils.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/upgrade/SchemaUtils.java index f0fdeeea56e3..e6a7b00104ae 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/upgrade/SchemaUtils.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/upgrade/SchemaUtils.java @@ -38,6 +38,7 @@ * Metadata related common classes */ public class SchemaUtils { + private static final Logger logger = LoggerFactory.getLogger(SchemaUtils.class); private SchemaUtils() { @@ -95,7 +96,8 @@ public static boolean isAGreatVersion(String schemaVersion, String version) { } } - // If the version and schema version is the same from 0 up to the arrlength-1 element,whoever has a larger arrLength has a larger version number + // If the version and schema version is the same from 0 up to the arrlength-1 element,whoever has a larger + // arrLength has a larger version number return schemaVersionArr.length > versionArr.length; } @@ -112,7 +114,8 @@ public static String getSoftVersion() throws IOException { softVersion = Strings.nullToEmpty(softVersion).replaceAll("\\s+|\r|\n", ""); } catch (FileNotFoundException e) { logger.error(e.getMessage(), e); - throw new RuntimeException("Failed to get the product version description file. The file could not be found", e); + throw new RuntimeException( + "Failed to get the product version description file. The file could not be found", e); } return softVersion; } diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/upgrade/WorkerGroupDao.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/upgrade/WorkerGroupDao.java index 936b1d477edf..e8a9fc7d8226 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/upgrade/WorkerGroupDao.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/upgrade/WorkerGroupDao.java @@ -18,16 +18,16 @@ package org.apache.dolphinscheduler.dao.upgrade; import org.apache.dolphinscheduler.common.utils.ConnectionUtils; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import java.sql.Connection; import java.sql.PreparedStatement; import java.sql.ResultSet; -import java.sql.SQLException; import java.util.HashMap; import java.util.Map; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + public class WorkerGroupDao { public static final Logger logger = LoggerFactory.getLogger(WorkerGroupDao.class); @@ -37,8 +37,8 @@ public class WorkerGroupDao { * @param conn jdbc connection * @return old worker group Map */ - public Map queryAllOldWorkerGroup(Connection conn){ - Map workerGroupMap = new HashMap<>(); + public Map queryAllOldWorkerGroup(Connection conn) { + Map workerGroupMap = new HashMap<>(); String sql = String.format("select id,name from t_ds_worker_group"); ResultSet rs = null; @@ -47,14 +47,14 @@ public Map queryAllOldWorkerGroup(Connection conn){ pstmt = conn.prepareStatement(sql); rs = pstmt.executeQuery(); - while (rs.next()){ + while (rs.next()) { int id = rs.getInt(1); String name = rs.getString(2); - workerGroupMap.put(id,name); + workerGroupMap.put(id, name); } } catch (Exception e) { - logger.error(e.getMessage(),e); + logger.error(e.getMessage(), e); throw new RuntimeException("sql: " + sql, e); } finally { ConnectionUtils.releaseResource(rs, pstmt, conn); diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/utils/BeanContext.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/utils/BeanContext.java index 4bf4dbdb83c8..78a333488c74 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/utils/BeanContext.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/utils/BeanContext.java @@ -27,6 +27,7 @@ */ @Component public class BeanContext implements ApplicationContextAware { + private static ApplicationContext applicationContext; public static ApplicationContext getApplicationContext() { @@ -42,7 +43,7 @@ public static ApplicationContext getApplicationContext() { */ @SuppressWarnings("unchecked") public static T getBean(String name) throws BeansException { - return (T)applicationContext.getBean(name); + return (T) applicationContext.getBean(name); } /** @@ -56,7 +57,6 @@ public static T getBean(Class clazz) throws BeansException { return applicationContext.getBean(clazz); } - /** * set applicationcontext * @param applicationContext applicationContext diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/utils/PostgreSQLPerformance.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/utils/PostgreSQLPerformance.java index 928575b98450..8440636ae83d 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/utils/PostgreSQLPerformance.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/utils/PostgreSQLPerformance.java @@ -59,7 +59,9 @@ public MonitorRecord getMonitorRecord(Connection conn) { } } - try (ResultSet rs3 = pstmt.executeQuery("select count(*) from pg_stat_activity pg where pg.state = 'active';")) { + try ( + ResultSet rs3 = + pstmt.executeQuery("select count(*) from pg_stat_activity pg where pg.state = 'active';")) { if (rs3.next()) { monitorRecord.setThreadsRunningConnections(rs3.getInt("count")); } diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/utils/ResourceProcessDefinitionUtils.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/utils/ResourceProcessDefinitionUtils.java index 483c052b83e6..69d9788b3233 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/utils/ResourceProcessDefinitionUtils.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/utils/ResourceProcessDefinitionUtils.java @@ -30,6 +30,7 @@ * resource process definition utils */ public class ResourceProcessDefinitionUtils { + /** * get resource process map key is resource id,value is the set of process definition code * @@ -47,7 +48,8 @@ public static Map> getResourceProcessDefinitionMap(List resourceIdSet = Arrays.stream(resourceIds).map(Integer::parseInt).collect(Collectors.toSet()); + Set resourceIdSet = + Arrays.stream(resourceIds).map(Integer::parseInt).collect(Collectors.toSet()); for (Integer resourceId : resourceIdSet) { Set codeSet; if (resourceResult.containsKey(resourceId)) { diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/utils/TaskInstanceUtils.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/utils/TaskInstanceUtils.java index 1f6ffdfd0918..431a022d5992 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/utils/TaskInstanceUtils.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/utils/TaskInstanceUtils.java @@ -54,7 +54,8 @@ public static void copyTaskInstance(TaskInstance source, TaskInstance target) { target.setAppLink(source.getAppLink()); target.setFlag(source.getFlag()); target.setDependency(source.getDependency()); - // todo: we need to cpoy the task params and then copy switchDependency, since the setSwitchDependency rely on task params, this is really a very bad practice. + // todo: we need to cpoy the task params and then copy switchDependency, since the setSwitchDependency rely on + // task params, this is really a very bad practice. target.setTaskParams(source.getTaskParams()); target.setSwitchDependency(source.getSwitchDependency()); target.setDuration(source.getDuration()); diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/utils/WorkflowUtils.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/utils/WorkflowUtils.java index e2c2ccb31c7f..b453392e0e59 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/utils/WorkflowUtils.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/utils/WorkflowUtils.java @@ -17,10 +17,11 @@ package org.apache.dolphinscheduler.dao.utils; -import java.util.Date; import org.apache.dolphinscheduler.common.utils.DateUtils; import org.apache.dolphinscheduler.dao.entity.ProcessInstance; +import java.util.Date; + /** * workflow utils */ @@ -33,11 +34,10 @@ public class WorkflowUtils { * @param processInstance workflow instance * @return workflow duration */ - public static String getWorkflowInstanceDuration (ProcessInstance processInstance) { - return processInstance.getState() != null && processInstance.getState().isFinished() ? - DateUtils.format2Duration(processInstance.getStartTime(), processInstance.getEndTime()) : - DateUtils.format2Duration(processInstance.getStartTime(), new Date()); + public static String getWorkflowInstanceDuration(ProcessInstance processInstance) { + return processInstance.getState() != null && processInstance.getState().isFinished() + ? DateUtils.format2Duration(processInstance.getStartTime(), processInstance.getEndTime()) + : DateUtils.format2Duration(processInstance.getStartTime(), new Date()); } } - diff --git a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/AlertDaoTest.java b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/AlertDaoTest.java index 7c7e3f04ab2b..f2cb503d9bfe 100644 --- a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/AlertDaoTest.java +++ b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/AlertDaoTest.java @@ -43,6 +43,7 @@ @Rollback @EnableTransactionManagement public class AlertDaoTest { + @Autowired private AlertDao alertDao; @@ -51,7 +52,7 @@ public void testAlertDao() { Alert alert = new Alert(); alert.setTitle("Mysql Exception"); alert.setContent("[\"alarm time：2018-02-05\", \"service name：MYSQL_ALTER\", \"alarm name：MYSQL_ALTER_DUMP\", " - + "\"get the alarm exception.！，interface error，exception information：timed out\", \"request address：http://blog.csdn.net/dreamInTheWorld/article/details/78539286\"]"); + + "\"get the alarm exception.！，interface error，exception information：timed out\", \"request address：http://blog.csdn.net/dreamInTheWorld/article/details/78539286\"]"); alert.setAlertGroupId(1); alert.setAlertStatus(AlertStatus.WAIT_EXECUTION); alertDao.addAlert(alert); @@ -63,7 +64,7 @@ public void testAlertDao() { @Test public void testAddAlertSendStatus() { - int insertCount = alertDao.addAlertSendStatus(AlertStatus.EXECUTION_SUCCESS,"success",1,1); + int insertCount = alertDao.addAlertSendStatus(AlertStatus.EXECUTION_SUCCESS, "success", 1, 1); Assertions.assertEquals(1, insertCount); } @@ -75,9 +76,9 @@ public void testSendServerStoppedAlert() { alertDao.sendServerStoppedAlert(alertGroupId, host, serverType); alertDao.sendServerStoppedAlert(alertGroupId, host, serverType); long count = alertDao.listPendingAlerts() - .stream() - .filter(alert -> alert.getContent().contains(host)) - .count(); + .stream() + .filter(alert -> alert.getContent().contains(host)) + .count(); Assertions.assertEquals(1L, count); } } diff --git a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/entity/ProcessDefinitionTest.java b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/entity/ProcessDefinitionTest.java index 3844e9295f63..31a40de612f3 100644 --- a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/entity/ProcessDefinitionTest.java +++ b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/entity/ProcessDefinitionTest.java @@ -29,11 +29,13 @@ public class ProcessDefinitionTest { public void getGlobalParamMapTest() { ProcessDefinition taskInstance = new ProcessDefinition(); - //sub process - taskInstance.setGlobalParams("[{\"prop\":\"selenium_global_parameters_1\",\"direct\":\"IN\",\"type\":\"VARCHAR\",\"value\":\"selenium_global_parameters_value_1\"}]"); + // sub process + taskInstance.setGlobalParams( + "[{\"prop\":\"selenium_global_parameters_1\",\"direct\":\"IN\",\"type\":\"VARCHAR\",\"value\":\"selenium_global_parameters_value_1\"}]"); taskInstance.getGlobalParamMap(); - Assertions.assertEquals("{selenium_global_parameters_1=selenium_global_parameters_value_1}",taskInstance.getGlobalParamMap().toString()); + Assertions.assertEquals("{selenium_global_parameters_1=selenium_global_parameters_value_1}", + taskInstance.getGlobalParamMap().toString()); } } diff --git a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/entity/TaskInstanceTest.java b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/entity/TaskInstanceTest.java index 392060afbd0d..cb68cf74948c 100644 --- a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/entity/TaskInstanceTest.java +++ b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/entity/TaskInstanceTest.java @@ -42,19 +42,19 @@ public class TaskInstanceTest { public void testTaskInstanceIsSubProcess() { TaskInstance taskInstance = new TaskInstance(); - //sub process + // sub process taskInstance.setTaskType(TASK_TYPE_SUB_PROCESS); Assertions.assertTrue(taskInstance.isSubProcess()); - //not sub process + // not sub process taskInstance.setTaskType("HTTP"); Assertions.assertFalse(taskInstance.isSubProcess()); - //sub process + // sub process taskInstance.setTaskType(TASK_TYPE_CONDITIONS); Assertions.assertTrue(taskInstance.isConditionsTask()); - //sub process + // sub process taskInstance.setTaskType(TASK_TYPE_DEPENDENT); Assertions.assertTrue(taskInstance.isDependTask()); } diff --git a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/AccessTokenMapperTest.java b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/AccessTokenMapperTest.java index bddc267545d8..b61975988a10 100644 --- a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/AccessTokenMapperTest.java +++ b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/AccessTokenMapperTest.java @@ -153,7 +153,6 @@ public void testSelectAccessTokenPage() throws Exception { Assertions.assertEquals(Integer.valueOf(generalAccessTokenPage.getRecords().size()), emptySize); } - /** * test update */ @@ -161,7 +160,7 @@ public void testSelectAccessTokenPage() throws Exception { public void testUpdate() throws Exception { Integer userId = 1; AccessToken accessToken = createAccessToken(userId); - //update + // update accessToken.setToken("56789"); accessToken.setExpireTime(DateUtils.getCurrentDate()); accessToken.setUpdateTime(DateUtils.getCurrentDate()); @@ -200,7 +199,7 @@ public void testDelete() throws Exception { * @throws Exception */ private Map createAccessTokens( - Integer count, String userName) throws Exception { + Integer count, String userName) throws Exception { User user = createUser(userName); @@ -249,7 +248,7 @@ private User createUser(String userName) { * @return accessToken */ private AccessToken createAccessToken(Integer userId, String userName) { - //insertOne + // insertOne AccessToken accessToken = new AccessToken(); accessToken.setUserName(userName); accessToken.setUserId(userId); @@ -277,4 +276,4 @@ private AccessToken createAccessToken(Integer userId) throws Exception { return createAccessToken(userId, null); } -} \ No newline at end of file +} diff --git a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/AlertGroupMapperTest.java b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/AlertGroupMapperTest.java index 21554e411747..54b112d8c2ce 100644 --- a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/AlertGroupMapperTest.java +++ b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/AlertGroupMapperTest.java @@ -17,9 +17,6 @@ package org.apache.dolphinscheduler.dao.mapper; -import static org.hamcrest.Matchers.greaterThan; -import static org.hamcrest.Matchers.greaterThanOrEqualTo; - import org.apache.dolphinscheduler.common.utils.DateUtils; import org.apache.dolphinscheduler.dao.BaseDaoTest; import org.apache.dolphinscheduler.dao.entity.AlertGroup; @@ -54,19 +51,17 @@ public void testInsert() { } - /** * test selectById */ @Test public void testSelectById() { AlertGroup alertGroup = createAlertGroup(); - //query + // query AlertGroup targetAlert = alertGroupMapper.selectById(alertGroup.getId()); Assertions.assertEquals(alertGroup, targetAlert); } - /** * test page */ @@ -117,7 +112,6 @@ public void testUpdate() { Assertions.assertEquals(alertGroup, resultAlertGroup); } - /** * test delete */ @@ -133,7 +127,6 @@ public void testDelete() { Assertions.assertNull(resultAlertGroup); } - /** * test query by groupname */ @@ -234,7 +227,7 @@ private AlertGroup createAlertGroup() { * @return AlertGroup map */ private Map createAlertGroups( - Integer count, String groupName) { + Integer count, String groupName) { Map alertGroupMap = new HashMap<>(); AlertGroup alertGroup = null; @@ -264,4 +257,4 @@ private Map createAlertGroups(Integer count) { return alertGroupMap; } -} \ No newline at end of file +} diff --git a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/AlertSendStatusMapperTest.java b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/AlertSendStatusMapperTest.java index e807d6927407..01f147e047be 100644 --- a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/AlertSendStatusMapperTest.java +++ b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/AlertSendStatusMapperTest.java @@ -30,6 +30,7 @@ * AlertSendStatus mapper test */ public class AlertSendStatusMapperTest extends BaseDaoTest { + @Autowired private AlertSendStatusMapper alertSendStatusMapper; diff --git a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/AuditLogMapperTest.java b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/AuditLogMapperTest.java index e1a51c900814..a18b407344be 100644 --- a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/AuditLogMapperTest.java +++ b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/AuditLogMapperTest.java @@ -67,4 +67,4 @@ public void testQueryResourceNameByType() { String resourceName = logMapper.queryResourceNameByType("USER", 1); Assertions.assertEquals("admin", resourceName); } -} \ No newline at end of file +} diff --git a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/CommandMapperTest.java b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/CommandMapperTest.java index 87e6ebe828c4..9548dd822384 100644 --- a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/CommandMapperTest.java +++ b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/CommandMapperTest.java @@ -17,7 +17,6 @@ package org.apache.dolphinscheduler.dao.mapper; - import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.enums.CommandType; import org.apache.dolphinscheduler.common.enums.FailureStrategy; diff --git a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/DataSourceUserMapperTest.java b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/DataSourceUserMapperTest.java index e8091769d91b..63fd447470b1 100644 --- a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/DataSourceUserMapperTest.java +++ b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/DataSourceUserMapperTest.java @@ -38,7 +38,7 @@ public class DataSourceUserMapperTest extends BaseDaoTest { * @return DatasourceUser */ private DatasourceUser insertOne() { - //insertOne + // insertOne DatasourceUser dataSourceUser = new DatasourceUser(); dataSourceUser.setUserId(4); dataSourceUser.setDatasourceId(1010); @@ -54,9 +54,9 @@ private DatasourceUser insertOne() { */ @Test public void testUpdate() { - //insertOne + // insertOne DatasourceUser dataSourceUser = insertOne(); - //update + // update dataSourceUser.setUpdateTime(new Date()); int update = dataSourceUserMapper.updateById(dataSourceUser); Assertions.assertEquals(update, 1); @@ -79,7 +79,7 @@ public void testDelete() { @Test public void testQuery() { DatasourceUser dataSourceUser = insertOne(); - //query + // query List dataSources = dataSourceUserMapper.selectList(null); Assertions.assertNotEquals(dataSources.size(), 0); } @@ -103,4 +103,4 @@ public void testDeleteByDatasourceId() { int delete = dataSourceUserMapper.deleteByDatasourceId(dataSourceUser.getDatasourceId()); Assertions.assertNotEquals(delete, 0); } -} \ No newline at end of file +} diff --git a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/DqRuleInputEntryMapperTest.java b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/DqRuleInputEntryMapperTest.java index ec4de0cf9fa7..b50095eed090 100644 --- a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/DqRuleInputEntryMapperTest.java +++ b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/DqRuleInputEntryMapperTest.java @@ -16,20 +16,23 @@ */ package org.apache.dolphinscheduler.dao.mapper; -import com.baomidou.mybatisplus.core.metadata.IPage; -import com.baomidou.mybatisplus.extension.plugins.pagination.Page; import org.apache.dolphinscheduler.dao.BaseDaoTest; import org.apache.dolphinscheduler.dao.entity.DqRule; import org.apache.dolphinscheduler.dao.entity.DqRuleInputEntry; + +import java.util.List; + import org.junit.jupiter.api.Test; import org.springframework.beans.factory.annotation.Autowired; -import java.util.List; +import com.baomidou.mybatisplus.core.metadata.IPage; +import com.baomidou.mybatisplus.extension.plugins.pagination.Page; /** * DQC rule mapper UT. */ public class DqRuleInputEntryMapperTest extends BaseDaoTest { + @Autowired private DqRuleInputEntryMapper dqRuleInputEntryMapper; diff --git a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/EnvironmentWorkerGroupRelationMapperTest.java b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/EnvironmentWorkerGroupRelationMapperTest.java index d683c47ad818..2d41cf02e5b9 100644 --- a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/EnvironmentWorkerGroupRelationMapperTest.java +++ b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/EnvironmentWorkerGroupRelationMapperTest.java @@ -56,7 +56,7 @@ public void clearTestData() { * @return ProcessDefinition */ private EnvironmentWorkerGroupRelation insertOne() { - //insertOne + // insertOne EnvironmentWorkerGroupRelation relation = new EnvironmentWorkerGroupRelation(); relation.setEnvironmentCode(1L); relation.setWorkerGroup("default"); @@ -73,7 +73,7 @@ private EnvironmentWorkerGroupRelation insertOne() { @Test public void testQuery() { insertOne(); - //query + // query List relations = environmentWorkerGroupRelationMapper.selectList(null); Assertions.assertEquals(relations.size(), 1); } @@ -81,14 +81,16 @@ public void testQuery() { @Test public void testQueryByEnvironmentCode() { EnvironmentWorkerGroupRelation relation = insertOne(); - List environmentWorkerGroupRelations = environmentWorkerGroupRelationMapper.queryByEnvironmentCode(1L); + List environmentWorkerGroupRelations = + environmentWorkerGroupRelationMapper.queryByEnvironmentCode(1L); Assertions.assertNotEquals(environmentWorkerGroupRelations.size(), 0); } @Test public void testQueryByWorkerGroupName() { EnvironmentWorkerGroupRelation relation = insertOne(); - List environmentWorkerGroupRelations = environmentWorkerGroupRelationMapper.queryByWorkerGroupName("default"); + List environmentWorkerGroupRelations = + environmentWorkerGroupRelationMapper.queryByWorkerGroupName("default"); Assertions.assertNotEquals(environmentWorkerGroupRelations.size(), 0); } diff --git a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/ErrorCommandMapperTest.java b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/ErrorCommandMapperTest.java index 6c717ca80cea..d4c4b74991bf 100644 --- a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/ErrorCommandMapperTest.java +++ b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/ErrorCommandMapperTest.java @@ -43,7 +43,7 @@ public class ErrorCommandMapperTest extends BaseDaoTest { * @return ErrorCommand */ private ErrorCommand insertOne() { - //insertOne + // insertOne ErrorCommand errorCommand = new ErrorCommand(); errorCommand.setId(10101); errorCommand.setCommandType(CommandType.START_PROCESS); @@ -53,10 +53,6 @@ private ErrorCommand insertOne() { return errorCommand; } - - - - /** * test query */ @@ -79,8 +75,7 @@ public void testQuery() { List commandCounts = errorCommandMapper.countCommandState( null, null, - new Long[0] - ); + new Long[0]); Long[] projectCodeArray = new Long[2]; projectCodeArray[0] = processDefinition.getProjectCode(); @@ -88,8 +83,7 @@ public void testQuery() { List commandCounts2 = errorCommandMapper.countCommandState( null, null, - projectCodeArray - ); + projectCodeArray); Assertions.assertNotEquals(commandCounts.size(), 0); Assertions.assertNotEquals(commandCounts2.size(), 0); diff --git a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/PluginDefineTest.java b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/PluginDefineTest.java index f96853e90324..53e874448829 100644 --- a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/PluginDefineTest.java +++ b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/PluginDefineTest.java @@ -49,7 +49,8 @@ public void testQeryByPluginType() { @Test public void testQueryByNameAndType() { PluginDefine pluginDefine = createPluginDefine(); - PluginDefine pluginDefineSaved = pluginDefineMapper.queryByNameAndType(pluginDefine.getPluginName(), pluginDefine.getPluginType()); + PluginDefine pluginDefineSaved = + pluginDefineMapper.queryByNameAndType(pluginDefine.getPluginName(), pluginDefine.getPluginType()); Assertions.assertNotNull(pluginDefineSaved); Assertions.assertEquals(pluginDefineSaved.getPluginType(), pluginDefine.getPluginType()); Assertions.assertEquals(pluginDefineSaved.getPluginName(), pluginDefine.getPluginName()); diff --git a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/ProcessInstanceMapMapperTest.java b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/ProcessInstanceMapMapperTest.java index a8013a41ffa2..904072a758b1 100644 --- a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/ProcessInstanceMapMapperTest.java +++ b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/ProcessInstanceMapMapperTest.java @@ -37,7 +37,7 @@ public class ProcessInstanceMapMapperTest extends BaseDaoTest { * @return ProcessInstanceMap */ private ProcessInstanceMap insertOne() { - //insertOne + // insertOne ProcessInstanceMap processInstanceMap = new ProcessInstanceMap(); processInstanceMap.setProcessInstanceId(0); processInstanceMap.setParentTaskInstanceId(0); @@ -51,9 +51,9 @@ private ProcessInstanceMap insertOne() { */ @Test public void testUpdate() { - //insertOne + // insertOne ProcessInstanceMap processInstanceMap = insertOne(); - //update + // update processInstanceMap.setParentProcessInstanceId(1); int update = processInstanceMapMapper.updateById(processInstanceMap); Assertions.assertEquals(1, update); @@ -75,7 +75,7 @@ public void testDelete() { @Test public void testQuery() { ProcessInstanceMap processInstanceMap = insertOne(); - //query + // query List dataSources = processInstanceMapMapper.selectList(null); Assertions.assertNotEquals(dataSources.size(), 0); } @@ -102,8 +102,7 @@ public void testDeleteByParentProcessId() { processInstanceMap.setParentProcessInstanceId(100); processInstanceMapMapper.updateById(processInstanceMap); int delete = processInstanceMapMapper.deleteByParentProcessId( - processInstanceMap.getParentProcessInstanceId() - ); + processInstanceMap.getParentProcessInstanceId()); Assertions.assertEquals(1, delete); } @@ -119,10 +118,10 @@ public void querySubIdListByParentId() { processInstanceMapMapper.updateById(processInstanceMap); - List subIds = processInstanceMapMapper.querySubIdListByParentId(processInstanceMap.getParentProcessInstanceId()); + List subIds = + processInstanceMapMapper.querySubIdListByParentId(processInstanceMap.getParentProcessInstanceId()); Assertions.assertNotEquals(subIds.size(), 0); - } -} \ No newline at end of file +} diff --git a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/ProcessInstanceMapperTest.java b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/ProcessInstanceMapperTest.java index 30839c5ea201..01a9b8b92ca1 100644 --- a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/ProcessInstanceMapperTest.java +++ b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/ProcessInstanceMapperTest.java @@ -25,12 +25,11 @@ import org.apache.dolphinscheduler.dao.entity.ProcessDefinition; import org.apache.dolphinscheduler.dao.entity.ProcessInstance; import org.apache.dolphinscheduler.dao.entity.Project; +import org.apache.dolphinscheduler.plugin.task.api.enums.TaskExecutionStatus; import java.util.Date; import java.util.List; -import org.apache.dolphinscheduler.plugin.task.api.enums.TaskExecutionStatus; - import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; import org.springframework.beans.factory.annotation.Autowired; @@ -306,7 +305,8 @@ public void testQueryLastSchedulerProcess() { processInstanceMapper.updateById(processInstance); ProcessInstance processInstance1 = - processInstanceMapper.queryLastSchedulerProcess(processInstance.getProcessDefinitionCode(), null, null, processInstance.getTestFlag()); + processInstanceMapper.queryLastSchedulerProcess(processInstance.getProcessDefinitionCode(), null, null, + processInstance.getTestFlag()); Assertions.assertNotEquals(processInstance1, null); processInstanceMapper.deleteById(processInstance.getId()); } @@ -325,7 +325,8 @@ public void testQueryLastRunningProcess() { WorkflowExecutionStatus.SUBMITTED_SUCCESS.ordinal()}; ProcessInstance processInstance1 = processInstanceMapper - .queryLastRunningProcess(processInstance.getProcessDefinitionCode(), null, null, processInstance.getTestFlag(), stateArray); + .queryLastRunningProcess(processInstance.getProcessDefinitionCode(), null, null, + processInstance.getTestFlag(), stateArray); Assertions.assertNotEquals(processInstance1, null); processInstanceMapper.deleteById(processInstance.getId()); @@ -342,12 +343,14 @@ public void testQueryLastManualProcess() { Date start = new Date(2019 - 1900, 1 - 1, 01, 0, 0, 0); Date end = new Date(2019 - 1900, 1 - 1, 01, 5, 0, 0); ProcessInstance processInstance1 = - processInstanceMapper.queryLastManualProcess(processInstance.getProcessDefinitionCode(), start, end, processInstance.getTestFlag()); + processInstanceMapper.queryLastManualProcess(processInstance.getProcessDefinitionCode(), start, end, + processInstance.getTestFlag()); Assertions.assertEquals(processInstance1.getId(), processInstance.getId()); start = new Date(2019 - 1900, 1 - 1, 01, 1, 0, 0); processInstance1 = - processInstanceMapper.queryLastManualProcess(processInstance.getProcessDefinitionCode(), start, end, processInstance.getTestFlag()); + processInstanceMapper.queryLastManualProcess(processInstance.getProcessDefinitionCode(), start, end, + processInstance.getTestFlag()); Assertions.assertNull(processInstance1); processInstanceMapper.deleteById(processInstance.getId()); diff --git a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/ProcessTaskRelationLogMapperTest.java b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/ProcessTaskRelationLogMapperTest.java index 1316fd26454c..3b82fd8e7f27 100644 --- a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/ProcessTaskRelationLogMapperTest.java +++ b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/ProcessTaskRelationLogMapperTest.java @@ -38,7 +38,7 @@ public class ProcessTaskRelationLogMapperTest extends BaseDaoTest { * @return ProcessDefinition */ private ProcessTaskRelationLog insertOne() { - //insertOne + // insertOne ProcessTaskRelationLog processTaskRelationLog = new ProcessTaskRelationLog(); processTaskRelationLog.setName("def 1"); processTaskRelationLog.setProcessDefinitionVersion(1); diff --git a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/ProcessTaskRelationMapperTest.java b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/ProcessTaskRelationMapperTest.java index bb63978594f4..e83be99be0c2 100644 --- a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/ProcessTaskRelationMapperTest.java +++ b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/ProcessTaskRelationMapperTest.java @@ -39,7 +39,7 @@ public class ProcessTaskRelationMapperTest extends BaseDaoTest { * @return ProcessDefinition */ private ProcessTaskRelation insertOne() { - //insertOne + // insertOne ProcessTaskRelation processTaskRelation = new ProcessTaskRelation(); processTaskRelation.setName("def 1"); diff --git a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/ProjectMapperTest.java b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/ProjectMapperTest.java index 2dc27d273fdb..066995818a80 100644 --- a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/ProjectMapperTest.java +++ b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/ProjectMapperTest.java @@ -50,7 +50,7 @@ public class ProjectMapperTest extends BaseDaoTest { * @return Project */ private Project insertOne() { - //insertOne + // insertOne Project project = new Project(); project.setName("ut project"); project.setUserId(111); @@ -66,10 +66,10 @@ private Project insertOne() { */ @Test public void testUpdate() { - //insertOne + // insertOne Project project = insertOne(); project.setCreateTime(new Date()); - //update + // update int update = projectMapper.updateById(project); Assertions.assertEquals(update, 1); } @@ -90,7 +90,7 @@ public void testDelete() { @Test public void testQuery() { Project project = insertOne(); - //query + // query List projects = projectMapper.selectList(null); Assertions.assertNotEquals(projects.size(), 0); } @@ -146,15 +146,13 @@ public void testQueryProjectListPaging() { Page page = new Page(1, 3); IPage projectIPage = projectMapper.queryProjectListPaging( - page, - null, - null - ); + page, + null, + null); IPage projectIPage1 = projectMapper.queryProjectListPaging( - page, - null, - project.getName() - ); + page, + null, + project.getName()); Assertions.assertEquals(projectIPage.getTotal(), 1); Assertions.assertEquals(projectIPage1.getTotal(), 1); } @@ -192,8 +190,7 @@ public void testQueryProjectExceptUserId() { Project project = insertOne(); List projects = projectMapper.queryProjectExceptUserId( - 100000 - ); + 100000); Assertions.assertNotEquals(projects.size(), 0); } @@ -230,4 +227,4 @@ public void testListAuthorizedProjects() { Assertions.assertEquals(projects.size(), 0); } -} \ No newline at end of file +} diff --git a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/ProjectUserMapperTest.java b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/ProjectUserMapperTest.java index ee7d0c5e2602..0307ba8697f6 100644 --- a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/ProjectUserMapperTest.java +++ b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/ProjectUserMapperTest.java @@ -38,7 +38,7 @@ public class ProjectUserMapperTest extends BaseDaoTest { * @return ProjectUser */ private ProjectUser insertOne() { - //insertOne + // insertOne ProjectUser projectUser = new ProjectUser(); projectUser.setProjectId(1010); projectUser.setUserId(111); @@ -51,10 +51,10 @@ private ProjectUser insertOne() { */ @Test public void testUpdate() { - //insertOne + // insertOne ProjectUser projectUser = insertOne(); projectUser.setCreateTime(new Date()); - //update + // update int update = projectUserMapper.updateById(projectUser); Assertions.assertEquals(update, 1); } @@ -75,7 +75,7 @@ public void testDelete() { @Test public void testQuery() { ProjectUser projectUser = insertOne(); - //query + // query List projectUsers = projectUserMapper.selectList(null); Assertions.assertNotEquals(projectUsers.size(), 0); } @@ -98,8 +98,9 @@ public void testDeleteProjectRelation() { @Test public void testQueryProjectRelation() { ProjectUser projectUser = insertOne(); - ProjectUser projectUser1 = projectUserMapper.queryProjectRelation(projectUser.getProjectId(), projectUser.getUserId()); + ProjectUser projectUser1 = + projectUserMapper.queryProjectRelation(projectUser.getProjectId(), projectUser.getUserId()); Assertions.assertNotEquals(projectUser1, null); } -} \ No newline at end of file +} diff --git a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/QueueMapperTest.java b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/QueueMapperTest.java index 8dc3c6d7abea..94ed6907b1e9 100644 --- a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/QueueMapperTest.java +++ b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/QueueMapperTest.java @@ -42,7 +42,7 @@ public class QueueMapperTest extends BaseDaoTest { * @return Queue */ private Queue insertOne() { - //insertOne + // insertOne Queue queue = new Queue(); queue.setQueueName("queue"); queue.setQueue("queue"); @@ -57,10 +57,10 @@ private Queue insertOne() { */ @Test public void testUpdate() { - //insertOne + // insertOne Queue queue = insertOne(); queue.setCreateTime(new Date()); - //update + // update int update = queueMapper.updateById(queue); Assertions.assertEquals(1, update); } @@ -81,7 +81,7 @@ public void testDelete() { @Test public void testQuery() { Queue queue = insertOne(); - //query + // query List queues = queueMapper.selectList(null); Assertions.assertNotEquals(queues.size(), 0); } @@ -93,12 +93,12 @@ public void testQuery() { public void testQueryQueuePaging() { Queue queue = insertOne(); - Page page = new Page(1,3); + Page page = new Page(1, 3); - IPage queueIPage= queueMapper.queryQueuePaging(page, Collections.singletonList(queue.getId()), null); + IPage queueIPage = queueMapper.queryQueuePaging(page, Collections.singletonList(queue.getId()), null); Assertions.assertNotEquals(queueIPage.getTotal(), 0); - queueIPage= queueMapper.queryQueuePaging(page, Collections.singletonList(queue.getId()), queue.getQueueName()); + queueIPage = queueMapper.queryQueuePaging(page, Collections.singletonList(queue.getId()), queue.getQueueName()); Assertions.assertNotEquals(queueIPage.getTotal(), 0); } diff --git a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/ResourceUserMapperTest.java b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/ResourceUserMapperTest.java index 9f6d1ead6df1..dd57bc2aca96 100644 --- a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/ResourceUserMapperTest.java +++ b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/ResourceUserMapperTest.java @@ -19,13 +19,14 @@ import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.dao.BaseDaoTest; import org.apache.dolphinscheduler.dao.entity.ResourcesUser; -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.Test; -import org.springframework.beans.factory.annotation.Autowired; import java.util.Date; import java.util.List; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; +import org.springframework.beans.factory.annotation.Autowired; + public class ResourceUserMapperTest extends BaseDaoTest { @Autowired @@ -35,8 +36,8 @@ public class ResourceUserMapperTest extends BaseDaoTest { * insert * @return ResourcesUser */ - private ResourcesUser insertOne(){ - //insertOne + private ResourcesUser insertOne() { + // insertOne ResourcesUser resourcesUser = new ResourcesUser(); resourcesUser.setCreateTime(new Date()); resourcesUser.setUpdateTime(new Date()); @@ -51,11 +52,11 @@ private ResourcesUser insertOne(){ * test update */ @Test - public void testUpdate(){ - //insertOne + public void testUpdate() { + // insertOne ResourcesUser queue = insertOne(); queue.setCreateTime(new Date()); - //update + // update int update = resourceUserMapper.updateById(queue); Assertions.assertEquals(1, update); } @@ -64,7 +65,7 @@ public void testUpdate(){ * test delete */ @Test - public void testDelete(){ + public void testDelete() { ResourcesUser queue = insertOne(); int delete = resourceUserMapper.deleteById(queue.getId()); Assertions.assertEquals(1, delete); @@ -76,7 +77,7 @@ public void testDelete(){ @Test public void testQuery() { ResourcesUser queue = insertOne(); - //query + // query List queues = resourceUserMapper.selectList(null); Assertions.assertNotEquals(queues.size(), 0); } @@ -107,4 +108,4 @@ public void testDeleteResourceUserArray() { resourceIdArray); Assertions.assertNotEquals(delete, 0); } -} \ No newline at end of file +} diff --git a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/SessionMapperTest.java b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/SessionMapperTest.java index 93d65d20d0b6..e2d374756985 100644 --- a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/SessionMapperTest.java +++ b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/SessionMapperTest.java @@ -38,7 +38,7 @@ public class SessionMapperTest extends BaseDaoTest { * @return Session */ private Session insertOne() { - //insertOne + // insertOne Session session = new Session(); session.setId(UUID.randomUUID().toString()); session.setLastLoginTime(new Date()); @@ -51,11 +51,11 @@ private Session insertOne() { * test update */ @Test - public void testUpdate(){ - //insertOne + public void testUpdate() { + // insertOne Session session = insertOne(); session.setLastLoginTime(new Date()); - //update + // update int update = sessionMapper.updateById(session); Assertions.assertEquals(update, 1); } @@ -64,7 +64,7 @@ public void testUpdate(){ * test delete */ @Test - public void testDelete(){ + public void testDelete() { Session session = insertOne(); int delete = sessionMapper.deleteById(session.getId()); Assertions.assertEquals(delete, 1); @@ -76,7 +76,7 @@ public void testDelete(){ @Test public void testQuery() { Session session = insertOne(); - //query + // query List sessions = sessionMapper.selectList(null); Assertions.assertNotEquals(sessions.size(), 0); } @@ -91,4 +91,4 @@ public void testQueryByUserId() { Assertions.assertNotEquals(sessions.size(), 0); } -} \ No newline at end of file +} diff --git a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/TaskDefinitionMapperTest.java b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/TaskDefinitionMapperTest.java index 72dbd148d624..aa8fa7161e5d 100644 --- a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/TaskDefinitionMapperTest.java +++ b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/TaskDefinitionMapperTest.java @@ -181,6 +181,6 @@ public void testNullLocalParamsOfTaskParams() { "{\"failRetryTimes\":\"0\",\"timeoutNotifyStrategy\":\"\",\"code\":\"5195043558720\",\"flag\":\"YES\",\"environmentCode\":\"-1\",\"taskDefinitionIndex\":2,\"taskPriority\":\"MEDIUM\",\"taskParams\":\"{\\\"preStatements\\\":null,\\\"postStatements\\\":null,\\\"type\\\":\\\"ADB_MYSQL\\\",\\\"database\\\":\\\"lijia\\\",\\\"sql\\\":\\\"create table nation_${random_serial_number} as select * from nation\\\",\\\"localParams\\\":null,\\\"Name\\\":\\\"create_table_as_select_nation\\\",\\\"FailRetryTimes\\\":0,\\\"dbClusterId\\\":\\\"amv-bp10o45925jpx959\\\",\\\"sendEmail\\\":false,\\\"displayRows\\\":10,\\\"limit\\\":10000,\\\"agentSource\\\":\\\"Workflow\\\",\\\"agentVersion\\\":\\\"Unkown\\\"}\",\"timeout\":\"0\",\"taskType\":\"ADB_MYSQL\",\"timeoutFlag\":\"CLOSE\",\"projectCode\":\"5191800302720\",\"name\":\"create_table_as_select_nation\",\"delayTime\":\"0\",\"workerGroup\":\"default\"}"; TaskDefinition definition = JSONUtils.parseObject(definitionJson, TaskDefinition.class); - Assertions.assertNull(definition.getTaskParamMap(),"Serialize the task definition success"); + Assertions.assertNull(definition.getTaskParamMap(), "Serialize the task definition success"); } } diff --git a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/TaskGroupMapperTest.java b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/TaskGroupMapperTest.java index 03d7bfc05543..f45beb2a7262 100644 --- a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/TaskGroupMapperTest.java +++ b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/TaskGroupMapperTest.java @@ -26,7 +26,6 @@ import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; - import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; diff --git a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/TaskInstanceMapperTest.java b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/TaskInstanceMapperTest.java index 849ffa9c8d62..be71c3561fae 100644 --- a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/TaskInstanceMapperTest.java +++ b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/TaskInstanceMapperTest.java @@ -183,8 +183,8 @@ public void testFindValidTaskListByProcessId() { task2.setFlag(Flag.NO); taskInstanceMapper.updateById(task2); List taskInstances1 = taskInstanceMapper.findValidTaskListByProcessId(task.getProcessInstanceId(), - Flag.NO, - processInstance.getTestFlag()); + Flag.NO, + processInstance.getTestFlag()); taskInstanceMapper.deleteById(task2.getId()); taskInstanceMapper.deleteById(task.getId()); Assertions.assertNotEquals(taskInstances.size(), 0); diff --git a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/TenantMapperTest.java b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/TenantMapperTest.java index 225a997de0ab..fc6c2479d08b 100644 --- a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/TenantMapperTest.java +++ b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/TenantMapperTest.java @@ -43,8 +43,8 @@ public class TenantMapperTest extends BaseDaoTest { * insert * @return Tenant */ - private Tenant insertOne(){ - //insertOne + private Tenant insertOne() { + // insertOne Tenant tenant = new Tenant(); tenant.setCreateTime(new Date()); tenant.setUpdateTime(new Date()); @@ -57,11 +57,11 @@ private Tenant insertOne(){ * test update */ @Test - public void testUpdate(){ - //insertOne + public void testUpdate() { + // insertOne Tenant tenant = insertOne(); tenant.setUpdateTime(new Date()); - //update + // update int update = tenantMapper.updateById(tenant); Assertions.assertEquals(1, update); } @@ -70,7 +70,7 @@ public void testUpdate(){ * test delete */ @Test - public void testDelete(){ + public void testDelete() { Tenant tenant = insertOne(); int delete = tenantMapper.deleteById(tenant.getId()); Assertions.assertEquals(1, delete); @@ -82,7 +82,7 @@ public void testDelete(){ @Test public void testQuery() { Tenant tenant = insertOne(); - //query + // query List tenants = tenantMapper.selectList(null); Assertions.assertNotEquals(tenants.size(), 0); } @@ -98,7 +98,6 @@ public void testQueryById() { queue.setQueue("ut queue"); queueMapper.insert(queue); - Tenant tenant = insertOne(); tenant.setQueueId(queue.getId()); tenantMapper.updateById(tenant); @@ -136,8 +135,9 @@ public void testQueryTenantPaging() { tenantMapper.updateById(tenant); Page page = new Page(1, 3); - //tenant.getTenantCode() used instead of tenant.getTenantName() - IPage tenantIPage = tenantMapper.queryTenantPaging(page, Collections.singletonList(tenant.getId()), tenant.getTenantCode()); + // tenant.getTenantCode() used instead of tenant.getTenantName() + IPage tenantIPage = + tenantMapper.queryTenantPaging(page, Collections.singletonList(tenant.getId()), tenant.getTenantCode()); Assertions.assertNotEquals(tenantIPage.getTotal(), 0); } @@ -148,4 +148,4 @@ public void testExistTenant() { insertOne(); Assertions.assertTrue(tenantMapper.existTenant(tenantCode)); } -} \ No newline at end of file +} diff --git a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/UDFUserMapperTest.java b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/UDFUserMapperTest.java index 16aae713d215..ec90fa2c82ea 100644 --- a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/UDFUserMapperTest.java +++ b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/UDFUserMapperTest.java @@ -45,7 +45,7 @@ public class UDFUserMapperTest extends BaseDaoTest { * insert * @return UDFUser */ - private UDFUser insertOne(){ + private UDFUser insertOne() { UDFUser udfUser = new UDFUser(); udfUser.setUdfId(1); udfUser.setUserId(1); @@ -61,7 +61,7 @@ private UDFUser insertOne(){ * @param udfFunc udfFunc * @return UDFUser */ - private UDFUser insertOne(User user,UdfFunc udfFunc){ + private UDFUser insertOne(User user, UdfFunc udfFunc) { UDFUser udfUser = new UDFUser(); udfUser.setUdfId(udfFunc.getId()); udfUser.setUserId(user.getId()); @@ -71,12 +71,11 @@ private UDFUser insertOne(User user,UdfFunc udfFunc){ return udfUser; } - /** * insert one user * @return User */ - private User insertOneUser(){ + private User insertOneUser() { User user = new User(); user.setUserName("user1"); user.setUserPassword("1"); @@ -94,7 +93,7 @@ private User insertOneUser(){ * insert one udf * @return UdfFunc */ - private UdfFunc insertOneUdfFunc(){ + private UdfFunc insertOneUdfFunc() { UdfFunc udfFunc = new UdfFunc(); udfFunc.setFuncName("dolphin_udf_func"); udfFunc.setClassName("org.apache.dolphinscheduler.test.mr"); @@ -111,12 +110,12 @@ private UdfFunc insertOneUdfFunc(){ * test update */ @Test - public void testUpdate(){ - //insertOneUser + public void testUpdate() { + // insertOneUser User user = insertOneUser(); - //insertOneUdfFunc + // insertOneUdfFunc UdfFunc udfFunc = insertOneUdfFunc(); - //insertOne + // insertOne UDFUser udfUser = insertOne(user, udfFunc); udfUser.setUserId(2); udfUser.setUdfId(2); @@ -129,12 +128,12 @@ public void testUpdate(){ * test delete */ @Test - public void testDelete(){ - //insertOneUser + public void testDelete() { + // insertOneUser User user = insertOneUser(); - //insertOneUdfFunc + // insertOneUdfFunc UdfFunc udfFunc = insertOneUdfFunc(); - //insertOne + // insertOne UDFUser udfUser = insertOne(user, udfFunc); int delete = udfUserMapper.deleteById(udfUser.getId()); Assertions.assertEquals(delete, 1); @@ -144,10 +143,10 @@ public void testDelete(){ * test query */ @Test - public void testQuery(){ - //insertOne + public void testQuery() { + // insertOne UDFUser udfUser = insertOne(); - //query + // query List udfUserList = udfUserMapper.selectList(null); Assertions.assertNotEquals(udfUserList.size(), 0); } @@ -157,11 +156,11 @@ public void testQuery(){ */ @Test public void testDeleteByUserId() { - //insertOneUser + // insertOneUser User user = insertOneUser(); - //insertOneUdfFunc + // insertOneUdfFunc UdfFunc udfFunc = insertOneUdfFunc(); - //insertOne + // insertOne UDFUser udfUser = insertOne(user, udfFunc); int delete = udfUserMapper.deleteByUserId(user.getId()); Assertions.assertEquals(delete, 1); @@ -173,13 +172,13 @@ public void testDeleteByUserId() { */ @Test public void testDeleteByUdfFuncId() { - //insertOneUser + // insertOneUser User user = insertOneUser(); - //insertOneUdfFunc + // insertOneUdfFunc UdfFunc udfFunc = insertOneUdfFunc(); - //insertOne + // insertOne UDFUser udfUser = insertOne(user, udfFunc); int delete = udfUserMapper.deleteByUdfFuncId(udfFunc.getId()); Assertions.assertEquals(delete, 1); } -} \ No newline at end of file +} diff --git a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/UserMapperTest.java b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/UserMapperTest.java index 600fda93f9e0..b9a1a0b01f0f 100644 --- a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/UserMapperTest.java +++ b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/UserMapperTest.java @@ -38,6 +38,7 @@ import com.baomidou.mybatisplus.extension.plugins.pagination.Page; public class UserMapperTest extends BaseDaoTest { + @Autowired private UserMapper userMapper; @@ -119,7 +120,7 @@ private User insertOne(Queue queue, Tenant tenant) { * @return AlertGroup */ private AlertGroup insertOneAlertGroup() { - //insertOne + // insertOne AlertGroup alertGroup = new AlertGroup(); alertGroup.setGroupName("alert group 1"); alertGroup.setDescription("alert test1"); @@ -137,7 +138,7 @@ private AlertGroup insertOneAlertGroup() { * @return AccessToken */ private AccessToken insertOneAccessToken(User user) { - //insertOne + // insertOne AccessToken accessToken = new AccessToken(); accessToken.setUserId(user.getId()); accessToken.setToken("secrettoken"); @@ -201,9 +202,9 @@ private Queue insertOneQueue() { */ @Test public void testUpdate() { - //insertOne + // insertOne User user = insertOne(); - //update + // update user.setEmail("xx-update@126.com"); user.setUserName("user1_update"); user.setUserType(UserType.ADMIN_USER); @@ -216,9 +217,9 @@ public void testUpdate() { */ @Test public void testDelete() { - //insertOne + // insertOne User user = insertOne(); - //delete + // delete int delete = userMapper.deleteById(user.getId()); Assertions.assertEquals(delete, 1); } @@ -228,9 +229,9 @@ public void testDelete() { */ @Test public void testQuery() { - //insertOne + // insertOne User user = insertOne(); - //query + // query List userList = userMapper.selectList(null); Assertions.assertNotEquals(userList.size(), 0); } @@ -240,26 +241,25 @@ public void testQuery() { */ @Test public void testQueryAllGeneralUser() { - //insertOne + // insertOne User user = insertOne(); - //queryAllGeneralUser + // queryAllGeneralUser List userList = userMapper.queryAllGeneralUser(); Assertions.assertNotEquals(userList.size(), 0); } - /** * test page */ @Test public void testQueryUserPaging() { - //insertOneQueue + // insertOneQueue Queue queue = insertOneQueue(); - //insertOneTenant + // insertOneTenant Tenant tenant = insertOneTenant(); - //insertOne + // insertOne User user = insertOne(queue, tenant); - //queryUserPaging + // queryUserPaging Page page = new Page(1, 3); IPage userIPage = userMapper.queryUserPaging(page, user.getUserName()); Assertions.assertNotEquals(userIPage.getTotal(), 0); @@ -270,11 +270,11 @@ public void testQueryUserPaging() { */ @Test public void testQueryDetailsById() { - //insertOneQueue and insertOneTenant + // insertOneQueue and insertOneTenant Queue queue = insertOneQueue(); Tenant tenant = insertOneTenant(queue); User user = insertOne(queue, tenant); - //queryDetailsById + // queryDetailsById User queryUser = userMapper.queryDetailsById(user.getId()); Assertions.assertEquals(user.getUserName(), queryUser.getUserName()); } @@ -284,11 +284,11 @@ public void testQueryDetailsById() { */ @Test public void testQueryTenantCodeByUserId() { - //insertOneTenant + // insertOneTenant Tenant tenant = insertOneTenant(); - //insertOne + // insertOne User user = insertOne(tenant); - //queryTenantCodeByUserId + // queryTenantCodeByUserId User queryUser = userMapper.queryTenantCodeByUserId(user.getId()); Assertions.assertEquals(queryUser, user); } @@ -298,11 +298,11 @@ public void testQueryTenantCodeByUserId() { */ @Test public void testQueryUserByToken() { - //insertOne + // insertOne User user = insertOne(); - //insertOneAccessToken + // insertOneAccessToken AccessToken accessToken = insertOneAccessToken(user); - //queryUserByToken + // queryUserByToken User userToken = userMapper.queryUserByToken(accessToken.getToken(), new Date()); Assertions.assertEquals(userToken, user); @@ -310,7 +310,7 @@ public void testQueryUserByToken() { @Test public void selectByIds() { - //insertOne + // insertOne User user = insertOne(); List userIds = new ArrayList<>(); userIds.add(user.getId()); diff --git a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/WorkFlowLineageMapperTest.java b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/WorkFlowLineageMapperTest.java index a607092ee73f..1d0083d11bdf 100644 --- a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/WorkFlowLineageMapperTest.java +++ b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/WorkFlowLineageMapperTest.java @@ -54,7 +54,7 @@ public class WorkFlowLineageMapperTest extends BaseDaoTest { * @return ProcessDefinition */ private ProcessTaskRelation insertOneProcessTaskRelation() { - //insertOne + // insertOne ProcessTaskRelation processTaskRelation = new ProcessTaskRelation(); processTaskRelation.setName("def 1"); @@ -75,7 +75,7 @@ private ProcessTaskRelation insertOneProcessTaskRelation() { * */ private void insertOneProcessDefinition() { - //insertOne + // insertOne ProcessDefinition processDefinition = new ProcessDefinition(); processDefinition.setCode(1L); processDefinition.setName("def 1"); @@ -91,7 +91,7 @@ private void insertOneProcessDefinition() { * */ private void insertOneSchedule(int id) { - //insertOne + // insertOne Schedule schedule = new Schedule(); schedule.setStartTime(new Date()); schedule.setEndTime(new Date()); @@ -111,15 +111,17 @@ public void testQueryWorkFlowLineageByName() { ProcessDefinition processDefinition = processDefinitionMapper.queryByCode(1L); insertOneSchedule(processDefinition.getId()); - List workFlowLineages = workFlowLineageMapper.queryWorkFlowLineageByName(processDefinition.getProjectCode(), processDefinition.getName()); + List workFlowLineages = workFlowLineageMapper + .queryWorkFlowLineageByName(processDefinition.getProjectCode(), processDefinition.getName()); Assertions.assertNotEquals(workFlowLineages.size(), 0); } @Test public void testQueryCodeRelation() { ProcessTaskRelation processTaskRelation = insertOneProcessTaskRelation(); - List workFlowLineages = workFlowLineageMapper.queryProcessLineageByCode(processTaskRelation.getProjectCode(), - processTaskRelation.getProcessDefinitionCode()); + List workFlowLineages = + workFlowLineageMapper.queryProcessLineageByCode(processTaskRelation.getProjectCode(), + processTaskRelation.getProcessDefinitionCode()); Assertions.assertNotEquals(workFlowLineages.size(), 0); } @@ -128,7 +130,8 @@ public void testQueryWorkFlowLineage() { insertOneProcessDefinition(); ProcessDefinition processDefinition = processDefinitionMapper.queryByCode(1L); insertOneProcessTaskRelation(); - List workFlowLineages = workFlowLineageMapper.queryProcessLineage(processDefinition.getProjectCode()); + List workFlowLineages = + workFlowLineageMapper.queryProcessLineage(processDefinition.getProjectCode()); Assertions.assertNotEquals(workFlowLineages.size(), 0); } @@ -137,7 +140,8 @@ public void testQueryWorkFlowLineageByCode() { insertOneProcessDefinition(); ProcessDefinition processDefinition = processDefinitionMapper.queryByCode(1L); insertOneSchedule(processDefinition.getId()); - WorkFlowLineage workFlowLineages = workFlowLineageMapper.queryWorkFlowLineageByCode(processDefinition.getProjectCode(), processDefinition.getCode()); + WorkFlowLineage workFlowLineages = workFlowLineageMapper + .queryWorkFlowLineageByCode(processDefinition.getProjectCode(), processDefinition.getCode()); Assertions.assertNotNull(workFlowLineages); } diff --git a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/WorkerGroupMapperTest.java b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/WorkerGroupMapperTest.java index 6ddff0b56051..afd474193ac0 100644 --- a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/WorkerGroupMapperTest.java +++ b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/WorkerGroupMapperTest.java @@ -17,12 +17,13 @@ package org.apache.dolphinscheduler.dao.mapper; +import org.apache.dolphinscheduler.dao.BaseDaoTest; +import org.apache.dolphinscheduler.dao.entity.WorkerGroup; + import java.util.Date; import java.util.List; -import javax.annotation.Resource; -import org.apache.dolphinscheduler.dao.BaseDaoTest; -import org.apache.dolphinscheduler.dao.entity.WorkerGroup; +import javax.annotation.Resource; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.Assertions; @@ -49,7 +50,7 @@ public void after() { */ public void clearTestData() { workerGroupMapper.queryAllWorkerGroup() - .forEach(workerGroup -> workerGroupMapper.deleteById(workerGroup.getId())); + .forEach(workerGroup -> workerGroupMapper.deleteById(workerGroup.getId())); } /** @@ -113,4 +114,4 @@ public void delete() { int delete = workerGroupMapper.deleteById(workerGroup.getId()); Assertions.assertEquals(1, delete); } -} \ No newline at end of file +} diff --git a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/upgrade/ProcessDefinitionDaoTest.java b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/upgrade/ProcessDefinitionDaoTest.java index 9480aa44040f..6a1599360bc9 100644 --- a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/upgrade/ProcessDefinitionDaoTest.java +++ b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/upgrade/ProcessDefinitionDaoTest.java @@ -29,32 +29,32 @@ @ActiveProfiles("h2") public class ProcessDefinitionDaoTest { + @Autowired private DataSource dataSource; final ProcessDefinitionDao processDefinitionDao = new ProcessDefinitionDao(); @Test public void testQueryAllProcessDefinition() { - //Map processDefinitionJsonMap = processDefinitionDao.queryAllProcessDefinition(dataSource.getConnection()); - //assertThat(processDefinitionJsonMap.size(),greaterThanOrEqualTo(0)); + // Map processDefinitionJsonMap = + // processDefinitionDao.queryAllProcessDefinition(dataSource.getConnection()); + // assertThat(processDefinitionJsonMap.size(),greaterThanOrEqualTo(0)); } @Test public void testUpdateProcessDefinitionJson() { - Map processDefinitionJsonMap = new HashMap<>(); - processDefinitionJsonMap.put(1,"test"); - //processDefinitionDao.updateProcessDefinitionJson(dataSource.getConnection(),processDefinitionJsonMap); + Map processDefinitionJsonMap = new HashMap<>(); + processDefinitionJsonMap.put(1, "test"); + // processDefinitionDao.updateProcessDefinitionJson(dataSource.getConnection(),processDefinitionJsonMap); } @Test public void testQueryAllProcessDefinitionException() { - //processDefinitionDao.queryAllProcessDefinition(null); + // processDefinitionDao.queryAllProcessDefinition(null); } @Test public void testUpdateProcessDefinitionJsonException() { - Assertions.assertThrows(Exception.class, () -> - processDefinitionDao.updateProcessDefinitionJson(null, null) - ); + Assertions.assertThrows(Exception.class, () -> processDefinitionDao.updateProcessDefinitionJson(null, null)); } } diff --git a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/upgrade/SchemaUtilsTest.java b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/upgrade/SchemaUtilsTest.java index 3b8885ad40ce..cc3bfeaea9f5 100644 --- a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/upgrade/SchemaUtilsTest.java +++ b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/upgrade/SchemaUtilsTest.java @@ -71,6 +71,6 @@ public void testGetAllSchemaList() { } catch (IOException ex) { Assertions.fail(ex.getMessage()); } - Assertions.assertFalse(CollectionUtils.isEmpty(list),"Can not find any schema files"); + Assertions.assertFalse(CollectionUtils.isEmpty(list), "Can not find any schema files"); } } diff --git a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/upgrade/WorkerGroupDaoTest.java b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/upgrade/WorkerGroupDaoTest.java index 3f1e1eb7097b..0c36d5f136ab 100644 --- a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/upgrade/WorkerGroupDaoTest.java +++ b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/upgrade/WorkerGroupDaoTest.java @@ -18,8 +18,6 @@ import org.apache.dolphinscheduler.dao.DaoConfiguration; -import java.util.Map; - import javax.sql.DataSource; import org.junit.jupiter.api.Assertions; @@ -36,15 +34,16 @@ @ExtendWith(MockitoExtension.class) @SpringBootApplication(scanBasePackageClasses = DaoConfiguration.class) public class WorkerGroupDaoTest { + @Autowired protected DataSource dataSource; @Test public void testQueryQueryAllOldWorkerGroupException() throws Exception { Assertions.assertThrows(Exception.class, () -> { - WorkerGroupDao workerGroupDao = new WorkerGroupDao(); - workerGroupDao.queryAllOldWorkerGroup(null); - }); + WorkerGroupDao workerGroupDao = new WorkerGroupDao(); + workerGroupDao.queryAllOldWorkerGroup(null); + }); } diff --git a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/utils/ResourceProcessDefinitionUtilsTest.java b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/utils/ResourceProcessDefinitionUtilsTest.java index 01135f3d4352..d12c5e19b73d 100644 --- a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/utils/ResourceProcessDefinitionUtilsTest.java +++ b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/utils/ResourceProcessDefinitionUtilsTest.java @@ -16,26 +16,27 @@ */ package org.apache.dolphinscheduler.dao.utils; -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.Test; - import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + /** * resource process definition utils test */ public class ResourceProcessDefinitionUtilsTest { + @Test - public void getResourceProcessDefinitionMapTest(){ - List> mapList = new ArrayList<>(); - Map map = new HashMap<>(); - map.put("code",1L); - map.put("resource_ids","1,2,3"); + public void getResourceProcessDefinitionMapTest() { + List> mapList = new ArrayList<>(); + Map map = new HashMap<>(); + map.put("code", 1L); + map.put("resource_ids", "1,2,3"); mapList.add(map); Assertions.assertNotNull(ResourceProcessDefinitionUtils.getResourceProcessDefinitionMap(mapList)); } -} \ No newline at end of file +} diff --git a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/utils/TaskInstanceUtilsTest.java b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/utils/TaskInstanceUtilsTest.java index 497c2fb8819e..cb74aeb44e03 100644 --- a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/utils/TaskInstanceUtilsTest.java +++ b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/utils/TaskInstanceUtilsTest.java @@ -40,4 +40,4 @@ void copyTaskInstance() { Assertions.assertEquals(target.getId(), source.getId()); Assertions.assertEquals(target.getName(), source.getName()); } -} \ No newline at end of file +} diff --git a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/utils/WorkflowUtilsTest.java b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/utils/WorkflowUtilsTest.java index 985066cbdbe8..13b839849d6b 100644 --- a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/utils/WorkflowUtilsTest.java +++ b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/utils/WorkflowUtilsTest.java @@ -17,10 +17,12 @@ package org.apache.dolphinscheduler.dao.utils; -import java.util.Date; import org.apache.dolphinscheduler.common.enums.WorkflowExecutionStatus; import org.apache.dolphinscheduler.common.utils.DateUtils; import org.apache.dolphinscheduler.dao.entity.ProcessInstance; + +import java.util.Date; + import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; @@ -48,4 +50,4 @@ public void testGetWorkflowInstanceDuration() { String successDuration = WorkflowUtils.getWorkflowInstanceDuration(processInstance); Assertions.assertEquals("1d 1h 10m 10s", successDuration); } -} \ No newline at end of file +} diff --git a/dolphinscheduler-data-quality/src/main/java/org/apache/dolphinscheduler/data/quality/DataQualityApplication.java b/dolphinscheduler-data-quality/src/main/java/org/apache/dolphinscheduler/data/quality/DataQualityApplication.java index cf37df8f7b94..191e557f440e 100644 --- a/dolphinscheduler-data-quality/src/main/java/org/apache/dolphinscheduler/data/quality/DataQualityApplication.java +++ b/dolphinscheduler-data-quality/src/main/java/org/apache/dolphinscheduler/data/quality/DataQualityApplication.java @@ -50,7 +50,8 @@ public static void main(String[] args) throws Exception { String dataQualityParameter = args[0]; - DataQualityConfiguration dataQualityConfiguration = JsonUtils.fromJson(dataQualityParameter,DataQualityConfiguration.class); + DataQualityConfiguration dataQualityConfiguration = + JsonUtils.fromJson(dataQualityParameter, DataQualityConfiguration.class); if (dataQualityConfiguration == null) { logger.info("DataQualityConfiguration is null"); System.exit(-1); @@ -60,13 +61,14 @@ public static void main(String[] args) throws Exception { EnvConfig envConfig = dataQualityConfiguration.getEnvConfig(); Config config = new Config(envConfig.getConfig()); - config.put("type",envConfig.getType()); + config.put("type", envConfig.getType()); if (Strings.isNullOrEmpty(config.getString(SPARK_APP_NAME))) { - config.put(SPARK_APP_NAME,dataQualityConfiguration.getName()); + config.put(SPARK_APP_NAME, dataQualityConfiguration.getName()); } SparkRuntimeEnvironment sparkRuntimeEnvironment = new SparkRuntimeEnvironment(config); - DataQualityContext dataQualityContext = new DataQualityContext(sparkRuntimeEnvironment,dataQualityConfiguration); + DataQualityContext dataQualityContext = + new DataQualityContext(sparkRuntimeEnvironment, dataQualityConfiguration); dataQualityContext.execute(); } } diff --git a/dolphinscheduler-data-quality/src/main/java/org/apache/dolphinscheduler/data/quality/config/BaseConfig.java b/dolphinscheduler-data-quality/src/main/java/org/apache/dolphinscheduler/data/quality/config/BaseConfig.java index 647016abf69e..60c5b810c3b4 100644 --- a/dolphinscheduler-data-quality/src/main/java/org/apache/dolphinscheduler/data/quality/config/BaseConfig.java +++ b/dolphinscheduler-data-quality/src/main/java/org/apache/dolphinscheduler/data/quality/config/BaseConfig.java @@ -32,12 +32,12 @@ public class BaseConfig implements IConfig { private String type; @JsonProperty("config") - private Map config; + private Map config; public BaseConfig() { } - public BaseConfig(String type, Map config) { + public BaseConfig(String type, Map config) { this.type = type; this.config = config; } diff --git a/dolphinscheduler-data-quality/src/main/java/org/apache/dolphinscheduler/data/quality/config/Config.java b/dolphinscheduler-data-quality/src/main/java/org/apache/dolphinscheduler/data/quality/config/Config.java index fbf7e70f842f..0bfe9e8df1ba 100644 --- a/dolphinscheduler-data-quality/src/main/java/org/apache/dolphinscheduler/data/quality/config/Config.java +++ b/dolphinscheduler-data-quality/src/main/java/org/apache/dolphinscheduler/data/quality/config/Config.java @@ -28,13 +28,13 @@ */ public class Config { - private Map configuration = new HashMap<>(); + private Map configuration = new HashMap<>(); public Config() { } - public Config(Map configuration) { + public Config(Map configuration) { if (configuration != null) { this.configuration = configuration; } @@ -45,7 +45,7 @@ public String getString(String key) { } public List getStringList(String key) { - return (List)configuration.get(key); + return (List) configuration.get(key); } public Integer getInt(String key) { @@ -81,7 +81,7 @@ public boolean isNotEmpty() { } public void put(String key, Object value) { - this.configuration.put(key,value); + this.configuration.put(key, value); } public void merge(Map configuration) { diff --git a/dolphinscheduler-data-quality/src/main/java/org/apache/dolphinscheduler/data/quality/config/DataQualityConfiguration.java b/dolphinscheduler-data-quality/src/main/java/org/apache/dolphinscheduler/data/quality/config/DataQualityConfiguration.java index 6ffd95566391..43a4a271c47a 100644 --- a/dolphinscheduler-data-quality/src/main/java/org/apache/dolphinscheduler/data/quality/config/DataQualityConfiguration.java +++ b/dolphinscheduler-data-quality/src/main/java/org/apache/dolphinscheduler/data/quality/config/DataQualityConfiguration.java @@ -43,7 +43,8 @@ public class DataQualityConfiguration implements IConfig { @JsonProperty("writers") private List writerConfigs; - public DataQualityConfiguration() {} + public DataQualityConfiguration() { + } public DataQualityConfiguration(String name, EnvConfig envConfig, @@ -114,7 +115,7 @@ public void validate() { } Preconditions.checkArgument(writerConfigs != null, "writer config should not be empty"); - for (WriterConfig writerConfig :writerConfigs) { + for (WriterConfig writerConfig : writerConfigs) { writerConfig.validate(); } } diff --git a/dolphinscheduler-data-quality/src/main/java/org/apache/dolphinscheduler/data/quality/config/EnvConfig.java b/dolphinscheduler-data-quality/src/main/java/org/apache/dolphinscheduler/data/quality/config/EnvConfig.java index 91d6854992fe..b37c857338b4 100644 --- a/dolphinscheduler-data-quality/src/main/java/org/apache/dolphinscheduler/data/quality/config/EnvConfig.java +++ b/dolphinscheduler-data-quality/src/main/java/org/apache/dolphinscheduler/data/quality/config/EnvConfig.java @@ -27,8 +27,8 @@ public class EnvConfig extends BaseConfig { public EnvConfig() { } - public EnvConfig(String type, Map config) { - super(type,config); + public EnvConfig(String type, Map config) { + super(type, config); } } diff --git a/dolphinscheduler-data-quality/src/main/java/org/apache/dolphinscheduler/data/quality/config/ReaderConfig.java b/dolphinscheduler-data-quality/src/main/java/org/apache/dolphinscheduler/data/quality/config/ReaderConfig.java index 72aa55686523..900be619ad15 100644 --- a/dolphinscheduler-data-quality/src/main/java/org/apache/dolphinscheduler/data/quality/config/ReaderConfig.java +++ b/dolphinscheduler-data-quality/src/main/java/org/apache/dolphinscheduler/data/quality/config/ReaderConfig.java @@ -24,9 +24,10 @@ */ public class ReaderConfig extends BaseConfig { - public ReaderConfig() {} + public ReaderConfig() { + } - public ReaderConfig(String type, Map config) { + public ReaderConfig(String type, Map config) { super(type, config); } } diff --git a/dolphinscheduler-data-quality/src/main/java/org/apache/dolphinscheduler/data/quality/config/TransformerConfig.java b/dolphinscheduler-data-quality/src/main/java/org/apache/dolphinscheduler/data/quality/config/TransformerConfig.java index 59d24c6b3b58..47a540760ba3 100644 --- a/dolphinscheduler-data-quality/src/main/java/org/apache/dolphinscheduler/data/quality/config/TransformerConfig.java +++ b/dolphinscheduler-data-quality/src/main/java/org/apache/dolphinscheduler/data/quality/config/TransformerConfig.java @@ -24,9 +24,10 @@ */ public class TransformerConfig extends BaseConfig { - public TransformerConfig() {} + public TransformerConfig() { + } - public TransformerConfig(String type, Map config) { + public TransformerConfig(String type, Map config) { super(type, config); } } diff --git a/dolphinscheduler-data-quality/src/main/java/org/apache/dolphinscheduler/data/quality/config/WriterConfig.java b/dolphinscheduler-data-quality/src/main/java/org/apache/dolphinscheduler/data/quality/config/WriterConfig.java index 75c5049311b4..9132c8930fe0 100644 --- a/dolphinscheduler-data-quality/src/main/java/org/apache/dolphinscheduler/data/quality/config/WriterConfig.java +++ b/dolphinscheduler-data-quality/src/main/java/org/apache/dolphinscheduler/data/quality/config/WriterConfig.java @@ -24,9 +24,10 @@ */ public class WriterConfig extends BaseConfig { - public WriterConfig() {} + public WriterConfig() { + } - public WriterConfig(String type, Map config) { + public WriterConfig(String type, Map config) { super(type, config); } } diff --git a/dolphinscheduler-data-quality/src/main/java/org/apache/dolphinscheduler/data/quality/context/DataQualityContext.java b/dolphinscheduler-data-quality/src/main/java/org/apache/dolphinscheduler/data/quality/context/DataQualityContext.java index 494bbad3641a..b702461533b5 100644 --- a/dolphinscheduler-data-quality/src/main/java/org/apache/dolphinscheduler/data/quality/context/DataQualityContext.java +++ b/dolphinscheduler-data-quality/src/main/java/org/apache/dolphinscheduler/data/quality/context/DataQualityContext.java @@ -50,16 +50,16 @@ public DataQualityContext(SparkRuntimeEnvironment sparkRuntimeEnvironment, public void execute() throws DataQualityException { List readers = ReaderFactory .getInstance() - .getReaders(this.sparkRuntimeEnvironment,dataQualityConfiguration.getReaderConfigs()); + .getReaders(this.sparkRuntimeEnvironment, dataQualityConfiguration.getReaderConfigs()); List transformers = TransformerFactory .getInstance() - .getTransformer(this.sparkRuntimeEnvironment,dataQualityConfiguration.getTransformerConfigs()); + .getTransformer(this.sparkRuntimeEnvironment, dataQualityConfiguration.getTransformerConfigs()); List writers = WriterFactory .getInstance() - .getWriters(this.sparkRuntimeEnvironment,dataQualityConfiguration.getWriterConfigs()); + .getWriters(this.sparkRuntimeEnvironment, dataQualityConfiguration.getWriterConfigs()); if (sparkRuntimeEnvironment.isBatch()) { - sparkRuntimeEnvironment.getBatchExecution().execute(readers,transformers,writers); + sparkRuntimeEnvironment.getBatchExecution().execute(readers, transformers, writers); } else { throw new DataQualityException("stream mode is not supported now"); } diff --git a/dolphinscheduler-data-quality/src/main/java/org/apache/dolphinscheduler/data/quality/enums/ReaderType.java b/dolphinscheduler-data-quality/src/main/java/org/apache/dolphinscheduler/data/quality/enums/ReaderType.java index b541367d96ef..e0cc02b3d870 100644 --- a/dolphinscheduler-data-quality/src/main/java/org/apache/dolphinscheduler/data/quality/enums/ReaderType.java +++ b/dolphinscheduler-data-quality/src/main/java/org/apache/dolphinscheduler/data/quality/enums/ReaderType.java @@ -21,6 +21,7 @@ * ReaderType */ public enum ReaderType { + /** * JDBC * HIVE @@ -29,7 +30,7 @@ public enum ReaderType { HIVE; public static ReaderType getType(String name) { - for (ReaderType type: ReaderType.values()) { + for (ReaderType type : ReaderType.values()) { if (type.name().equalsIgnoreCase(name)) { return type; } diff --git a/dolphinscheduler-data-quality/src/main/java/org/apache/dolphinscheduler/data/quality/enums/TransformerType.java b/dolphinscheduler-data-quality/src/main/java/org/apache/dolphinscheduler/data/quality/enums/TransformerType.java index b98adc4163a2..07744d9eec17 100644 --- a/dolphinscheduler-data-quality/src/main/java/org/apache/dolphinscheduler/data/quality/enums/TransformerType.java +++ b/dolphinscheduler-data-quality/src/main/java/org/apache/dolphinscheduler/data/quality/enums/TransformerType.java @@ -21,13 +21,14 @@ * TransformerType */ public enum TransformerType { + /** * JDBC */ SQL; public static TransformerType getType(String name) { - for (TransformerType type: TransformerType.values()) { + for (TransformerType type : TransformerType.values()) { if (type.name().equalsIgnoreCase(name)) { return type; } diff --git a/dolphinscheduler-data-quality/src/main/java/org/apache/dolphinscheduler/data/quality/enums/WriterType.java b/dolphinscheduler-data-quality/src/main/java/org/apache/dolphinscheduler/data/quality/enums/WriterType.java index 369de959be6e..f46430a4f68f 100644 --- a/dolphinscheduler-data-quality/src/main/java/org/apache/dolphinscheduler/data/quality/enums/WriterType.java +++ b/dolphinscheduler-data-quality/src/main/java/org/apache/dolphinscheduler/data/quality/enums/WriterType.java @@ -21,6 +21,7 @@ * WriterType */ public enum WriterType { + /** * JDBC */ @@ -29,7 +30,7 @@ public enum WriterType { HDFS_FILE; public static WriterType getType(String name) { - for (WriterType type: WriterType.values()) { + for (WriterType type : WriterType.values()) { if (type.name().equalsIgnoreCase(name)) { return type; } diff --git a/dolphinscheduler-data-quality/src/main/java/org/apache/dolphinscheduler/data/quality/execution/Execution.java b/dolphinscheduler-data-quality/src/main/java/org/apache/dolphinscheduler/data/quality/execution/Execution.java index 1644b2adde1b..527e10397ad8 100644 --- a/dolphinscheduler-data-quality/src/main/java/org/apache/dolphinscheduler/data/quality/execution/Execution.java +++ b/dolphinscheduler-data-quality/src/main/java/org/apache/dolphinscheduler/data/quality/execution/Execution.java @@ -25,6 +25,7 @@ * Execution */ public interface Execution { + /** * execute * @param readers readers diff --git a/dolphinscheduler-data-quality/src/main/java/org/apache/dolphinscheduler/data/quality/execution/SparkBatchExecution.java b/dolphinscheduler-data-quality/src/main/java/org/apache/dolphinscheduler/data/quality/execution/SparkBatchExecution.java index b805253dc3bf..cf55c1c23de4 100644 --- a/dolphinscheduler-data-quality/src/main/java/org/apache/dolphinscheduler/data/quality/execution/SparkBatchExecution.java +++ b/dolphinscheduler-data-quality/src/main/java/org/apache/dolphinscheduler/data/quality/execution/SparkBatchExecution.java @@ -49,12 +49,12 @@ public void execute(List readers, List transforme if (!readers.isEmpty()) { Dataset ds = readers.get(0).read(environment); - for (BatchTransformer tf:transformers) { + for (BatchTransformer tf : transformers) { ds = executeTransformer(environment, tf, ds); registerTransformTempView(tf, ds); } - for (BatchWriter sink: writers) { + for (BatchWriter sink : writers) { executeWriter(environment, sink, ds); } } @@ -77,18 +77,20 @@ private void registerInputTempView(BatchReader reader, SparkRuntimeEnvironment e registerTempView(tableName, reader.read(environment)); } else { throw new ConfigRuntimeException( - "[" + reader.getClass().getName() + "] must be registered as dataset, please set \"output_table\" config"); + "[" + reader.getClass().getName() + + "] must be registered as dataset, please set \"output_table\" config"); } } - private Dataset executeTransformer(SparkRuntimeEnvironment environment, BatchTransformer transformer, Dataset dataset) { + private Dataset executeTransformer(SparkRuntimeEnvironment environment, BatchTransformer transformer, + Dataset dataset) { Config config = transformer.getConfig(); Dataset inputDataset; Dataset outputDataset = null; if (Boolean.TRUE.equals(config.has(INPUT_TABLE))) { String[] tableNames = config.getString(INPUT_TABLE).split(","); - for (String sourceTableName: tableNames) { + for (String sourceTableName : tableNames) { inputDataset = environment.sparkSession().read().table(sourceTableName); if (outputDataset == null) { diff --git a/dolphinscheduler-data-quality/src/main/java/org/apache/dolphinscheduler/data/quality/execution/SparkRuntimeEnvironment.java b/dolphinscheduler-data-quality/src/main/java/org/apache/dolphinscheduler/data/quality/execution/SparkRuntimeEnvironment.java index b746392bdaaf..2e46a3c44b83 100644 --- a/dolphinscheduler-data-quality/src/main/java/org/apache/dolphinscheduler/data/quality/execution/SparkRuntimeEnvironment.java +++ b/dolphinscheduler-data-quality/src/main/java/org/apache/dolphinscheduler/data/quality/execution/SparkRuntimeEnvironment.java @@ -53,8 +53,8 @@ public void prepare() { private SparkConf createSparkConf() { SparkConf conf = new SparkConf(); this.config.entrySet() - .forEach(entry -> conf.set(entry.getKey(), String.valueOf(entry.getValue()))); - conf.set("spark.sql.crossJoin.enabled","true"); + .forEach(entry -> conf.set(entry.getKey(), String.valueOf(entry.getValue()))); + conf.set("spark.sql.crossJoin.enabled", "true"); return conf; } diff --git a/dolphinscheduler-data-quality/src/main/java/org/apache/dolphinscheduler/data/quality/flow/Component.java b/dolphinscheduler-data-quality/src/main/java/org/apache/dolphinscheduler/data/quality/flow/Component.java index ee5cae205aa8..c85c2cf50d14 100644 --- a/dolphinscheduler-data-quality/src/main/java/org/apache/dolphinscheduler/data/quality/flow/Component.java +++ b/dolphinscheduler-data-quality/src/main/java/org/apache/dolphinscheduler/data/quality/flow/Component.java @@ -45,8 +45,8 @@ default ValidateResult validate(List requiredOptions) { if (!nonExistsOptions.isEmpty()) { return new ValidateResult( false, - nonExistsOptions.stream().map(option -> - "[" + option + "]").collect(Collectors.joining(",")) + " is not exist"); + nonExistsOptions.stream().map(option -> "[" + option + "]").collect(Collectors.joining(",")) + + " is not exist"); } else { return new ValidateResult(true, ""); } diff --git a/dolphinscheduler-data-quality/src/main/java/org/apache/dolphinscheduler/data/quality/flow/batch/reader/HiveReader.java b/dolphinscheduler-data-quality/src/main/java/org/apache/dolphinscheduler/data/quality/flow/batch/reader/HiveReader.java index cd76edc36446..4022077a0161 100644 --- a/dolphinscheduler-data-quality/src/main/java/org/apache/dolphinscheduler/data/quality/flow/batch/reader/HiveReader.java +++ b/dolphinscheduler-data-quality/src/main/java/org/apache/dolphinscheduler/data/quality/flow/batch/reader/HiveReader.java @@ -57,7 +57,7 @@ public ValidateResult validateConfig() { @Override public void prepare(SparkRuntimeEnvironment prepareEnv) { if (Strings.isNullOrEmpty(config.getString(SQL))) { - config.put(SQL,"select * from " + config.getString(DATABASE) + "." + config.getString(TABLE)); + config.put(SQL, "select * from " + config.getString(DATABASE) + "." + config.getString(TABLE)); } } diff --git a/dolphinscheduler-data-quality/src/main/java/org/apache/dolphinscheduler/data/quality/flow/batch/reader/JdbcReader.java b/dolphinscheduler-data-quality/src/main/java/org/apache/dolphinscheduler/data/quality/flow/batch/reader/JdbcReader.java index 0484c15fa2de..96f980fcb79f 100644 --- a/dolphinscheduler-data-quality/src/main/java/org/apache/dolphinscheduler/data/quality/flow/batch/reader/JdbcReader.java +++ b/dolphinscheduler-data-quality/src/main/java/org/apache/dolphinscheduler/data/quality/flow/batch/reader/JdbcReader.java @@ -85,8 +85,8 @@ private DataFrameReader jdbcReader(SparkSession sparkSession) { Config jdbcConfig = ConfigUtils.extractSubConfig(config, JDBC + DOTS, false); if (!config.isEmpty()) { - Map optionMap = new HashMap<>(16); - jdbcConfig.entrySet().forEach(x -> optionMap.put(x.getKey(),String.valueOf(x.getValue()))); + Map optionMap = new HashMap<>(16); + jdbcConfig.entrySet().forEach(x -> optionMap.put(x.getKey(), String.valueOf(x.getValue()))); reader.options(optionMap); } diff --git a/dolphinscheduler-data-quality/src/main/java/org/apache/dolphinscheduler/data/quality/flow/batch/reader/ReaderFactory.java b/dolphinscheduler-data-quality/src/main/java/org/apache/dolphinscheduler/data/quality/flow/batch/reader/ReaderFactory.java index 9d9eb3d8e72a..cafe3decdb8c 100644 --- a/dolphinscheduler-data-quality/src/main/java/org/apache/dolphinscheduler/data/quality/flow/batch/reader/ReaderFactory.java +++ b/dolphinscheduler-data-quality/src/main/java/org/apache/dolphinscheduler/data/quality/flow/batch/reader/ReaderFactory.java @@ -33,6 +33,7 @@ public class ReaderFactory { private static class Singleton { + static ReaderFactory instance = new ReaderFactory(); } @@ -40,7 +41,8 @@ public static ReaderFactory getInstance() { return Singleton.instance; } - public List getReaders(SparkRuntimeEnvironment sparkRuntimeEnvironment, List readerConfigs) throws DataQualityException { + public List getReaders(SparkRuntimeEnvironment sparkRuntimeEnvironment, + List readerConfigs) throws DataQualityException { List readerList = new ArrayList<>(); diff --git a/dolphinscheduler-data-quality/src/main/java/org/apache/dolphinscheduler/data/quality/flow/batch/transformer/TransformerFactory.java b/dolphinscheduler-data-quality/src/main/java/org/apache/dolphinscheduler/data/quality/flow/batch/transformer/TransformerFactory.java index cb5e45eaed5f..bc5a19eac9cc 100644 --- a/dolphinscheduler-data-quality/src/main/java/org/apache/dolphinscheduler/data/quality/flow/batch/transformer/TransformerFactory.java +++ b/dolphinscheduler-data-quality/src/main/java/org/apache/dolphinscheduler/data/quality/flow/batch/transformer/TransformerFactory.java @@ -33,6 +33,7 @@ public class TransformerFactory { private static class Singleton { + static TransformerFactory instance = new TransformerFactory(); } @@ -40,11 +41,12 @@ public static TransformerFactory getInstance() { return Singleton.instance; } - public List getTransformer(SparkRuntimeEnvironment sparkRuntimeEnvironment, List transformerConfigs) throws DataQualityException { + public List getTransformer(SparkRuntimeEnvironment sparkRuntimeEnvironment, + List transformerConfigs) throws DataQualityException { List transformers = new ArrayList<>(); - for (TransformerConfig transformerConfig:transformerConfigs) { + for (TransformerConfig transformerConfig : transformerConfigs) { BatchTransformer transformer = getTransformer(transformerConfig); if (transformer != null) { transformer.validateConfig(); diff --git a/dolphinscheduler-data-quality/src/main/java/org/apache/dolphinscheduler/data/quality/flow/batch/writer/JdbcWriter.java b/dolphinscheduler-data-quality/src/main/java/org/apache/dolphinscheduler/data/quality/flow/batch/writer/JdbcWriter.java index f1c098965c34..0010c4c6e601 100644 --- a/dolphinscheduler-data-quality/src/main/java/org/apache/dolphinscheduler/data/quality/flow/batch/writer/JdbcWriter.java +++ b/dolphinscheduler-data-quality/src/main/java/org/apache/dolphinscheduler/data/quality/flow/batch/writer/JdbcWriter.java @@ -64,7 +64,7 @@ public ValidateResult validateConfig() { @Override public void prepare(SparkRuntimeEnvironment prepareEnv) { if (Strings.isNullOrEmpty(config.getString(SAVE_MODE))) { - config.put(SAVE_MODE,APPEND); + config.put(SAVE_MODE, APPEND); } } @@ -75,13 +75,13 @@ public void write(Dataset data, SparkRuntimeEnvironment env) { } data.write() - .format(JDBC) - .option(DRIVER,config.getString(DRIVER)) - .option(URL,config.getString(URL)) - .option(DB_TABLE, config.getString(TABLE)) - .option(USER, config.getString(USER)) - .option(PASSWORD, config.getString(PASSWORD)) - .mode(config.getString(SAVE_MODE)) - .save(); + .format(JDBC) + .option(DRIVER, config.getString(DRIVER)) + .option(URL, config.getString(URL)) + .option(DB_TABLE, config.getString(TABLE)) + .option(USER, config.getString(USER)) + .option(PASSWORD, config.getString(PASSWORD)) + .mode(config.getString(SAVE_MODE)) + .save(); } } diff --git a/dolphinscheduler-data-quality/src/main/java/org/apache/dolphinscheduler/data/quality/flow/batch/writer/WriterFactory.java b/dolphinscheduler-data-quality/src/main/java/org/apache/dolphinscheduler/data/quality/flow/batch/writer/WriterFactory.java index f907748a4dfc..e9fc2af6c851 100644 --- a/dolphinscheduler-data-quality/src/main/java/org/apache/dolphinscheduler/data/quality/flow/batch/writer/WriterFactory.java +++ b/dolphinscheduler-data-quality/src/main/java/org/apache/dolphinscheduler/data/quality/flow/batch/writer/WriterFactory.java @@ -35,6 +35,7 @@ public class WriterFactory { private static class Singleton { + static WriterFactory instance = new WriterFactory(); } @@ -42,11 +43,12 @@ public static WriterFactory getInstance() { return Singleton.instance; } - public List getWriters(SparkRuntimeEnvironment sparkRuntimeEnvironment, List writerConfigs) throws DataQualityException { + public List getWriters(SparkRuntimeEnvironment sparkRuntimeEnvironment, + List writerConfigs) throws DataQualityException { List writerList = new ArrayList<>(); - for (WriterConfig writerConfig:writerConfigs) { + for (WriterConfig writerConfig : writerConfigs) { BatchWriter writer = getWriter(writerConfig); if (writer != null) { writer.validateConfig(); diff --git a/dolphinscheduler-data-quality/src/main/java/org/apache/dolphinscheduler/data/quality/flow/batch/writer/file/BaseFileWriter.java b/dolphinscheduler-data-quality/src/main/java/org/apache/dolphinscheduler/data/quality/flow/batch/writer/file/BaseFileWriter.java index fbe311fe2a6b..56ad58044ec6 100644 --- a/dolphinscheduler-data-quality/src/main/java/org/apache/dolphinscheduler/data/quality/flow/batch/writer/file/BaseFileWriter.java +++ b/dolphinscheduler-data-quality/src/main/java/org/apache/dolphinscheduler/data/quality/flow/batch/writer/file/BaseFileWriter.java @@ -59,11 +59,11 @@ public Config getConfig() { @Override public void prepare(SparkRuntimeEnvironment prepareEnv) { - Map defaultConfig = new HashMap<>(); + Map defaultConfig = new HashMap<>(); defaultConfig.put(PARTITION_BY, Collections.emptyList()); - defaultConfig.put(SAVE_MODE,"error"); - defaultConfig.put(SERIALIZER,"csv"); + defaultConfig.put(SAVE_MODE, "error"); + defaultConfig.put(SERIALIZER, "csv"); config.merge(defaultConfig); } @@ -75,7 +75,8 @@ protected ValidateResult checkConfigImpl(List allowedUri) { if (dir.startsWith("/") || uriInAllowedSchema(dir, allowedUri)) { return new ValidateResult(true, ""); } else { - return new ValidateResult(false, "invalid path URI, please set the following allowed schemas: " + String.join(",", allowedUri)); + return new ValidateResult(false, + "invalid path URI, please set the following allowed schemas: " + String.join(",", allowedUri)); } } else { return new ValidateResult(false, "please specify [path] as non-empty string"); @@ -101,8 +102,8 @@ protected void outputImpl(Dataset df, String defaultUriSchema) { Config fileConfig = ConfigUtils.extractSubConfig(config, "options.", false); if (fileConfig.isNotEmpty()) { - Map optionMap = new HashMap<>(16); - fileConfig.entrySet().forEach(x -> optionMap.put(x.getKey(),String.valueOf(x.getValue()))); + Map optionMap = new HashMap<>(16); + fileConfig.entrySet().forEach(x -> optionMap.put(x.getKey(), String.valueOf(x.getValue()))); writer.options(optionMap); } diff --git a/dolphinscheduler-data-quality/src/main/java/org/apache/dolphinscheduler/data/quality/flow/batch/writer/file/HdfsFileWriter.java b/dolphinscheduler-data-quality/src/main/java/org/apache/dolphinscheduler/data/quality/flow/batch/writer/file/HdfsFileWriter.java index 7aba1de244e3..7fd39618689c 100644 --- a/dolphinscheduler-data-quality/src/main/java/org/apache/dolphinscheduler/data/quality/flow/batch/writer/file/HdfsFileWriter.java +++ b/dolphinscheduler-data-quality/src/main/java/org/apache/dolphinscheduler/data/quality/flow/batch/writer/file/HdfsFileWriter.java @@ -37,7 +37,7 @@ public HdfsFileWriter(Config config) { @Override public void write(Dataset data, SparkRuntimeEnvironment environment) { - outputImpl(data,"hdfs://"); + outputImpl(data, "hdfs://"); } @Override diff --git a/dolphinscheduler-data-quality/src/main/java/org/apache/dolphinscheduler/data/quality/flow/batch/writer/file/LocalFileWriter.java b/dolphinscheduler-data-quality/src/main/java/org/apache/dolphinscheduler/data/quality/flow/batch/writer/file/LocalFileWriter.java index a35db3cfbefd..1741ff20fb07 100644 --- a/dolphinscheduler-data-quality/src/main/java/org/apache/dolphinscheduler/data/quality/flow/batch/writer/file/LocalFileWriter.java +++ b/dolphinscheduler-data-quality/src/main/java/org/apache/dolphinscheduler/data/quality/flow/batch/writer/file/LocalFileWriter.java @@ -37,7 +37,7 @@ public LocalFileWriter(Config config) { @Override public void write(Dataset data, SparkRuntimeEnvironment environment) { - outputImpl(data,"file://"); + outputImpl(data, "file://"); } @Override diff --git a/dolphinscheduler-data-quality/src/main/java/org/apache/dolphinscheduler/data/quality/utils/JsonUtils.java b/dolphinscheduler-data-quality/src/main/java/org/apache/dolphinscheduler/data/quality/utils/JsonUtils.java index e35eeaa886dc..f94d1035085c 100644 --- a/dolphinscheduler-data-quality/src/main/java/org/apache/dolphinscheduler/data/quality/utils/JsonUtils.java +++ b/dolphinscheduler-data-quality/src/main/java/org/apache/dolphinscheduler/data/quality/utils/JsonUtils.java @@ -48,10 +48,10 @@ public class JsonUtils { private static final ObjectMapper MAPPER = new ObjectMapper() .configure(FAIL_ON_UNKNOWN_PROPERTIES, false) .configure(ACCEPT_EMPTY_ARRAY_AS_NULL_OBJECT, true) - .configure(ACCEPT_EMPTY_STRING_AS_NULL_OBJECT,true) + .configure(ACCEPT_EMPTY_STRING_AS_NULL_OBJECT, true) .configure(READ_UNKNOWN_ENUM_VALUES_AS_NULL, true) .configure(REQUIRE_SETTERS_FOR_GETTERS, true) - .configure(FAIL_ON_EMPTY_BEANS,false) + .configure(FAIL_ON_EMPTY_BEANS, false) .setTimeZone(TimeZone.getDefault()) .setDateFormat(new SimpleDateFormat(Constants.YYYY_MM_DD_HH_MM_SS)); diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-all/pom.xml b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-all/pom.xml index cd564e2a2d23..f5f966bba421 100644 --- a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-all/pom.xml +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-all/pom.xml @@ -15,15 +15,14 @@ ~ See the License for the specific language governing permissions and ~ limitations under the License. --> - + 4.0.0 - dolphinscheduler-datasource-plugin org.apache.dolphinscheduler + dolphinscheduler-datasource-plugin dev-SNAPSHOT - 4.0.0 dolphinscheduler-datasource-all diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/client/CommonDataSourceClient.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/client/CommonDataSourceClient.java index 50de82f93dfa..ac3f96d46c3f 100644 --- a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/client/CommonDataSourceClient.java +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/client/CommonDataSourceClient.java @@ -89,14 +89,15 @@ protected void setDefaultValidationQuery(BaseConnectionParam baseConnectionParam @Override public void checkClient() { - //Checking data source client + // Checking data source client Stopwatch stopwatch = Stopwatch.createStarted(); try { this.jdbcTemplate.execute(this.baseConnectionParam.getValidationQuery()); } catch (Exception e) { throw new RuntimeException("JDBC connect failed", e); } finally { - logger.info("Time to execute check jdbc client with sql {} for {} ms ", this.baseConnectionParam.getValidationQuery(), stopwatch.elapsed(TimeUnit.MILLISECONDS)); + logger.info("Time to execute check jdbc client with sql {} for {} ms ", + this.baseConnectionParam.getValidationQuery(), stopwatch.elapsed(TimeUnit.MILLISECONDS)); } } diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/datasource/BaseHDFSConnectionParam.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/datasource/BaseHDFSConnectionParam.java index 728a91956da1..fd14837dad24 100644 --- a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/datasource/BaseHDFSConnectionParam.java +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/datasource/BaseHDFSConnectionParam.java @@ -20,6 +20,7 @@ import org.apache.dolphinscheduler.spi.datasource.BaseConnectionParam; public class BaseHDFSConnectionParam extends BaseConnectionParam { + protected String principal; protected String javaSecurityKrb5Conf; protected String loginUserKeytabUsername; diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/datasource/DataSourceProcessor.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/datasource/DataSourceProcessor.java index 3cfac2566398..e30a6381256f 100644 --- a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/datasource/DataSourceProcessor.java +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/datasource/DataSourceProcessor.java @@ -34,7 +34,6 @@ public interface DataSourceProcessor { */ BaseDataSourceParamDTO castDatasourceParamDTO(String paramJson); - /** * check datasource param is valid */ diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/plugin/DataSourcePluginManager.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/plugin/DataSourcePluginManager.java index 52b76d2c41b7..6425064b3d31 100644 --- a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/plugin/DataSourcePluginManager.java +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/plugin/DataSourcePluginManager.java @@ -17,19 +17,21 @@ package org.apache.dolphinscheduler.plugin.datasource.api.plugin; +import static java.lang.String.format; + import org.apache.dolphinscheduler.spi.datasource.DataSourceChannel; import org.apache.dolphinscheduler.spi.datasource.DataSourceChannelFactory; import org.apache.dolphinscheduler.spi.plugin.PrioritySPIFactory; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import java.util.Collections; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; -import static java.lang.String.format; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; public class DataSourcePluginManager { + private static final Logger logger = LoggerFactory.getLogger(DataSourcePluginManager.class); private final Map datasourceClientMap = new ConcurrentHashMap<>(); @@ -40,7 +42,8 @@ public Map getDataSourceChannelMap() { public void installPlugin() { - PrioritySPIFactory prioritySPIFactory = new PrioritySPIFactory<>(DataSourceChannelFactory.class); + PrioritySPIFactory prioritySPIFactory = + new PrioritySPIFactory<>(DataSourceChannelFactory.class); for (Map.Entry entry : prioritySPIFactory.getSPIMap().entrySet()) { final DataSourceChannelFactory factory = entry.getValue(); final String name = entry.getKey(); diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/plugin/DataSourceProcessorManager.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/plugin/DataSourceProcessorManager.java index b694f4534fcf..dbdbe3bf687c 100644 --- a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/plugin/DataSourceProcessorManager.java +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/plugin/DataSourceProcessorManager.java @@ -30,6 +30,7 @@ import org.slf4j.LoggerFactory; public class DataSourceProcessorManager { + private static final Logger logger = LoggerFactory.getLogger(DataSourceProcessorManager.class); private static final Map dataSourceProcessorMap = new ConcurrentHashMap<>(); diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/plugin/DataSourceProcessorProvider.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/plugin/DataSourceProcessorProvider.java index 7cdd5c98d1f3..0b4994116a97 100644 --- a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/plugin/DataSourceProcessorProvider.java +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/plugin/DataSourceProcessorProvider.java @@ -18,13 +18,14 @@ package org.apache.dolphinscheduler.plugin.datasource.api.plugin; import org.apache.dolphinscheduler.plugin.datasource.api.datasource.DataSourceProcessor; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import java.util.Map; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; public class DataSourceProcessorProvider { + private static final Logger logger = LoggerFactory.getLogger(DataSourceProcessorProvider.class); private DataSourceProcessorManager dataSourcePluginManager; @@ -34,6 +35,7 @@ private DataSourceProcessorProvider() { } private static class DataSourceClientProviderHolder { + private static final DataSourceProcessorProvider INSTANCE = new DataSourceProcessorProvider(); } diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/utils/CommonUtils.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/utils/CommonUtils.java index 73af3517dbd6..977727250858 100644 --- a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/utils/CommonUtils.java +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/utils/CommonUtils.java @@ -17,7 +17,15 @@ package org.apache.dolphinscheduler.plugin.datasource.api.utils; -import static org.apache.dolphinscheduler.plugin.task.api.TaskConstants.*; +import static org.apache.dolphinscheduler.plugin.task.api.TaskConstants.DATA_QUALITY_JAR_NAME; +import static org.apache.dolphinscheduler.plugin.task.api.TaskConstants.HADOOP_SECURITY_AUTHENTICATION; +import static org.apache.dolphinscheduler.plugin.task.api.TaskConstants.HADOOP_SECURITY_AUTHENTICATION_STARTUP_STATE; +import static org.apache.dolphinscheduler.plugin.task.api.TaskConstants.JAVA_SECURITY_KRB5_CONF; +import static org.apache.dolphinscheduler.plugin.task.api.TaskConstants.JAVA_SECURITY_KRB5_CONF_PATH; +import static org.apache.dolphinscheduler.plugin.task.api.TaskConstants.KERBEROS; +import static org.apache.dolphinscheduler.plugin.task.api.TaskConstants.LOGIN_USER_KEY_TAB_PATH; +import static org.apache.dolphinscheduler.plugin.task.api.TaskConstants.LOGIN_USER_KEY_TAB_USERNAME; +import static org.apache.dolphinscheduler.plugin.task.api.TaskConstants.RESOURCE_UPLOAD_PATH; import static org.apache.dolphinscheduler.spi.utils.Constants.RESOURCE_STORAGE_TYPE; import org.apache.dolphinscheduler.common.enums.ResUploadType; diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/utils/DataSourceUtils.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/utils/DataSourceUtils.java index 80c2d68cc896..7d95f60658a7 100644 --- a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/utils/DataSourceUtils.java +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/utils/DataSourceUtils.java @@ -17,19 +17,21 @@ package org.apache.dolphinscheduler.plugin.datasource.api.utils; -import com.fasterxml.jackson.databind.JsonNode; import org.apache.dolphinscheduler.plugin.datasource.api.datasource.BaseDataSourceParamDTO; import org.apache.dolphinscheduler.plugin.datasource.api.datasource.DataSourceProcessor; import org.apache.dolphinscheduler.plugin.datasource.api.plugin.DataSourceProcessorProvider; import org.apache.dolphinscheduler.spi.datasource.ConnectionParam; import org.apache.dolphinscheduler.spi.enums.DbType; import org.apache.dolphinscheduler.spi.utils.JSONUtils; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import java.sql.Connection; import java.util.Map; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.fasterxml.jackson.databind.JsonNode; + public class DataSourceUtils { public DataSourceUtils() { @@ -83,7 +85,8 @@ public static BaseDataSourceParamDTO buildDatasourceParamDTO(DbType dbType, Stri } public static DataSourceProcessor getDatasourceProcessor(DbType dbType) { - Map dataSourceProcessorMap = DataSourceProcessorProvider.getInstance().getDataSourceProcessorMap(); + Map dataSourceProcessorMap = + DataSourceProcessorProvider.getInstance().getDataSourceProcessorMap(); if (!dataSourceProcessorMap.containsKey(dbType.name())) { throw new IllegalArgumentException("illegal datasource type"); } diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/test/java/org/apache/dolphinscheduler/plugin/datasource/api/datasource/MySQLConnectionParam.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/test/java/org/apache/dolphinscheduler/plugin/datasource/api/datasource/MySQLConnectionParam.java index 05c9840795fa..edfebe4fe67d 100644 --- a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/test/java/org/apache/dolphinscheduler/plugin/datasource/api/datasource/MySQLConnectionParam.java +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/test/java/org/apache/dolphinscheduler/plugin/datasource/api/datasource/MySQLConnectionParam.java @@ -23,6 +23,7 @@ * Used for test. */ public class MySQLConnectionParam extends BaseConnectionParam { + @Override public String toString() { return "MySQLConnectionParam{" diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-athena/pom.xml b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-athena/pom.xml index 0ca15dc16de9..932cc2af293b 100644 --- a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-athena/pom.xml +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-athena/pom.xml @@ -15,16 +15,14 @@ ~ See the License for the specific language governing permissions and ~ limitations under the License. --> - - + 4.0.0 - dolphinscheduler-datasource-plugin org.apache.dolphinscheduler + dolphinscheduler-datasource-plugin dev-SNAPSHOT - 4.0.0 dolphinscheduler-datasource-athena jar @@ -41,4 +39,4 @@ - \ No newline at end of file + diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-athena/src/main/java/org/apache/dolphinscheduler/plugin/datasource/athena/AthenaDataSourceChannel.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-athena/src/main/java/org/apache/dolphinscheduler/plugin/datasource/athena/AthenaDataSourceChannel.java index 216c0112ac81..a83c6b9fb787 100644 --- a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-athena/src/main/java/org/apache/dolphinscheduler/plugin/datasource/athena/AthenaDataSourceChannel.java +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-athena/src/main/java/org/apache/dolphinscheduler/plugin/datasource/athena/AthenaDataSourceChannel.java @@ -23,8 +23,9 @@ import org.apache.dolphinscheduler.spi.enums.DbType; public class AthenaDataSourceChannel implements DataSourceChannel { + @Override public DataSourceClient createDataSourceClient(BaseConnectionParam baseConnectionParam, DbType dbType) { - return new AthenaDataSourceClient(baseConnectionParam,dbType); + return new AthenaDataSourceClient(baseConnectionParam, dbType); } } diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-athena/src/main/java/org/apache/dolphinscheduler/plugin/datasource/athena/AthenaDataSourceChannelFactory.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-athena/src/main/java/org/apache/dolphinscheduler/plugin/datasource/athena/AthenaDataSourceChannelFactory.java index d1439570c367..1b2ed367d0a5 100644 --- a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-athena/src/main/java/org/apache/dolphinscheduler/plugin/datasource/athena/AthenaDataSourceChannelFactory.java +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-athena/src/main/java/org/apache/dolphinscheduler/plugin/datasource/athena/AthenaDataSourceChannelFactory.java @@ -24,6 +24,7 @@ @AutoService(DataSourceChannelFactory.class) public class AthenaDataSourceChannelFactory implements DataSourceChannelFactory { + @Override public DataSourceChannel create() { return new AthenaDataSourceChannel(); diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-athena/src/main/java/org/apache/dolphinscheduler/plugin/datasource/athena/AthenaDataSourceClient.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-athena/src/main/java/org/apache/dolphinscheduler/plugin/datasource/athena/AthenaDataSourceClient.java index 58bde0cce832..20755d2f3021 100644 --- a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-athena/src/main/java/org/apache/dolphinscheduler/plugin/datasource/athena/AthenaDataSourceClient.java +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-athena/src/main/java/org/apache/dolphinscheduler/plugin/datasource/athena/AthenaDataSourceClient.java @@ -22,6 +22,7 @@ import org.apache.dolphinscheduler.spi.enums.DbType; public class AthenaDataSourceClient extends CommonDataSourceClient { + public AthenaDataSourceClient(BaseConnectionParam baseConnectionParam, DbType dbType) { super(baseConnectionParam, dbType); } diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-athena/src/main/java/org/apache/dolphinscheduler/plugin/datasource/athena/param/AthenaDataSourceParamDTO.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-athena/src/main/java/org/apache/dolphinscheduler/plugin/datasource/athena/param/AthenaDataSourceParamDTO.java index ea89de7a3c61..b52b5cc362af 100644 --- a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-athena/src/main/java/org/apache/dolphinscheduler/plugin/datasource/athena/param/AthenaDataSourceParamDTO.java +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-athena/src/main/java/org/apache/dolphinscheduler/plugin/datasource/athena/param/AthenaDataSourceParamDTO.java @@ -26,6 +26,7 @@ @Data @EqualsAndHashCode(callSuper = true) public class AthenaDataSourceParamDTO extends BaseDataSourceParamDTO { + protected String awsRegion; @Override diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-athena/src/main/java/org/apache/dolphinscheduler/plugin/datasource/athena/param/AthenaDataSourceProcessor.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-athena/src/main/java/org/apache/dolphinscheduler/plugin/datasource/athena/param/AthenaDataSourceProcessor.java index 4187b37a5001..402e34808f60 100644 --- a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-athena/src/main/java/org/apache/dolphinscheduler/plugin/datasource/athena/param/AthenaDataSourceProcessor.java +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-athena/src/main/java/org/apache/dolphinscheduler/plugin/datasource/athena/param/AthenaDataSourceProcessor.java @@ -17,8 +17,6 @@ package org.apache.dolphinscheduler.plugin.datasource.athena.param; -import com.google.auto.service.AutoService; -import org.apache.commons.collections4.MapUtils; import org.apache.dolphinscheduler.plugin.datasource.api.datasource.AbstractDataSourceProcessor; import org.apache.dolphinscheduler.plugin.datasource.api.datasource.BaseDataSourceParamDTO; import org.apache.dolphinscheduler.plugin.datasource.api.datasource.DataSourceProcessor; @@ -30,6 +28,8 @@ import org.apache.dolphinscheduler.spi.utils.JSONUtils; import org.apache.dolphinscheduler.spi.utils.StringUtils; +import org.apache.commons.collections4.MapUtils; + import java.sql.Connection; import java.sql.DriverManager; import java.sql.SQLException; @@ -38,6 +38,8 @@ import java.util.List; import java.util.Map; +import com.google.auto.service.AutoService; + @AutoService(DataSourceProcessor.class) public class AthenaDataSourceProcessor extends AbstractDataSourceProcessor { @@ -48,11 +50,9 @@ public BaseDataSourceParamDTO castDatasourceParamDTO(String paramJson) { @Override public BaseDataSourceParamDTO createDatasourceParamDTO(String connectionJson) { - AthenaConnectionParam - connectionParams = (AthenaConnectionParam) this.createConnectionParams(connectionJson); + AthenaConnectionParam connectionParams = (AthenaConnectionParam) this.createConnectionParams(connectionJson); - AthenaDataSourceParamDTO - athenaDatasourceParamDTO = new AthenaDataSourceParamDTO(); + AthenaDataSourceParamDTO athenaDatasourceParamDTO = new AthenaDataSourceParamDTO(); athenaDatasourceParamDTO.setAwsRegion(connectionParams.getAwsRegion()); athenaDatasourceParamDTO.setDatabase(connectionParams.getDatabase()); athenaDatasourceParamDTO.setUserName(connectionParams.getUser()); @@ -66,8 +66,7 @@ public BaseConnectionParam createConnectionParams(BaseDataSourceParamDTO datasou AthenaDataSourceParamDTO athenaParam = (AthenaDataSourceParamDTO) datasourceParam; String address = String.format("%s%s=%s;", Constants.JDBC_ATHENA, "AwsRegion", athenaParam.getAwsRegion()); - AthenaConnectionParam - athenaConnectionParam = new AthenaConnectionParam(); + AthenaConnectionParam athenaConnectionParam = new AthenaConnectionParam(); athenaConnectionParam.setUser(athenaParam.getUserName()); athenaConnectionParam.setPassword(PasswordUtils.encodePassword(athenaParam.getPassword())); athenaConnectionParam.setAwsRegion(athenaParam.getAwsRegion()); @@ -99,8 +98,7 @@ public String getValidationQuery() { @Override public String getJdbcUrl(ConnectionParam connectionParam) { - AthenaConnectionParam - athenaConnectionParam = (AthenaConnectionParam) connectionParam; + AthenaConnectionParam athenaConnectionParam = (AthenaConnectionParam) connectionParam; if (!StringUtils.isEmpty(athenaConnectionParam.getOther())) { return String.format("%s%s", athenaConnectionParam.getJdbcUrl(), athenaConnectionParam.getOther()); } @@ -112,7 +110,7 @@ public Connection getConnection(ConnectionParam connectionParam) throws ClassNot AthenaConnectionParam athenaConnectionParam = (AthenaConnectionParam) connectionParam; Class.forName(this.getDatasourceDriver()); return DriverManager.getConnection(this.getJdbcUrl(connectionParam), - athenaConnectionParam.getUser(), PasswordUtils.decodePassword(athenaConnectionParam.getPassword())); + athenaConnectionParam.getUser(), PasswordUtils.decodePassword(athenaConnectionParam.getPassword())); } @Override diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-clickhouse/src/main/java/org/apache/dolphinscheduler/plugin/datasource/clickhouse/ClickHouseDataSourceChannelFactory.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-clickhouse/src/main/java/org/apache/dolphinscheduler/plugin/datasource/clickhouse/ClickHouseDataSourceChannelFactory.java index 03534e7312f9..d756226522fb 100644 --- a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-clickhouse/src/main/java/org/apache/dolphinscheduler/plugin/datasource/clickhouse/ClickHouseDataSourceChannelFactory.java +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-clickhouse/src/main/java/org/apache/dolphinscheduler/plugin/datasource/clickhouse/ClickHouseDataSourceChannelFactory.java @@ -24,6 +24,7 @@ @AutoService(DataSourceChannelFactory.class) public class ClickHouseDataSourceChannelFactory implements DataSourceChannelFactory { + @Override public String getName() { return "clickhouse"; diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-clickhouse/src/main/java/org/apache/dolphinscheduler/plugin/datasource/clickhouse/param/ClickHouseConnectionParam.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-clickhouse/src/main/java/org/apache/dolphinscheduler/plugin/datasource/clickhouse/param/ClickHouseConnectionParam.java index 4cf56ffffb2c..df31fae5758c 100644 --- a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-clickhouse/src/main/java/org/apache/dolphinscheduler/plugin/datasource/clickhouse/param/ClickHouseConnectionParam.java +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-clickhouse/src/main/java/org/apache/dolphinscheduler/plugin/datasource/clickhouse/param/ClickHouseConnectionParam.java @@ -20,6 +20,7 @@ import org.apache.dolphinscheduler.spi.datasource.BaseConnectionParam; public class ClickHouseConnectionParam extends BaseConnectionParam { + @Override public String toString() { return "ClickHouseConnectionParam{" diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-clickhouse/src/main/java/org/apache/dolphinscheduler/plugin/datasource/clickhouse/param/ClickHouseDataSourceProcessor.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-clickhouse/src/main/java/org/apache/dolphinscheduler/plugin/datasource/clickhouse/param/ClickHouseDataSourceProcessor.java index 7bbc5ba4c51b..5310c7bae6fd 100644 --- a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-clickhouse/src/main/java/org/apache/dolphinscheduler/plugin/datasource/clickhouse/param/ClickHouseDataSourceProcessor.java +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-clickhouse/src/main/java/org/apache/dolphinscheduler/plugin/datasource/clickhouse/param/ClickHouseDataSourceProcessor.java @@ -49,8 +49,7 @@ public BaseDataSourceParamDTO castDatasourceParamDTO(String paramJson) { public BaseDataSourceParamDTO createDatasourceParamDTO(String connectionJson) { ClickHouseConnectionParam connectionParams = (ClickHouseConnectionParam) createConnectionParams(connectionJson); - ClickHouseDataSourceParamDTO - clickHouseDatasourceParamDTO = new ClickHouseDataSourceParamDTO(); + ClickHouseDataSourceParamDTO clickHouseDatasourceParamDTO = new ClickHouseDataSourceParamDTO(); clickHouseDatasourceParamDTO.setDatabase(connectionParams.getDatabase()); clickHouseDatasourceParamDTO.setUserName(connectionParams.getUser()); clickHouseDatasourceParamDTO.setOther(parseOther(connectionParams.getOther())); @@ -66,7 +65,8 @@ public BaseDataSourceParamDTO createDatasourceParamDTO(String connectionJson) { @Override public ConnectionParam createConnectionParams(BaseDataSourceParamDTO datasourceParam) { ClickHouseDataSourceParamDTO clickHouseParam = (ClickHouseDataSourceParamDTO) datasourceParam; - String address = String.format("%s%s:%s", Constants.JDBC_CLICKHOUSE, clickHouseParam.getHost(), clickHouseParam.getPort()); + String address = String.format("%s%s:%s", Constants.JDBC_CLICKHOUSE, clickHouseParam.getHost(), + clickHouseParam.getPort()); String jdbcUrl = address + "/" + clickHouseParam.getDatabase(); ClickHouseConnectionParam clickhouseConnectionParam = new ClickHouseConnectionParam(); @@ -112,7 +112,8 @@ public Connection getConnection(ConnectionParam connectionParam) throws ClassNot ClickHouseConnectionParam clickhouseConnectionParam = (ClickHouseConnectionParam) connectionParam; Class.forName(getDatasourceDriver()); return DriverManager.getConnection(getJdbcUrl(clickhouseConnectionParam), - clickhouseConnectionParam.getUser(), PasswordUtils.decodePassword(clickhouseConnectionParam.getPassword())); + clickhouseConnectionParam.getUser(), + PasswordUtils.decodePassword(clickhouseConnectionParam.getPassword())); } @Override diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-db2/src/main/java/org/apache/dolphinscheduler/plugin/datasource/db2/param/Db2ConnectionParam.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-db2/src/main/java/org/apache/dolphinscheduler/plugin/datasource/db2/param/Db2ConnectionParam.java index 88be896c25e0..a0a166769041 100644 --- a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-db2/src/main/java/org/apache/dolphinscheduler/plugin/datasource/db2/param/Db2ConnectionParam.java +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-db2/src/main/java/org/apache/dolphinscheduler/plugin/datasource/db2/param/Db2ConnectionParam.java @@ -20,6 +20,7 @@ import org.apache.dolphinscheduler.spi.datasource.BaseConnectionParam; public class Db2ConnectionParam extends BaseConnectionParam { + @Override public String toString() { return "Db2ConnectionParam{" diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-db2/src/main/java/org/apache/dolphinscheduler/plugin/datasource/db2/param/Db2DataSourceProcessor.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-db2/src/main/java/org/apache/dolphinscheduler/plugin/datasource/db2/param/Db2DataSourceProcessor.java index e8272e0c1045..eaf07332fd32 100644 --- a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-db2/src/main/java/org/apache/dolphinscheduler/plugin/datasource/db2/param/Db2DataSourceProcessor.java +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-db2/src/main/java/org/apache/dolphinscheduler/plugin/datasource/db2/param/Db2DataSourceProcessor.java @@ -50,8 +50,7 @@ public BaseDataSourceParamDTO castDatasourceParamDTO(String paramJson) { public BaseDataSourceParamDTO createDatasourceParamDTO(String connectionJson) { Db2ConnectionParam connectionParams = (Db2ConnectionParam) createConnectionParams(connectionJson); - Db2DataSourceParamDTO - db2DatasourceParamDTO = new Db2DataSourceParamDTO(); + Db2DataSourceParamDTO db2DatasourceParamDTO = new Db2DataSourceParamDTO(); db2DatasourceParamDTO.setDatabase(connectionParams.getDatabase()); db2DatasourceParamDTO.setOther(parseOther(connectionParams.getOther())); db2DatasourceParamDTO.setUserName(db2DatasourceParamDTO.getUserName()); diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-hive/src/main/java/org/apache/dolphinscheduler/plugin/datasource/hive/HiveDataSourceChannelFactory.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-hive/src/main/java/org/apache/dolphinscheduler/plugin/datasource/hive/HiveDataSourceChannelFactory.java index 12556b66be9c..96ee007c8dbc 100644 --- a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-hive/src/main/java/org/apache/dolphinscheduler/plugin/datasource/hive/HiveDataSourceChannelFactory.java +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-hive/src/main/java/org/apache/dolphinscheduler/plugin/datasource/hive/HiveDataSourceChannelFactory.java @@ -24,6 +24,7 @@ @AutoService(DataSourceChannelFactory.class) public class HiveDataSourceChannelFactory implements DataSourceChannelFactory { + @Override public String getName() { return "hive"; diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-hive/src/main/java/org/apache/dolphinscheduler/plugin/datasource/hive/param/HiveConnectionParam.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-hive/src/main/java/org/apache/dolphinscheduler/plugin/datasource/hive/param/HiveConnectionParam.java index ac432df4d611..a403ff2b27fc 100644 --- a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-hive/src/main/java/org/apache/dolphinscheduler/plugin/datasource/hive/param/HiveConnectionParam.java +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-hive/src/main/java/org/apache/dolphinscheduler/plugin/datasource/hive/param/HiveConnectionParam.java @@ -20,6 +20,7 @@ import org.apache.dolphinscheduler.plugin.datasource.api.datasource.BaseHDFSConnectionParam; public class HiveConnectionParam extends BaseHDFSConnectionParam { + @Override public String toString() { return "HiveConnectionParam{" diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-hive/src/main/java/org/apache/dolphinscheduler/plugin/datasource/hive/param/HiveDataSourceProcessor.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-hive/src/main/java/org/apache/dolphinscheduler/plugin/datasource/hive/param/HiveDataSourceProcessor.java index ab721c977421..1cbc0f493939 100644 --- a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-hive/src/main/java/org/apache/dolphinscheduler/plugin/datasource/hive/param/HiveDataSourceProcessor.java +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-hive/src/main/java/org/apache/dolphinscheduler/plugin/datasource/hive/param/HiveDataSourceProcessor.java @@ -50,8 +50,7 @@ public BaseDataSourceParamDTO castDatasourceParamDTO(String paramJson) { @Override public BaseDataSourceParamDTO createDatasourceParamDTO(String connectionJson) { - HiveDataSourceParamDTO - hiveDataSourceParamDTO = new HiveDataSourceParamDTO(); + HiveDataSourceParamDTO hiveDataSourceParamDTO = new HiveDataSourceParamDTO(); HiveConnectionParam hiveConnectionParam = (HiveConnectionParam) createConnectionParams(connectionJson); hiveDataSourceParamDTO.setDatabase(hiveConnectionParam.getDatabase()); diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-mysql/src/main/java/org/apache/dolphinscheduler/plugin/datasource/mysql/MySQLDataSourceChannelFactory.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-mysql/src/main/java/org/apache/dolphinscheduler/plugin/datasource/mysql/MySQLDataSourceChannelFactory.java index 64788c01111b..e57fc7e61d26 100644 --- a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-mysql/src/main/java/org/apache/dolphinscheduler/plugin/datasource/mysql/MySQLDataSourceChannelFactory.java +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-mysql/src/main/java/org/apache/dolphinscheduler/plugin/datasource/mysql/MySQLDataSourceChannelFactory.java @@ -24,6 +24,7 @@ @AutoService(DataSourceChannelFactory.class) public class MySQLDataSourceChannelFactory implements DataSourceChannelFactory { + @Override public String getName() { return "mysql"; diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-mysql/src/main/java/org/apache/dolphinscheduler/plugin/datasource/mysql/param/MySQLConnectionParam.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-mysql/src/main/java/org/apache/dolphinscheduler/plugin/datasource/mysql/param/MySQLConnectionParam.java index 17e31525929f..89137c0016ac 100644 --- a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-mysql/src/main/java/org/apache/dolphinscheduler/plugin/datasource/mysql/param/MySQLConnectionParam.java +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-mysql/src/main/java/org/apache/dolphinscheduler/plugin/datasource/mysql/param/MySQLConnectionParam.java @@ -20,6 +20,7 @@ import org.apache.dolphinscheduler.spi.datasource.BaseConnectionParam; public class MySQLConnectionParam extends BaseConnectionParam { + @Override public String toString() { return "MySQLConnectionParam{" diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-mysql/src/main/java/org/apache/dolphinscheduler/plugin/datasource/mysql/param/MySQLDataSourceProcessor.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-mysql/src/main/java/org/apache/dolphinscheduler/plugin/datasource/mysql/param/MySQLDataSourceProcessor.java index 5d13cec7f570..3da0072f2fb2 100644 --- a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-mysql/src/main/java/org/apache/dolphinscheduler/plugin/datasource/mysql/param/MySQLDataSourceProcessor.java +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-mysql/src/main/java/org/apache/dolphinscheduler/plugin/datasource/mysql/param/MySQLDataSourceProcessor.java @@ -55,7 +55,8 @@ public class MySQLDataSourceProcessor extends AbstractDataSourceProcessor { private static final String ALLOW_URL_IN_LOCAL_IN_FILE_NAME = "allowUrlInLocalInfile"; - private static final String APPEND_PARAMS = "allowLoadLocalInfile=false&autoDeserialize=false&allowLocalInfile=false&allowUrlInLocalInfile=false"; + private static final String APPEND_PARAMS = + "allowLoadLocalInfile=false&autoDeserialize=false&allowLocalInfile=false&allowUrlInLocalInfile=false"; @Override public BaseDataSourceParamDTO castDatasourceParamDTO(String paramJson) { @@ -64,10 +65,8 @@ public BaseDataSourceParamDTO castDatasourceParamDTO(String paramJson) { @Override public BaseDataSourceParamDTO createDatasourceParamDTO(String connectionJson) { - MySQLConnectionParam - connectionParams = (MySQLConnectionParam) createConnectionParams(connectionJson); - MySQLDataSourceParamDTO - mysqlDatasourceParamDTO = new MySQLDataSourceParamDTO(); + MySQLConnectionParam connectionParams = (MySQLConnectionParam) createConnectionParams(connectionJson); + MySQLDataSourceParamDTO mysqlDatasourceParamDTO = new MySQLDataSourceParamDTO(); mysqlDatasourceParamDTO.setUserName(connectionParams.getUser()); mysqlDatasourceParamDTO.setDatabase(connectionParams.getDatabase()); @@ -85,11 +84,11 @@ public BaseDataSourceParamDTO createDatasourceParamDTO(String connectionJson) { @Override public BaseConnectionParam createConnectionParams(BaseDataSourceParamDTO dataSourceParam) { MySQLDataSourceParamDTO mysqlDatasourceParam = (MySQLDataSourceParamDTO) dataSourceParam; - String address = String.format("%s%s:%s", Constants.JDBC_MYSQL, mysqlDatasourceParam.getHost(), mysqlDatasourceParam.getPort()); + String address = String.format("%s%s:%s", Constants.JDBC_MYSQL, mysqlDatasourceParam.getHost(), + mysqlDatasourceParam.getPort()); String jdbcUrl = String.format("%s/%s", address, mysqlDatasourceParam.getDatabase()); - MySQLConnectionParam - mysqlConnectionParam = new MySQLConnectionParam(); + MySQLConnectionParam mysqlConnectionParam = new MySQLConnectionParam(); mysqlConnectionParam.setJdbcUrl(jdbcUrl); mysqlConnectionParam.setDatabase(mysqlDatasourceParam.getDatabase()); mysqlConnectionParam.setAddress(address); @@ -120,8 +119,7 @@ public String getValidationQuery() { @Override public String getJdbcUrl(ConnectionParam connectionParam) { - MySQLConnectionParam - mysqlConnectionParam = (MySQLConnectionParam) connectionParam; + MySQLConnectionParam mysqlConnectionParam = (MySQLConnectionParam) connectionParam; String jdbcUrl = mysqlConnectionParam.getJdbcUrl(); if (!StringUtils.isEmpty(mysqlConnectionParam.getOther())) { return String.format("%s?%s&%s", jdbcUrl, mysqlConnectionParam.getOther(), APPEND_PARAMS); diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-oracle/src/main/java/org/apache/dolphinscheduler/plugin/datasource/oracle/OracleDataSourceChannelFactory.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-oracle/src/main/java/org/apache/dolphinscheduler/plugin/datasource/oracle/OracleDataSourceChannelFactory.java index 8fcc47766b81..dedbce494693 100644 --- a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-oracle/src/main/java/org/apache/dolphinscheduler/plugin/datasource/oracle/OracleDataSourceChannelFactory.java +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-oracle/src/main/java/org/apache/dolphinscheduler/plugin/datasource/oracle/OracleDataSourceChannelFactory.java @@ -24,6 +24,7 @@ @AutoService(DataSourceChannelFactory.class) public class OracleDataSourceChannelFactory implements DataSourceChannelFactory { + @Override public String getName() { return "oracle"; diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-oracle/src/main/java/org/apache/dolphinscheduler/plugin/datasource/oracle/param/OracleConnectionParam.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-oracle/src/main/java/org/apache/dolphinscheduler/plugin/datasource/oracle/param/OracleConnectionParam.java index 9bbac59e155f..9babd1ed21af 100644 --- a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-oracle/src/main/java/org/apache/dolphinscheduler/plugin/datasource/oracle/param/OracleConnectionParam.java +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-oracle/src/main/java/org/apache/dolphinscheduler/plugin/datasource/oracle/param/OracleConnectionParam.java @@ -21,6 +21,7 @@ import org.apache.dolphinscheduler.spi.enums.DbConnectType; public class OracleConnectionParam extends BaseConnectionParam { + protected DbConnectType connectType; public DbConnectType getConnectType() { diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-postgresql/src/main/java/org/apache/dolphinscheduler/plugin/datasource/postgresql/PostgreSQLDataSourceChannelFactory.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-postgresql/src/main/java/org/apache/dolphinscheduler/plugin/datasource/postgresql/PostgreSQLDataSourceChannelFactory.java index 4572a09af4cd..8aa6e566b781 100644 --- a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-postgresql/src/main/java/org/apache/dolphinscheduler/plugin/datasource/postgresql/PostgreSQLDataSourceChannelFactory.java +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-postgresql/src/main/java/org/apache/dolphinscheduler/plugin/datasource/postgresql/PostgreSQLDataSourceChannelFactory.java @@ -24,6 +24,7 @@ @AutoService(DataSourceChannelFactory.class) public class PostgreSQLDataSourceChannelFactory implements DataSourceChannelFactory { + @Override public String getName() { return "postgresql"; diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-postgresql/src/main/java/org/apache/dolphinscheduler/plugin/datasource/postgresql/param/PostgreSQLConnectionParam.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-postgresql/src/main/java/org/apache/dolphinscheduler/plugin/datasource/postgresql/param/PostgreSQLConnectionParam.java index 94353a772fe3..d39681344c95 100644 --- a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-postgresql/src/main/java/org/apache/dolphinscheduler/plugin/datasource/postgresql/param/PostgreSQLConnectionParam.java +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-postgresql/src/main/java/org/apache/dolphinscheduler/plugin/datasource/postgresql/param/PostgreSQLConnectionParam.java @@ -20,6 +20,7 @@ import org.apache.dolphinscheduler.spi.datasource.BaseConnectionParam; public class PostgreSQLConnectionParam extends BaseConnectionParam { + @Override public String toString() { return "PostgreSQLConnectionParam{" diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-postgresql/src/main/java/org/apache/dolphinscheduler/plugin/datasource/postgresql/param/PostgreSQLDataSourceProcessor.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-postgresql/src/main/java/org/apache/dolphinscheduler/plugin/datasource/postgresql/param/PostgreSQLDataSourceProcessor.java index 79db1dff5412..373e5a092ac8 100644 --- a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-postgresql/src/main/java/org/apache/dolphinscheduler/plugin/datasource/postgresql/param/PostgreSQLDataSourceProcessor.java +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-postgresql/src/main/java/org/apache/dolphinscheduler/plugin/datasource/postgresql/param/PostgreSQLDataSourceProcessor.java @@ -49,8 +49,7 @@ public BaseDataSourceParamDTO castDatasourceParamDTO(String paramJson) { @Override public BaseDataSourceParamDTO createDatasourceParamDTO(String connectionJson) { PostgreSQLConnectionParam connectionParams = (PostgreSQLConnectionParam) createConnectionParams(connectionJson); - PostgreSQLDataSourceParamDTO - postgreSqlDatasourceParamDTO = new PostgreSQLDataSourceParamDTO(); + PostgreSQLDataSourceParamDTO postgreSqlDatasourceParamDTO = new PostgreSQLDataSourceParamDTO(); postgreSqlDatasourceParamDTO.setDatabase(connectionParams.getDatabase()); postgreSqlDatasourceParamDTO.setUserName(connectionParams.getUser()); postgreSqlDatasourceParamDTO.setOther(parseOther(connectionParams.getOther())); @@ -67,7 +66,8 @@ public BaseDataSourceParamDTO createDatasourceParamDTO(String connectionJson) { @Override public BaseConnectionParam createConnectionParams(BaseDataSourceParamDTO datasourceParam) { PostgreSQLDataSourceParamDTO postgreSqlParam = (PostgreSQLDataSourceParamDTO) datasourceParam; - String address = String.format("%s%s:%s", Constants.JDBC_POSTGRESQL, postgreSqlParam.getHost(), postgreSqlParam.getPort()); + String address = String.format("%s%s:%s", Constants.JDBC_POSTGRESQL, postgreSqlParam.getHost(), + postgreSqlParam.getPort()); String jdbcUrl = String.format("%s/%s", address, postgreSqlParam.getDatabase()); PostgreSQLConnectionParam postgreSqlConnectionParam = new PostgreSQLConnectionParam(); @@ -113,7 +113,8 @@ public Connection getConnection(ConnectionParam connectionParam) throws ClassNot PostgreSQLConnectionParam postgreSqlConnectionParam = (PostgreSQLConnectionParam) connectionParam; Class.forName(getDatasourceDriver()); return DriverManager.getConnection(getJdbcUrl(postgreSqlConnectionParam), - postgreSqlConnectionParam.getUser(), PasswordUtils.decodePassword(postgreSqlConnectionParam.getPassword())); + postgreSqlConnectionParam.getUser(), + PasswordUtils.decodePassword(postgreSqlConnectionParam.getPassword())); } @Override diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-presto/src/main/java/org/apache/dolphinscheduler/plugin/datasource/presto/PrestoDataSourceChannelFactory.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-presto/src/main/java/org/apache/dolphinscheduler/plugin/datasource/presto/PrestoDataSourceChannelFactory.java index 95d0f25b8f30..ed1292ffc99e 100644 --- a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-presto/src/main/java/org/apache/dolphinscheduler/plugin/datasource/presto/PrestoDataSourceChannelFactory.java +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-presto/src/main/java/org/apache/dolphinscheduler/plugin/datasource/presto/PrestoDataSourceChannelFactory.java @@ -24,6 +24,7 @@ @AutoService(DataSourceChannelFactory.class) public class PrestoDataSourceChannelFactory implements DataSourceChannelFactory { + @Override public String getName() { return "presto"; diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-presto/src/main/java/org/apache/dolphinscheduler/plugin/datasource/presto/param/PrestoConnectionParam.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-presto/src/main/java/org/apache/dolphinscheduler/plugin/datasource/presto/param/PrestoConnectionParam.java index 8b590ce8bb5e..eba0cba306ab 100644 --- a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-presto/src/main/java/org/apache/dolphinscheduler/plugin/datasource/presto/param/PrestoConnectionParam.java +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-presto/src/main/java/org/apache/dolphinscheduler/plugin/datasource/presto/param/PrestoConnectionParam.java @@ -20,6 +20,7 @@ import org.apache.dolphinscheduler.spi.datasource.BaseConnectionParam; public class PrestoConnectionParam extends BaseConnectionParam { + @Override public String toString() { return "PrestoConnectionParam{" diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-presto/src/main/java/org/apache/dolphinscheduler/plugin/datasource/presto/param/PrestoDataSourceProcessor.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-presto/src/main/java/org/apache/dolphinscheduler/plugin/datasource/presto/param/PrestoDataSourceProcessor.java index 79b5c3cc52ef..4779af9ce09e 100644 --- a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-presto/src/main/java/org/apache/dolphinscheduler/plugin/datasource/presto/param/PrestoDataSourceProcessor.java +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-presto/src/main/java/org/apache/dolphinscheduler/plugin/datasource/presto/param/PrestoDataSourceProcessor.java @@ -55,8 +55,7 @@ public BaseDataSourceParamDTO createDatasourceParamDTO(String connectionJson) { String[] hostSeperator = connectionParams.getAddress().split(Constants.DOUBLE_SLASH); String[] hostPortArray = hostSeperator[hostSeperator.length - 1].split(Constants.COMMA); - PrestoDataSourceParamDTO - prestoDatasourceParamDTO = new PrestoDataSourceParamDTO(); + PrestoDataSourceParamDTO prestoDatasourceParamDTO = new PrestoDataSourceParamDTO(); prestoDatasourceParamDTO.setPort(Integer.parseInt(hostPortArray[0].split(Constants.COLON)[1])); prestoDatasourceParamDTO.setHost(hostPortArray[0].split(Constants.COLON)[0]); prestoDatasourceParamDTO.setDatabase(connectionParams.getDatabase()); diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-redshift/pom.xml b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-redshift/pom.xml index 0a290f31f52b..041dbea12eae 100644 --- a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-redshift/pom.xml +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-redshift/pom.xml @@ -15,15 +15,14 @@ ~ See the License for the specific language governing permissions and ~ limitations under the License. --> - + 4.0.0 - dolphinscheduler-datasource-plugin org.apache.dolphinscheduler + dolphinscheduler-datasource-plugin dev-SNAPSHOT - 4.0.0 dolphinscheduler-datasource-redshift jar @@ -41,4 +40,4 @@ - \ No newline at end of file + diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-redshift/src/main/java/org/apache/dolphinscheduler/plugin/datasource/redshift/RedshiftDataSourceChannel.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-redshift/src/main/java/org/apache/dolphinscheduler/plugin/datasource/redshift/RedshiftDataSourceChannel.java index 13e81724781b..9992f51199cb 100644 --- a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-redshift/src/main/java/org/apache/dolphinscheduler/plugin/datasource/redshift/RedshiftDataSourceChannel.java +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-redshift/src/main/java/org/apache/dolphinscheduler/plugin/datasource/redshift/RedshiftDataSourceChannel.java @@ -23,8 +23,9 @@ import org.apache.dolphinscheduler.spi.enums.DbType; public class RedshiftDataSourceChannel implements DataSourceChannel { + @Override public DataSourceClient createDataSourceClient(BaseConnectionParam baseConnectionParam, DbType dbType) { - return new RedshiftDataSourceClient(baseConnectionParam,dbType); + return new RedshiftDataSourceClient(baseConnectionParam, dbType); } } diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-redshift/src/main/java/org/apache/dolphinscheduler/plugin/datasource/redshift/RedshiftDataSourceChannelFactory.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-redshift/src/main/java/org/apache/dolphinscheduler/plugin/datasource/redshift/RedshiftDataSourceChannelFactory.java index 576a519a2e7d..25a587ae06c4 100644 --- a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-redshift/src/main/java/org/apache/dolphinscheduler/plugin/datasource/redshift/RedshiftDataSourceChannelFactory.java +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-redshift/src/main/java/org/apache/dolphinscheduler/plugin/datasource/redshift/RedshiftDataSourceChannelFactory.java @@ -24,6 +24,7 @@ @AutoService(DataSourceChannelFactory.class) public class RedshiftDataSourceChannelFactory implements DataSourceChannelFactory { + @Override public DataSourceChannel create() { return new RedshiftDataSourceChannel(); diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-redshift/src/main/java/org/apache/dolphinscheduler/plugin/datasource/redshift/RedshiftDataSourceClient.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-redshift/src/main/java/org/apache/dolphinscheduler/plugin/datasource/redshift/RedshiftDataSourceClient.java index f10f7e1f8093..c9a31080d586 100644 --- a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-redshift/src/main/java/org/apache/dolphinscheduler/plugin/datasource/redshift/RedshiftDataSourceClient.java +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-redshift/src/main/java/org/apache/dolphinscheduler/plugin/datasource/redshift/RedshiftDataSourceClient.java @@ -22,6 +22,7 @@ import org.apache.dolphinscheduler.spi.enums.DbType; public class RedshiftDataSourceClient extends CommonDataSourceClient { + public RedshiftDataSourceClient(BaseConnectionParam baseConnectionParam, DbType dbType) { super(baseConnectionParam, dbType); } diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-redshift/src/main/java/org/apache/dolphinscheduler/plugin/datasource/redshift/param/RedshiftConnectionParam.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-redshift/src/main/java/org/apache/dolphinscheduler/plugin/datasource/redshift/param/RedshiftConnectionParam.java index 6022d62b471a..dd3c2d1b6eae 100644 --- a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-redshift/src/main/java/org/apache/dolphinscheduler/plugin/datasource/redshift/param/RedshiftConnectionParam.java +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-redshift/src/main/java/org/apache/dolphinscheduler/plugin/datasource/redshift/param/RedshiftConnectionParam.java @@ -20,6 +20,7 @@ import org.apache.dolphinscheduler.spi.datasource.BaseConnectionParam; public class RedshiftConnectionParam extends BaseConnectionParam { + @Override public String toString() { return "RedshiftConnectionParam{" diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-redshift/src/main/java/org/apache/dolphinscheduler/plugin/datasource/redshift/param/RedshiftDataSourceProcessor.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-redshift/src/main/java/org/apache/dolphinscheduler/plugin/datasource/redshift/param/RedshiftDataSourceProcessor.java index c8586a275c97..88cebe4dc22d 100644 --- a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-redshift/src/main/java/org/apache/dolphinscheduler/plugin/datasource/redshift/param/RedshiftDataSourceProcessor.java +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-redshift/src/main/java/org/apache/dolphinscheduler/plugin/datasource/redshift/param/RedshiftDataSourceProcessor.java @@ -50,14 +50,12 @@ public BaseDataSourceParamDTO castDatasourceParamDTO(String paramJson) { @Override public BaseDataSourceParamDTO createDatasourceParamDTO(String connectionJson) { - RedshiftConnectionParam - connectionParams = (RedshiftConnectionParam) createConnectionParams(connectionJson); + RedshiftConnectionParam connectionParams = (RedshiftConnectionParam) createConnectionParams(connectionJson); String[] hostSeperator = connectionParams.getAddress().split(Constants.DOUBLE_SLASH); String[] hostPortArray = hostSeperator[hostSeperator.length - 1].split(Constants.COMMA); - RedshiftDataSourceParamDTO - redshiftDatasourceParamDTO = new RedshiftDataSourceParamDTO(); + RedshiftDataSourceParamDTO redshiftDatasourceParamDTO = new RedshiftDataSourceParamDTO(); redshiftDatasourceParamDTO.setPort(Integer.parseInt(hostPortArray[0].split(Constants.COLON)[1])); redshiftDatasourceParamDTO.setHost(hostPortArray[0].split(Constants.COLON)[0]); redshiftDatasourceParamDTO.setDatabase(connectionParams.getDatabase()); @@ -70,11 +68,11 @@ public BaseDataSourceParamDTO createDatasourceParamDTO(String connectionJson) { @Override public BaseConnectionParam createConnectionParams(BaseDataSourceParamDTO datasourceParam) { RedshiftDataSourceParamDTO redshiftParam = (RedshiftDataSourceParamDTO) datasourceParam; - String address = String.format("%s%s:%s", Constants.JDBC_REDSHIFT, redshiftParam.getHost(), redshiftParam.getPort()); + String address = + String.format("%s%s:%s", Constants.JDBC_REDSHIFT, redshiftParam.getHost(), redshiftParam.getPort()); String jdbcUrl = address + Constants.SLASH + redshiftParam.getDatabase(); - RedshiftConnectionParam - redshiftConnectionParam = new RedshiftConnectionParam(); + RedshiftConnectionParam redshiftConnectionParam = new RedshiftConnectionParam(); redshiftConnectionParam.setUser(redshiftParam.getUserName()); redshiftConnectionParam.setPassword(PasswordUtils.encodePassword(redshiftParam.getPassword())); redshiftConnectionParam.setOther(transformOther(redshiftParam.getOther())); @@ -105,8 +103,7 @@ public String getValidationQuery() { @Override public String getJdbcUrl(ConnectionParam connectionParam) { - RedshiftConnectionParam - redshiftConnectionParam = (RedshiftConnectionParam) connectionParam; + RedshiftConnectionParam redshiftConnectionParam = (RedshiftConnectionParam) connectionParam; if (!StringUtils.isEmpty(redshiftConnectionParam.getOther())) { return String.format("%s?%s", redshiftConnectionParam.getJdbcUrl(), redshiftConnectionParam.getOther()); } diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-spark/src/main/java/org/apache/dolphinscheduler/plugin/datasource/spark/SparkDataSourceChannelFactory.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-spark/src/main/java/org/apache/dolphinscheduler/plugin/datasource/spark/SparkDataSourceChannelFactory.java index f49c0b3fd9f2..dbda3da5bdef 100644 --- a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-spark/src/main/java/org/apache/dolphinscheduler/plugin/datasource/spark/SparkDataSourceChannelFactory.java +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-spark/src/main/java/org/apache/dolphinscheduler/plugin/datasource/spark/SparkDataSourceChannelFactory.java @@ -24,6 +24,7 @@ @AutoService(DataSourceChannelFactory.class) public class SparkDataSourceChannelFactory implements DataSourceChannelFactory { + @Override public String getName() { return "spark"; diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-spark/src/main/java/org/apache/dolphinscheduler/plugin/datasource/spark/param/SparkConnectionParam.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-spark/src/main/java/org/apache/dolphinscheduler/plugin/datasource/spark/param/SparkConnectionParam.java index 2dbeaa7a198c..b33288d64b5b 100644 --- a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-spark/src/main/java/org/apache/dolphinscheduler/plugin/datasource/spark/param/SparkConnectionParam.java +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-spark/src/main/java/org/apache/dolphinscheduler/plugin/datasource/spark/param/SparkConnectionParam.java @@ -20,6 +20,7 @@ import org.apache.dolphinscheduler.plugin.datasource.api.datasource.BaseHDFSConnectionParam; public class SparkConnectionParam extends BaseHDFSConnectionParam { + @Override public String toString() { return "SparkConnectionParam{" diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-spark/src/main/java/org/apache/dolphinscheduler/plugin/datasource/spark/param/SparkDataSourceProcessor.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-spark/src/main/java/org/apache/dolphinscheduler/plugin/datasource/spark/param/SparkDataSourceProcessor.java index f5dfb2e5ca80..b572ebe86ad1 100644 --- a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-spark/src/main/java/org/apache/dolphinscheduler/plugin/datasource/spark/param/SparkDataSourceProcessor.java +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-spark/src/main/java/org/apache/dolphinscheduler/plugin/datasource/spark/param/SparkDataSourceProcessor.java @@ -53,11 +53,9 @@ public BaseDataSourceParamDTO castDatasourceParamDTO(String paramJson) { @Override public BaseDataSourceParamDTO createDatasourceParamDTO(String connectionJson) { - SparkConnectionParam - connectionParams = (SparkConnectionParam) createConnectionParams(connectionJson); + SparkConnectionParam connectionParams = (SparkConnectionParam) createConnectionParams(connectionJson); - SparkDataSourceParamDTO - sparkDatasourceParamDTO = new SparkDataSourceParamDTO(); + SparkDataSourceParamDTO sparkDatasourceParamDTO = new SparkDataSourceParamDTO(); sparkDatasourceParamDTO.setDatabase(connectionParams.getDatabase()); sparkDatasourceParamDTO.setUserName(connectionParams.getUser()); sparkDatasourceParamDTO.setOther(parseOther(connectionParams.getOther())); @@ -68,7 +66,8 @@ public BaseDataSourceParamDTO createDatasourceParamDTO(String connectionJson) { StringBuilder hosts = new StringBuilder(); String[] tmpArray = connectionParams.getAddress().split(Constants.DOUBLE_SLASH); String[] hostPortArray = tmpArray[tmpArray.length - 1].split(Constants.COMMA); - Arrays.stream(hostPortArray).forEach(hostPort -> hosts.append(hostPort.split(Constants.COLON)[0]).append(Constants.COMMA)); + Arrays.stream(hostPortArray) + .forEach(hostPort -> hosts.append(hostPort.split(Constants.COLON)[0]).append(Constants.COMMA)); hosts.deleteCharAt(hosts.length() - 1); sparkDatasourceParamDTO.setHost(hosts.toString()); @@ -89,8 +88,7 @@ public BaseConnectionParam createConnectionParams(BaseDataSourceParamDTO dataSou String jdbcUrl = address + "/" + sparkDatasourceParam.getDatabase(); - SparkConnectionParam - sparkConnectionParam = new SparkConnectionParam(); + SparkConnectionParam sparkConnectionParam = new SparkConnectionParam(); sparkConnectionParam.setPassword(PasswordUtils.encodePassword(sparkDatasourceParam.getPassword())); sparkConnectionParam.setUser(sparkDatasourceParam.getUserName()); sparkConnectionParam.setOther(transformOther(sparkDatasourceParam.getOther())); @@ -128,8 +126,7 @@ public String getValidationQuery() { @Override public String getJdbcUrl(ConnectionParam connectionParam) { - SparkConnectionParam - sparkConnectionParam = (SparkConnectionParam) connectionParam; + SparkConnectionParam sparkConnectionParam = (SparkConnectionParam) connectionParam; if (!StringUtils.isEmpty(sparkConnectionParam.getOther())) { return String.format("%s;%s", sparkConnectionParam.getJdbcUrl(), sparkConnectionParam.getOther()); } diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-sqlserver/src/main/java/org/apache/dolphinscheduler/plugin/datasource/sqlserver/SQLServerDataSourceChannelFactory.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-sqlserver/src/main/java/org/apache/dolphinscheduler/plugin/datasource/sqlserver/SQLServerDataSourceChannelFactory.java index 58a0a7468b9c..e76f520d1e19 100644 --- a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-sqlserver/src/main/java/org/apache/dolphinscheduler/plugin/datasource/sqlserver/SQLServerDataSourceChannelFactory.java +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-sqlserver/src/main/java/org/apache/dolphinscheduler/plugin/datasource/sqlserver/SQLServerDataSourceChannelFactory.java @@ -24,6 +24,7 @@ @AutoService(DataSourceChannelFactory.class) public class SQLServerDataSourceChannelFactory implements DataSourceChannelFactory { + @Override public String getName() { return "sqlserver"; diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-sqlserver/src/main/java/org/apache/dolphinscheduler/plugin/datasource/sqlserver/param/SQLServerConnectionParam.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-sqlserver/src/main/java/org/apache/dolphinscheduler/plugin/datasource/sqlserver/param/SQLServerConnectionParam.java index c6771cd091e8..777d6d3da8bd 100644 --- a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-sqlserver/src/main/java/org/apache/dolphinscheduler/plugin/datasource/sqlserver/param/SQLServerConnectionParam.java +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-sqlserver/src/main/java/org/apache/dolphinscheduler/plugin/datasource/sqlserver/param/SQLServerConnectionParam.java @@ -20,6 +20,7 @@ import org.apache.dolphinscheduler.spi.datasource.BaseConnectionParam; public class SQLServerConnectionParam extends BaseConnectionParam { + @Override public String toString() { return "SQLServerConnectionParam{" diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-sqlserver/src/main/java/org/apache/dolphinscheduler/plugin/datasource/sqlserver/param/SQLServerDataSourceProcessor.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-sqlserver/src/main/java/org/apache/dolphinscheduler/plugin/datasource/sqlserver/param/SQLServerDataSourceProcessor.java index ee874979ae33..fbe07da93880 100644 --- a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-sqlserver/src/main/java/org/apache/dolphinscheduler/plugin/datasource/sqlserver/param/SQLServerDataSourceProcessor.java +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-sqlserver/src/main/java/org/apache/dolphinscheduler/plugin/datasource/sqlserver/param/SQLServerDataSourceProcessor.java @@ -64,7 +64,8 @@ public BaseDataSourceParamDTO createDatasourceParamDTO(String connectionJson) { @Override public BaseConnectionParam createConnectionParams(BaseDataSourceParamDTO datasourceParam) { SQLServerDataSourceParamDTO sqlServerParam = (SQLServerDataSourceParamDTO) datasourceParam; - String address = String.format("%s%s:%s", Constants.JDBC_SQLSERVER, sqlServerParam.getHost(), sqlServerParam.getPort()); + String address = + String.format("%s%s:%s", Constants.JDBC_SQLSERVER, sqlServerParam.getHost(), sqlServerParam.getPort()); String jdbcUrl = address + ";databaseName=" + sqlServerParam.getDatabase(); SQLServerConnectionParam sqlServerConnectionParam = new SQLServerConnectionParam(); diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/MasterServer.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/MasterServer.java index 6419a973652a..b334865e880f 100644 --- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/MasterServer.java +++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/MasterServer.java @@ -29,6 +29,9 @@ import org.apache.dolphinscheduler.server.master.runner.MasterSchedulerBootstrap; import org.apache.dolphinscheduler.service.bean.SpringApplicationContext; import org.apache.dolphinscheduler.service.task.TaskPluginManager; + +import javax.annotation.PostConstruct; + import org.quartz.SchedulerException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -39,8 +42,6 @@ import org.springframework.context.annotation.ComponentScan; import org.springframework.transaction.annotation.EnableTransactionManagement; -import javax.annotation.PostConstruct; - @SpringBootApplication @ComponentScan("org.apache.dolphinscheduler") @EnableTransactionManagement diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/builder/TaskExecutionContextBuilder.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/builder/TaskExecutionContextBuilder.java index 262a5acc62d3..6c37353ebd3e 100644 --- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/builder/TaskExecutionContextBuilder.java +++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/builder/TaskExecutionContextBuilder.java @@ -33,11 +33,12 @@ import org.apache.dolphinscheduler.plugin.task.api.enums.TaskTimeoutStrategy; import org.apache.dolphinscheduler.plugin.task.api.model.Property; import org.apache.dolphinscheduler.plugin.task.api.parameters.resource.ResourceParametersHelper; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import java.util.Map; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + /** * TaskExecutionContext builder */ diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/cache/ProcessInstanceExecCacheManager.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/cache/ProcessInstanceExecCacheManager.java index bf5a08b47772..1352d125c377 100644 --- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/cache/ProcessInstanceExecCacheManager.java +++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/cache/ProcessInstanceExecCacheManager.java @@ -17,7 +17,6 @@ package org.apache.dolphinscheduler.server.master.cache; -import lombok.NonNull; import org.apache.dolphinscheduler.server.master.runner.WorkflowExecuteRunnable; import java.util.Collection; @@ -68,4 +67,4 @@ public interface ProcessInstanceExecCacheManager { Collection getAll(); void clearCache(); -} \ No newline at end of file +} diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/cache/StreamTaskInstanceExecCacheManager.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/cache/StreamTaskInstanceExecCacheManager.java index 681d5129bcbc..e4be3e839792 100644 --- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/cache/StreamTaskInstanceExecCacheManager.java +++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/cache/StreamTaskInstanceExecCacheManager.java @@ -18,7 +18,6 @@ package org.apache.dolphinscheduler.server.master.cache; import org.apache.dolphinscheduler.server.master.runner.StreamTaskExecuteRunnable; -import org.apache.dolphinscheduler.server.master.runner.WorkflowExecuteRunnable; import java.util.Collection; @@ -49,4 +48,4 @@ public interface StreamTaskInstanceExecCacheManager { * @return all streamTaskExecuteRunnable in cache */ Collection getAll(); -} \ No newline at end of file +} diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/cache/impl/ProcessInstanceExecCacheManagerImpl.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/cache/impl/ProcessInstanceExecCacheManagerImpl.java index 3588e3d33621..105aa3970879 100644 --- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/cache/impl/ProcessInstanceExecCacheManagerImpl.java +++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/cache/impl/ProcessInstanceExecCacheManagerImpl.java @@ -26,12 +26,12 @@ import javax.annotation.PostConstruct; +import lombok.NonNull; + import org.springframework.stereotype.Component; import com.google.common.collect.ImmutableList; -import lombok.NonNull; - /** * cache of process instance id and WorkflowExecuteThread */ @@ -75,4 +75,4 @@ public Collection getAll() { public void clearCache() { processInstanceExecMaps.clear(); } -} \ No newline at end of file +} diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/cache/impl/StreamTaskInstanceExecCacheManagerImpl.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/cache/impl/StreamTaskInstanceExecCacheManagerImpl.java index 1033ab35e5f2..2d4e46b7c611 100644 --- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/cache/impl/StreamTaskInstanceExecCacheManagerImpl.java +++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/cache/impl/StreamTaskInstanceExecCacheManagerImpl.java @@ -18,29 +18,28 @@ package org.apache.dolphinscheduler.server.master.cache.impl; import org.apache.dolphinscheduler.server.master.cache.StreamTaskInstanceExecCacheManager; -import org.apache.dolphinscheduler.server.master.metrics.ProcessInstanceMetrics; import org.apache.dolphinscheduler.server.master.metrics.TaskMetrics; import org.apache.dolphinscheduler.server.master.runner.StreamTaskExecuteRunnable; -import org.apache.dolphinscheduler.server.master.runner.WorkflowExecuteRunnable; import java.util.Collection; import java.util.concurrent.ConcurrentHashMap; import javax.annotation.PostConstruct; +import lombok.NonNull; + import org.springframework.stereotype.Component; import com.google.common.collect.ImmutableList; -import lombok.NonNull; - /** * cache of process instance id and WorkflowExecuteThread */ @Component public class StreamTaskInstanceExecCacheManagerImpl implements StreamTaskInstanceExecCacheManager { - private final ConcurrentHashMap streamTaskInstanceExecMaps = new ConcurrentHashMap<>(); + private final ConcurrentHashMap streamTaskInstanceExecMaps = + new ConcurrentHashMap<>(); @PostConstruct public void registerMetrics() { @@ -71,4 +70,4 @@ public void cache(int taskInstanceId, @NonNull StreamTaskExecuteRunnable streamT public Collection getAll() { return ImmutableList.copyOf(streamTaskInstanceExecMaps.values()); } -} \ No newline at end of file +} diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/config/MasterConfig.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/config/MasterConfig.java index b0426f477a4a..f3b3d78fb783 100644 --- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/config/MasterConfig.java +++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/config/MasterConfig.java @@ -17,12 +17,18 @@ package org.apache.dolphinscheduler.server.master.config; -import lombok.Data; +import static org.apache.dolphinscheduler.common.Constants.REGISTRY_DOLPHINSCHEDULER_MASTERS; + import org.apache.dolphinscheduler.common.utils.NetUtils; import org.apache.dolphinscheduler.registry.api.ConnectStrategyProperties; import org.apache.dolphinscheduler.server.master.dispatch.host.assign.HostSelector; import org.apache.dolphinscheduler.server.master.processor.queue.TaskExecuteRunnable; import org.apache.dolphinscheduler.server.master.runner.WorkflowExecuteRunnable; + +import java.time.Duration; + +import lombok.Data; + import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.boot.context.properties.ConfigurationProperties; @@ -31,10 +37,6 @@ import org.springframework.validation.Validator; import org.springframework.validation.annotation.Validated; -import java.time.Duration; - -import static org.apache.dolphinscheduler.common.Constants.REGISTRY_DOLPHINSCHEDULER_MASTERS; - @Data @Validated @Configuration diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/consumer/TaskPriorityQueueConsumer.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/consumer/TaskPriorityQueueConsumer.java index f073fd55f3dc..5d31f82fe93f 100644 --- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/consumer/TaskPriorityQueueConsumer.java +++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/consumer/TaskPriorityQueueConsumer.java @@ -17,7 +17,6 @@ package org.apache.dolphinscheduler.server.master.consumer; -import org.apache.commons.collections.CollectionUtils; import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.lifecycle.ServerLifeCycleManager; import org.apache.dolphinscheduler.common.thread.BaseDaemonThread; @@ -40,12 +39,9 @@ import org.apache.dolphinscheduler.service.process.ProcessService; import org.apache.dolphinscheduler.service.queue.TaskPriority; import org.apache.dolphinscheduler.service.queue.TaskPriorityQueue; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.stereotype.Component; -import javax.annotation.PostConstruct; +import org.apache.commons.collections.CollectionUtils; + import java.util.ArrayList; import java.util.Collections; import java.util.List; @@ -55,6 +51,13 @@ import java.util.concurrent.ThreadPoolExecutor; import java.util.concurrent.TimeUnit; +import javax.annotation.PostConstruct; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; + /** * TaskUpdateQueue consumer */ diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/controller/WorkflowExecuteController.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/controller/WorkflowExecuteController.java index 793dc40439cb..28f26d5cb86a 100644 --- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/controller/WorkflowExecuteController.java +++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/controller/WorkflowExecuteController.java @@ -45,7 +45,8 @@ public class WorkflowExecuteController { @GetMapping("") @ResponseStatus(HttpStatus.OK) public WorkflowExecuteDto queryExecuteData(@RequestParam("id") int processInstanceId) { - Optional workflowExecuteDtoOptional = executingService.queryWorkflowExecutingData(processInstanceId); + Optional workflowExecuteDtoOptional = + executingService.queryWorkflowExecutingData(processInstanceId); return workflowExecuteDtoOptional.orElse(null); } } diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/dispatch/ExecutorDispatcher.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/dispatch/ExecutorDispatcher.java index d439c28267b2..3b1a0d48390b 100644 --- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/dispatch/ExecutorDispatcher.java +++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/dispatch/ExecutorDispatcher.java @@ -85,7 +85,7 @@ public Boolean dispatch(final ExecutionContext context) throws ExecuteException Host host = hostManager.select(context); if (StringUtils.isEmpty(host.getAddress())) { logger.warn("fail to execute : {} due to no suitable worker, current task needs worker group {} to execute", - context.getCommand(), context.getWorkerGroup()); + context.getCommand(), context.getWorkerGroup()); return false; } context.setHost(host); diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/dispatch/executor/NettyExecutorManager.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/dispatch/executor/NettyExecutorManager.java index 62da82156fd4..93541a9b3763 100644 --- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/dispatch/executor/NettyExecutorManager.java +++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/dispatch/executor/NettyExecutorManager.java @@ -27,8 +27,6 @@ import org.apache.dolphinscheduler.server.master.dispatch.context.ExecutionContext; import org.apache.dolphinscheduler.server.master.dispatch.enums.ExecutorType; import org.apache.dolphinscheduler.server.master.dispatch.exceptions.ExecuteException; -import org.apache.dolphinscheduler.server.master.processor.TaskExecuteResponseProcessor; -import org.apache.dolphinscheduler.server.master.processor.TaskExecuteRunningProcessor; import org.apache.dolphinscheduler.server.master.processor.TaskKillResponseProcessor; import org.apache.dolphinscheduler.server.master.processor.TaskRecallProcessor; import org.apache.dolphinscheduler.server.master.registry.ServerNodeManager; diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/dispatch/host/CommonHostManager.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/dispatch/host/CommonHostManager.java index bbf84a3d7400..fd7dc97e3e51 100644 --- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/dispatch/host/CommonHostManager.java +++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/dispatch/host/CommonHostManager.java @@ -17,7 +17,6 @@ package org.apache.dolphinscheduler.server.master.dispatch.host; -import org.apache.commons.collections.CollectionUtils; import org.apache.dolphinscheduler.common.model.WorkerHeartBeat; import org.apache.dolphinscheduler.remote.utils.Host; import org.apache.dolphinscheduler.server.master.dispatch.context.ExecutionContext; @@ -26,14 +25,14 @@ import org.apache.dolphinscheduler.server.master.registry.ServerNodeManager; import org.apache.commons.collections.CollectionUtils; -import org.apache.commons.lang3.StringUtils; -import org.springframework.beans.factory.annotation.Autowired; import java.util.ArrayList; import java.util.Collection; import java.util.List; import java.util.Set; +import org.springframework.beans.factory.annotation.Autowired; + /** * common host manager */ diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/dispatch/host/LowerWeightHostManager.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/dispatch/host/LowerWeightHostManager.java index 3ad46ddd53bc..0ac1a53180ad 100644 --- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/dispatch/host/LowerWeightHostManager.java +++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/dispatch/host/LowerWeightHostManager.java @@ -17,7 +17,6 @@ package org.apache.dolphinscheduler.server.master.dispatch.host; -import org.apache.commons.collections.CollectionUtils; import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.model.WorkerHeartBeat; import org.apache.dolphinscheduler.remote.utils.Host; @@ -26,10 +25,9 @@ import org.apache.dolphinscheduler.server.master.dispatch.host.assign.HostWorker; import org.apache.dolphinscheduler.server.master.dispatch.host.assign.LowerWeightRoundRobin; import org.apache.dolphinscheduler.server.master.registry.WorkerInfoChangeListener; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import javax.annotation.PostConstruct; +import org.apache.commons.collections.CollectionUtils; + import java.util.Collection; import java.util.HashMap; import java.util.HashSet; @@ -40,6 +38,11 @@ import java.util.concurrent.locks.Lock; import java.util.concurrent.locks.ReentrantLock; +import javax.annotation.PostConstruct; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + /** * lower weight host manager */ @@ -90,8 +93,8 @@ public HostWorker select(Collection nodes) { throw new UnsupportedOperationException("not support"); } - private class WorkerWeightListener implements WorkerInfoChangeListener { + @Override public void notify(Map> workerGroups, Map workerNodeInfo) { syncWorkerResources(workerGroups, workerNodeInfo); diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/dispatch/host/assign/AbstractSelector.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/dispatch/host/assign/AbstractSelector.java index 0e61599e7953..693ff856e506 100644 --- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/dispatch/host/assign/AbstractSelector.java +++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/dispatch/host/assign/AbstractSelector.java @@ -24,7 +24,8 @@ /** * AbstractSelector */ -public abstract class AbstractSelector implements Selector { +public abstract class AbstractSelector implements Selector { + @Override public T select(Collection source) { @@ -36,7 +37,7 @@ public T select(Collection source) { * if only one , return directly */ if (source.size() == 1) { - return (T)source.toArray()[0]; + return (T) source.toArray()[0]; } return doSelect(source); } diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/dispatch/host/assign/HostWeight.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/dispatch/host/assign/HostWeight.java index a44158223513..b3cb2ae75c0f 100644 --- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/dispatch/host/assign/HostWeight.java +++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/dispatch/host/assign/HostWeight.java @@ -39,7 +39,8 @@ public class HostWeight { private final int waitingTaskCount; - public HostWeight(HostWorker hostWorker, double cpu, double memory, double loadAverage, int waitingTaskCount, long startTime) { + public HostWeight(HostWorker hostWorker, double cpu, double memory, double loadAverage, int waitingTaskCount, + long startTime) { this.hostWorker = hostWorker; this.weight = calculateWeight(cpu, memory, loadAverage, startTime); this.currentWeight = this.weight; @@ -63,7 +64,7 @@ public HostWorker getHostWorker() { } public Host getHost() { - return (Host)hostWorker; + return (Host) hostWorker; } public int getWaitingTaskCount() { @@ -73,11 +74,11 @@ public int getWaitingTaskCount() { @Override public String toString() { return "HostWeight{" - + "hostWorker=" + hostWorker - + ", weight=" + weight - + ", currentWeight=" + currentWeight - + ", waitingTaskCount=" + waitingTaskCount - + '}'; + + "hostWorker=" + hostWorker + + ", weight=" + weight + + ", currentWeight=" + currentWeight + + ", waitingTaskCount=" + waitingTaskCount + + '}'; } private double calculateWeight(double cpu, double memory, double loadAverage, long startTime) { diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/dispatch/host/assign/LowerWeightRoundRobin.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/dispatch/host/assign/LowerWeightRoundRobin.java index 3388129b2a0a..8ddfff6da58c 100644 --- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/dispatch/host/assign/LowerWeightRoundRobin.java +++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/dispatch/host/assign/LowerWeightRoundRobin.java @@ -62,20 +62,20 @@ private List canAssignTaskHost(Collection sources) { if (CollectionUtils.isEmpty(sources)) { return Collections.emptyList(); } - List zeroWaitingTask = sources.stream().filter(h -> h.getWaitingTaskCount() == 0).collect(Collectors.toList()); + List zeroWaitingTask = + sources.stream().filter(h -> h.getWaitingTaskCount() == 0).collect(Collectors.toList()); if (!zeroWaitingTask.isEmpty()) { return zeroWaitingTask; } HostWeight hostWeight = sources.stream().min(Comparator.comparing(HostWeight::getWaitingTaskCount)).get(); List waitingTask = Lists.newArrayList(hostWeight); - List equalWaitingTask = sources.stream().filter(h -> !h.getHost().equals(hostWeight.getHost()) && h.getWaitingTaskCount() == hostWeight.getWaitingTaskCount()) - .collect(Collectors.toList()); + List equalWaitingTask = sources.stream() + .filter(h -> !h.getHost().equals(hostWeight.getHost()) + && h.getWaitingTaskCount() == hostWeight.getWaitingTaskCount()) + .collect(Collectors.toList()); if (!equalWaitingTask.isEmpty()) { waitingTask.addAll(equalWaitingTask); } return waitingTask; } } - - - diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/dispatch/host/assign/RoundRobinSelector.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/dispatch/host/assign/RoundRobinSelector.java index 5639fc4ce2b9..8f21acef6d1e 100644 --- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/dispatch/host/assign/RoundRobinSelector.java +++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/dispatch/host/assign/RoundRobinSelector.java @@ -33,13 +33,15 @@ @Service public class RoundRobinSelector extends AbstractSelector { - private ConcurrentMap> workGroupWeightMap = new ConcurrentHashMap<>(); + private ConcurrentMap> workGroupWeightMap = + new ConcurrentHashMap<>(); private static final int RECYCLE_PERIOD = 100000; private AtomicBoolean updateLock = new AtomicBoolean(); protected static class WeightedRoundRobin { + private int weight; private AtomicLong current = new AtomicLong(0); private long lastUpdate; diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/event/StateEvent.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/event/StateEvent.java index f3a9a6579eaf..845c77e296f9 100644 --- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/event/StateEvent.java +++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/event/StateEvent.java @@ -17,13 +17,13 @@ package org.apache.dolphinscheduler.server.master.event; -import lombok.NonNull; import org.apache.dolphinscheduler.common.enums.StateEventType; -import io.netty.channel.Channel; - import javax.annotation.Nullable; +import lombok.NonNull; +import io.netty.channel.Channel; + /** * state event */ diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/event/StateEventHandler.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/event/StateEventHandler.java index 00808b2e2954..93d4a7d4b3aa 100644 --- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/event/StateEventHandler.java +++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/event/StateEventHandler.java @@ -29,8 +29,8 @@ public interface StateEventHandler { * @throws StateEventHandleException this exception means it can be recovered. * @throws StateEventHandleError this exception means it cannot be recovered, so the event need to drop. */ - boolean handleStateEvent(WorkflowExecuteRunnable workflowExecuteRunnable, StateEvent stateEvent) - throws StateEventHandleException, StateEventHandleError; + boolean handleStateEvent(WorkflowExecuteRunnable workflowExecuteRunnable, + StateEvent stateEvent) throws StateEventHandleException, StateEventHandleError; StateEventType getEventType(); } diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/event/StateEventHandlerManager.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/event/StateEventHandlerManager.java index b436b5589057..b91da8cc511c 100644 --- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/event/StateEventHandlerManager.java +++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/event/StateEventHandlerManager.java @@ -30,8 +30,8 @@ public class StateEventHandlerManager { static { ServiceLoader.load(StateEventHandler.class) - .forEach(stateEventHandler -> stateEventHandlerMap.put(stateEventHandler.getEventType(), - stateEventHandler)); + .forEach(stateEventHandler -> stateEventHandlerMap.put(stateEventHandler.getEventType(), + stateEventHandler)); } public static Optional getStateEventHandler(StateEventType stateEventType) { diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/event/TaskRetryStateEventHandler.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/event/TaskRetryStateEventHandler.java index 1d5f7a388359..4a55b6b43080 100644 --- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/event/TaskRetryStateEventHandler.java +++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/event/TaskRetryStateEventHandler.java @@ -17,15 +17,17 @@ package org.apache.dolphinscheduler.server.master.event; -import com.google.auto.service.AutoService; import org.apache.dolphinscheduler.common.enums.StateEventType; import org.apache.dolphinscheduler.dao.entity.TaskInstance; import org.apache.dolphinscheduler.server.master.metrics.TaskMetrics; import org.apache.dolphinscheduler.server.master.runner.WorkflowExecuteRunnable; + +import java.util.Map; + import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.util.Map; +import com.google.auto.service.AutoService; @AutoService(StateEventHandler.class) public class TaskRetryStateEventHandler implements StateEventHandler { diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/event/TaskRunningEventHandler.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/event/TaskRunningEventHandler.java index fe449a9f6f02..3dc86b3a37d8 100644 --- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/event/TaskRunningEventHandler.java +++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/event/TaskRunningEventHandler.java @@ -27,13 +27,14 @@ import org.apache.dolphinscheduler.server.master.runner.WorkflowExecuteRunnable; import org.apache.dolphinscheduler.server.master.runner.WorkflowExecuteThreadPool; import org.apache.dolphinscheduler.service.process.ProcessService; + +import java.util.Optional; + import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; -import java.util.Optional; - @Component public class TaskRunningEventHandler implements TaskEventHandler { diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/event/TaskStateEvent.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/event/TaskStateEvent.java index 1ad96b93465f..157d62e86370 100644 --- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/event/TaskStateEvent.java +++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/event/TaskStateEvent.java @@ -17,14 +17,15 @@ package org.apache.dolphinscheduler.server.master.event; -import io.netty.channel.Channel; +import org.apache.dolphinscheduler.common.enums.StateEventType; +import org.apache.dolphinscheduler.plugin.task.api.enums.TaskExecutionStatus; + import lombok.AllArgsConstructor; import lombok.Builder; import lombok.Data; import lombok.NoArgsConstructor; import lombok.NonNull; -import org.apache.dolphinscheduler.common.enums.StateEventType; -import org.apache.dolphinscheduler.plugin.task.api.enums.TaskExecutionStatus; +import io.netty.channel.Channel; @Data @Builder diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/event/TaskTimeoutStateEventHandler.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/event/TaskTimeoutStateEventHandler.java index 9efe773b19d5..034bf8107f99 100644 --- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/event/TaskTimeoutStateEventHandler.java +++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/event/TaskTimeoutStateEventHandler.java @@ -17,7 +17,6 @@ package org.apache.dolphinscheduler.server.master.event; -import com.google.auto.service.AutoService; import org.apache.dolphinscheduler.common.enums.StateEventType; import org.apache.dolphinscheduler.common.enums.TimeoutFlag; import org.apache.dolphinscheduler.dao.entity.TaskInstance; @@ -28,9 +27,12 @@ import org.apache.dolphinscheduler.server.master.runner.task.TaskAction; import java.util.Map; + import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import com.google.auto.service.AutoService; + @AutoService(StateEventHandler.class) public class TaskTimeoutStateEventHandler implements StateEventHandler { @@ -38,17 +40,17 @@ public class TaskTimeoutStateEventHandler implements StateEventHandler { @Override public boolean handleStateEvent(WorkflowExecuteRunnable workflowExecuteRunnable, - StateEvent stateEvent) throws StateEventHandleError { + StateEvent stateEvent) throws StateEventHandleError { TaskStateEvent taskStateEvent = (TaskStateEvent) stateEvent; TaskMetrics.incTaskInstanceByState("timeout"); workflowExecuteRunnable.checkTaskInstanceByStateEvent(taskStateEvent); TaskInstance taskInstance = - workflowExecuteRunnable.getTaskInstance(taskStateEvent.getTaskInstanceId()).orElseThrow( - () -> new StateEventHandleError(String.format( - "Cannot find the task instance from workflow execute runnable, taskInstanceId: %s", - taskStateEvent.getTaskInstanceId()))); + workflowExecuteRunnable.getTaskInstance(taskStateEvent.getTaskInstanceId()).orElseThrow( + () -> new StateEventHandleError(String.format( + "Cannot find the task instance from workflow execute runnable, taskInstanceId: %s", + taskStateEvent.getTaskInstanceId()))); logger.info("Handle task instance state timout event, taskInstanceId: {}", taskStateEvent.getTaskInstanceId()); @@ -56,22 +58,22 @@ public boolean handleStateEvent(WorkflowExecuteRunnable workflowExecuteRunnable, return true; } TaskTimeoutStrategy taskTimeoutStrategy = taskInstance.getTaskDefine() - .getTimeoutNotifyStrategy(); + .getTimeoutNotifyStrategy(); Map activeTaskProcessMap = workflowExecuteRunnable - .getActiveTaskProcessMap(); + .getActiveTaskProcessMap(); if ((TaskTimeoutStrategy.FAILED == taskTimeoutStrategy - || TaskTimeoutStrategy.WARNFAILED == taskTimeoutStrategy)) { + || TaskTimeoutStrategy.WARNFAILED == taskTimeoutStrategy)) { if (activeTaskProcessMap.containsKey(taskInstance.getTaskCode())) { ITaskProcessor taskProcessor = activeTaskProcessMap.get(taskInstance.getTaskCode()); taskProcessor.action(TaskAction.TIMEOUT); } else { logger.warn( - "cannot find the task processor for task {}, so skip task processor action.", - taskInstance.getTaskCode()); + "cannot find the task processor for task {}, so skip task processor action.", + taskInstance.getTaskCode()); } } if (TaskTimeoutStrategy.WARN == taskTimeoutStrategy - || TaskTimeoutStrategy.WARNFAILED == taskTimeoutStrategy) { + || TaskTimeoutStrategy.WARNFAILED == taskTimeoutStrategy) { workflowExecuteRunnable.processTimeout(); workflowExecuteRunnable.taskTimeout(taskInstance); } diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/event/TaskWaitTaskGroupStateHandler.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/event/TaskWaitTaskGroupStateHandler.java index 8dcca91d8943..02d08a6616dd 100644 --- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/event/TaskWaitTaskGroupStateHandler.java +++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/event/TaskWaitTaskGroupStateHandler.java @@ -20,10 +20,11 @@ import org.apache.dolphinscheduler.common.enums.StateEventType; import org.apache.dolphinscheduler.server.master.runner.WorkflowExecuteRunnable; -import com.google.auto.service.AutoService; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import com.google.auto.service.AutoService; + @AutoService(StateEventHandler.class) public class TaskWaitTaskGroupStateHandler implements StateEventHandler { diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/event/WorkflowBlockStateEventHandler.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/event/WorkflowBlockStateEventHandler.java index cac30a8072cf..5c9f901e68ea 100644 --- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/event/WorkflowBlockStateEventHandler.java +++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/event/WorkflowBlockStateEventHandler.java @@ -36,14 +36,14 @@ public class WorkflowBlockStateEventHandler implements StateEventHandler { private static final Logger logger = LoggerFactory.getLogger(WorkflowBlockStateEventHandler.class); @Override - public boolean handleStateEvent(WorkflowExecuteRunnable workflowExecuteRunnable, StateEvent stateEvent) - throws StateEventHandleError { + public boolean handleStateEvent(WorkflowExecuteRunnable workflowExecuteRunnable, + StateEvent stateEvent) throws StateEventHandleError { logger.info("Handle workflow instance state block event"); Optional taskInstanceOptional = - workflowExecuteRunnable.getTaskInstance(stateEvent.getTaskInstanceId()); + workflowExecuteRunnable.getTaskInstance(stateEvent.getTaskInstanceId()); if (!taskInstanceOptional.isPresent()) { throw new StateEventHandleError("Cannot find taskInstance from taskMap by taskInstanceId: " - + stateEvent.getTaskInstanceId()); + + stateEvent.getTaskInstanceId()); } TaskInstance task = taskInstanceOptional.get(); diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/event/WorkflowStartEventHandler.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/event/WorkflowStartEventHandler.java index 8c5e27c1170a..13d82f067147 100644 --- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/event/WorkflowStartEventHandler.java +++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/event/WorkflowStartEventHandler.java @@ -53,27 +53,27 @@ public class WorkflowStartEventHandler implements WorkflowEventHandler { public void handleWorkflowEvent(final WorkflowEvent workflowEvent) throws WorkflowEventHandleError { logger.info("Handle workflow start event, begin to start a workflow, event: {}", workflowEvent); WorkflowExecuteRunnable workflowExecuteRunnable = processInstanceExecCacheManager.getByProcessInstanceId( - workflowEvent.getWorkflowInstanceId()); + workflowEvent.getWorkflowInstanceId()); if (workflowExecuteRunnable == null) { throw new WorkflowEventHandleError( - "The workflow start event is invalid, cannot find the workflow instance from cache"); + "The workflow start event is invalid, cannot find the workflow instance from cache"); } ProcessInstanceMetrics.incProcessInstanceByState("submit"); ProcessInstance processInstance = workflowExecuteRunnable.getProcessInstance(); CompletableFuture.supplyAsync(workflowExecuteRunnable::call, workflowExecuteThreadPool) - .thenAccept(workflowSubmitStatue -> { - if (WorkflowSubmitStatue.SUCCESS == workflowSubmitStatue) { - // submit failed will resend the event to workflow event queue - logger.info("Success submit the workflow instance"); - if (processInstance.getTimeout() > 0) { - stateWheelExecuteThread.addProcess4TimeoutCheck(processInstance); + .thenAccept(workflowSubmitStatue -> { + if (WorkflowSubmitStatue.SUCCESS == workflowSubmitStatue) { + // submit failed will resend the event to workflow event queue + logger.info("Success submit the workflow instance"); + if (processInstance.getTimeout() > 0) { + stateWheelExecuteThread.addProcess4TimeoutCheck(processInstance); + } + } else { + logger.error("Failed to submit the workflow instance, will resend the workflow start event: {}", + workflowEvent); + workflowEventQueue.addEvent(workflowEvent); } - } else { - logger.error("Failed to submit the workflow instance, will resend the workflow start event: {}", - workflowEvent); - workflowEventQueue.addEvent(workflowEvent); - } - }); + }); } @Override diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/event/WorkflowStateEvent.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/event/WorkflowStateEvent.java index 326710822808..cf65575caa3e 100644 --- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/event/WorkflowStateEvent.java +++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/event/WorkflowStateEvent.java @@ -17,11 +17,16 @@ package org.apache.dolphinscheduler.server.master.event; -import io.netty.channel.Channel; -import lombok.*; import org.apache.dolphinscheduler.common.enums.StateEventType; import org.apache.dolphinscheduler.common.enums.WorkflowExecutionStatus; +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; +import lombok.NonNull; +import io.netty.channel.Channel; + @Data @Builder @AllArgsConstructor diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/event/WorkflowStateEventHandler.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/event/WorkflowStateEventHandler.java index c3f49111a73e..65015f04582f 100644 --- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/event/WorkflowStateEventHandler.java +++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/event/WorkflowStateEventHandler.java @@ -17,15 +17,17 @@ package org.apache.dolphinscheduler.server.master.event; -import com.google.auto.service.AutoService; import org.apache.dolphinscheduler.common.enums.StateEventType; import org.apache.dolphinscheduler.dao.entity.ProcessDefinition; import org.apache.dolphinscheduler.dao.entity.ProcessInstance; import org.apache.dolphinscheduler.server.master.metrics.ProcessInstanceMetrics; import org.apache.dolphinscheduler.server.master.runner.WorkflowExecuteRunnable; + import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import com.google.auto.service.AutoService; + @AutoService(StateEventHandler.class) public class WorkflowStateEventHandler implements StateEventHandler { diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/event/WorkflowTimeoutStateEventHandler.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/event/WorkflowTimeoutStateEventHandler.java index 4a9fd993923a..d68e97c5e355 100644 --- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/event/WorkflowTimeoutStateEventHandler.java +++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/event/WorkflowTimeoutStateEventHandler.java @@ -21,10 +21,11 @@ import org.apache.dolphinscheduler.server.master.metrics.ProcessInstanceMetrics; import org.apache.dolphinscheduler.server.master.runner.WorkflowExecuteRunnable; -import com.google.auto.service.AutoService; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import com.google.auto.service.AutoService; + @AutoService(StateEventHandler.class) public class WorkflowTimeoutStateEventHandler implements StateEventHandler { diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/metrics/MasterServerMetrics.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/metrics/MasterServerMetrics.java index 0ed522e7cd2e..a0d153ab626c 100644 --- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/metrics/MasterServerMetrics.java +++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/metrics/MasterServerMetrics.java @@ -17,9 +17,9 @@ package org.apache.dolphinscheduler.server.master.metrics; +import lombok.experimental.UtilityClass; import io.micrometer.core.instrument.Counter; import io.micrometer.core.instrument.Metrics; -import lombok.experimental.UtilityClass; @UtilityClass public class MasterServerMetrics { diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/metrics/ProcessInstanceMetrics.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/metrics/ProcessInstanceMetrics.java index 867f8ae548ee..d6107784240f 100644 --- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/metrics/ProcessInstanceMetrics.java +++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/metrics/ProcessInstanceMetrics.java @@ -23,13 +23,14 @@ import java.util.concurrent.TimeUnit; import java.util.function.Supplier; +import lombok.experimental.UtilityClass; + import com.google.common.collect.ImmutableSet; import io.micrometer.core.instrument.Counter; import io.micrometer.core.instrument.Gauge; import io.micrometer.core.instrument.Metrics; import io.micrometer.core.instrument.Timer; -import lombok.experimental.UtilityClass; @UtilityClass public class ProcessInstanceMetrics { @@ -46,21 +47,20 @@ public class ProcessInstanceMetrics { Counter.builder("ds.workflow.instance.count") .tag("state", state) .description(String.format("Process instance %s total count", state)) - .register(Metrics.globalRegistry) - ); + .register(Metrics.globalRegistry)); } } private final Timer commandQueryTimer = - Timer.builder("ds.workflow.command.query.duration") - .description("Command query duration") - .register(Metrics.globalRegistry); + Timer.builder("ds.workflow.command.query.duration") + .description("Command query duration") + .register(Metrics.globalRegistry); private final Timer processInstanceGenerateTimer = - Timer.builder("ds.workflow.instance.generate.duration") - .description("Process instance generated duration") - .register(Metrics.globalRegistry); + Timer.builder("ds.workflow.instance.generate.duration") + .description("Process instance generated duration") + .register(Metrics.globalRegistry); public void recordCommandQueryTime(long milliseconds) { commandQueryTimer.record(milliseconds, TimeUnit.MILLISECONDS); @@ -72,14 +72,14 @@ public void recordProcessInstanceGenerateTime(long milliseconds) { public synchronized void registerProcessInstanceRunningGauge(Supplier function) { Gauge.builder("ds.workflow.instance.running", function) - .description("The current running process instance count") - .register(Metrics.globalRegistry); + .description("The current running process instance count") + .register(Metrics.globalRegistry); } public synchronized void registerProcessInstanceResubmitGauge(Supplier function) { Gauge.builder("ds.workflow.instance.resubmit", function) - .description("The current process instance need to resubmit count") - .register(Metrics.globalRegistry); + .description("The current process instance need to resubmit count") + .register(Metrics.globalRegistry); } public void incProcessInstanceByState(final String state) { diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/metrics/TaskMetrics.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/metrics/TaskMetrics.java index e1b1307b803c..9b901d7b6055 100644 --- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/metrics/TaskMetrics.java +++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/metrics/TaskMetrics.java @@ -17,17 +17,19 @@ package org.apache.dolphinscheduler.server.master.metrics; -import com.facebook.presto.jdbc.internal.guava.collect.ImmutableSet; -import io.micrometer.core.instrument.Counter; -import io.micrometer.core.instrument.Gauge; -import io.micrometer.core.instrument.Metrics; -import lombok.experimental.UtilityClass; - import java.util.HashMap; import java.util.Map; import java.util.Set; import java.util.function.Supplier; +import lombok.experimental.UtilityClass; + +import com.facebook.presto.jdbc.internal.guava.collect.ImmutableSet; + +import io.micrometer.core.instrument.Counter; +import io.micrometer.core.instrument.Gauge; +import io.micrometer.core.instrument.Metrics; + @UtilityClass public class TaskMetrics { diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/processor/CacheProcessor.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/processor/CacheProcessor.java index f9b9a119dcee..39c4776ce134 100644 --- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/processor/CacheProcessor.java +++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/processor/CacheProcessor.java @@ -32,7 +32,6 @@ import org.springframework.stereotype.Component; import com.google.common.base.Preconditions; - import io.netty.channel.Channel; /** @@ -48,7 +47,8 @@ public class CacheProcessor implements NettyRequestProcessor { @Override public void process(Channel channel, Command command) { - Preconditions.checkArgument(CommandType.CACHE_EXPIRE == command.getType(), String.format("invalid command type: %s", command.getType())); + Preconditions.checkArgument(CommandType.CACHE_EXPIRE == command.getType(), + String.format("invalid command type: %s", command.getType())); CacheExpireCommand cacheExpireCommand = JSONUtils.parseObject(command.getBody(), CacheExpireCommand.class); diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/processor/HostUpdateResponseProcessor.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/processor/HostUpdateResponseProcessor.java index 322870b2c1db..c96d4fec066a 100644 --- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/processor/HostUpdateResponseProcessor.java +++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/processor/HostUpdateResponseProcessor.java @@ -26,7 +26,6 @@ import org.slf4j.LoggerFactory; import com.google.common.base.Preconditions; - import io.netty.channel.Channel; public class HostUpdateResponseProcessor implements NettyRequestProcessor { @@ -35,9 +34,11 @@ public class HostUpdateResponseProcessor implements NettyRequestProcessor { @Override public void process(Channel channel, Command command) { - Preconditions.checkArgument(CommandType.PROCESS_HOST_UPDATE_RESPONSE == command.getType(), String.format("invalid command type : %s", command.getType())); + Preconditions.checkArgument(CommandType.PROCESS_HOST_UPDATE_RESPONSE == command.getType(), + String.format("invalid command type : %s", command.getType())); - HostUpdateResponseProcessor responseCommand = JSONUtils.parseObject(command.getBody(), HostUpdateResponseProcessor.class); + HostUpdateResponseProcessor responseCommand = + JSONUtils.parseObject(command.getBody(), HostUpdateResponseProcessor.class); logger.info("received process host response command : {}", responseCommand); } } diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/processor/StateEventProcessor.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/processor/StateEventProcessor.java index a54fc2cbac82..c9d9b6fe8ec9 100644 --- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/processor/StateEventProcessor.java +++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/processor/StateEventProcessor.java @@ -17,8 +17,6 @@ package org.apache.dolphinscheduler.server.master.processor; -import com.google.common.base.Preconditions; -import io.netty.channel.Channel; import org.apache.dolphinscheduler.common.enums.StateEventType; import org.apache.dolphinscheduler.common.enums.WorkflowExecutionStatus; import org.apache.dolphinscheduler.common.utils.JSONUtils; @@ -31,11 +29,15 @@ import org.apache.dolphinscheduler.server.master.event.WorkflowStateEvent; import org.apache.dolphinscheduler.server.master.processor.queue.StateEventResponseService; import org.apache.dolphinscheduler.service.utils.LoggerUtils; + import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; +import com.google.common.base.Preconditions; +import io.netty.channel.Channel; + /** * handle state event received from master/api */ diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/processor/TaskEventProcessor.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/processor/TaskEventProcessor.java index ce31b1a01899..43822fce5899 100644 --- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/processor/TaskEventProcessor.java +++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/processor/TaskEventProcessor.java @@ -17,8 +17,6 @@ package org.apache.dolphinscheduler.server.master.processor; -import com.google.common.base.Preconditions; -import io.netty.channel.Channel; import org.apache.dolphinscheduler.common.enums.StateEventType; import org.apache.dolphinscheduler.common.utils.JSONUtils; import org.apache.dolphinscheduler.remote.command.Command; @@ -28,11 +26,15 @@ import org.apache.dolphinscheduler.server.master.event.TaskStateEvent; import org.apache.dolphinscheduler.server.master.processor.queue.StateEventResponseService; import org.apache.dolphinscheduler.service.utils.LoggerUtils; + import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; +import com.google.common.base.Preconditions; +import io.netty.channel.Channel; + /** * handle state event received from master/api */ diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/processor/TaskExecuteResponseProcessor.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/processor/TaskExecuteResponseProcessor.java index 8f66b612dbb1..378562609c18 100644 --- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/processor/TaskExecuteResponseProcessor.java +++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/processor/TaskExecuteResponseProcessor.java @@ -25,13 +25,13 @@ import org.apache.dolphinscheduler.server.master.processor.queue.TaskEvent; import org.apache.dolphinscheduler.server.master.processor.queue.TaskEventService; import org.apache.dolphinscheduler.service.utils.LoggerUtils; + import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; import com.google.common.base.Preconditions; - import io.netty.channel.Channel; /** @@ -55,16 +55,16 @@ public class TaskExecuteResponseProcessor implements NettyRequestProcessor { @Override public void process(Channel channel, Command command) { Preconditions.checkArgument(CommandType.TASK_EXECUTE_RESULT == command.getType(), - String.format("invalid command type : %s", command.getType())); + String.format("invalid command type : %s", command.getType())); TaskExecuteResultCommand taskExecuteResultMessage = JSONUtils.parseObject(command.getBody(), - TaskExecuteResultCommand.class); + TaskExecuteResultCommand.class); TaskEvent taskResultEvent = TaskEvent.newResultEvent(taskExecuteResultMessage, - channel, - taskExecuteResultMessage.getMessageSenderAddress()); + channel, + taskExecuteResultMessage.getMessageSenderAddress()); try { LoggerUtils.setWorkflowAndTaskInstanceIDMDC(taskResultEvent.getProcessInstanceId(), - taskResultEvent.getTaskInstanceId()); + taskResultEvent.getTaskInstanceId()); logger.info("Received task execute result, event: {}", taskResultEvent); taskEventService.addEvent(taskResultEvent); diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/processor/TaskExecuteRunningProcessor.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/processor/TaskExecuteRunningProcessor.java index 47a66a71f029..2a0d2650334f 100644 --- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/processor/TaskExecuteRunningProcessor.java +++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/processor/TaskExecuteRunningProcessor.java @@ -31,7 +31,6 @@ import org.springframework.stereotype.Component; import com.google.common.base.Preconditions; - import io.netty.channel.Channel; /** @@ -53,13 +52,15 @@ public class TaskExecuteRunningProcessor implements NettyRequestProcessor { */ @Override public void process(Channel channel, Command command) { - Preconditions.checkArgument(CommandType.TASK_EXECUTE_RUNNING == command.getType(), String.format("invalid command type : %s", command.getType())); - TaskExecuteRunningCommand taskExecuteRunningMessage = JSONUtils.parseObject(command.getBody(), TaskExecuteRunningCommand.class); + Preconditions.checkArgument(CommandType.TASK_EXECUTE_RUNNING == command.getType(), + String.format("invalid command type : %s", command.getType())); + TaskExecuteRunningCommand taskExecuteRunningMessage = + JSONUtils.parseObject(command.getBody(), TaskExecuteRunningCommand.class); logger.info("taskExecuteRunningCommand: {}", taskExecuteRunningMessage); TaskEvent taskEvent = TaskEvent.newRunningEvent(taskExecuteRunningMessage, - channel, - taskExecuteRunningMessage.getMessageSenderAddress()); + channel, + taskExecuteRunningMessage.getMessageSenderAddress()); taskEventService.addEvent(taskEvent); } diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/processor/TaskExecuteStartProcessor.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/processor/TaskExecuteStartProcessor.java index 2f395fd5b0fd..c1520b07df1c 100644 --- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/processor/TaskExecuteStartProcessor.java +++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/processor/TaskExecuteStartProcessor.java @@ -23,7 +23,6 @@ import org.apache.dolphinscheduler.remote.command.CommandType; import org.apache.dolphinscheduler.remote.command.TaskExecuteStartCommand; import org.apache.dolphinscheduler.remote.processor.NettyRequestProcessor; -import org.apache.dolphinscheduler.server.master.cache.StreamTaskInstanceExecCacheManager; import org.apache.dolphinscheduler.server.master.runner.StreamTaskExecuteRunnable; import org.apache.dolphinscheduler.server.master.runner.StreamTaskExecuteThreadPool; import org.apache.dolphinscheduler.service.process.ProcessService; @@ -34,7 +33,6 @@ import org.springframework.stereotype.Component; import com.google.common.base.Preconditions; - import io.netty.channel.Channel; /** @@ -53,13 +51,18 @@ public class TaskExecuteStartProcessor implements NettyRequestProcessor { @Override public void process(Channel channel, Command command) { - Preconditions.checkArgument(CommandType.TASK_EXECUTE_START == command.getType(), String.format("invalid command type : %s", command.getType())); - TaskExecuteStartCommand taskExecuteStartCommand = JSONUtils.parseObject(command.getBody(), TaskExecuteStartCommand.class); + Preconditions.checkArgument(CommandType.TASK_EXECUTE_START == command.getType(), + String.format("invalid command type : %s", command.getType())); + TaskExecuteStartCommand taskExecuteStartCommand = + JSONUtils.parseObject(command.getBody(), TaskExecuteStartCommand.class); logger.info("taskExecuteStartCommand: {}", taskExecuteStartCommand); - TaskDefinition taskDefinition = processService.findTaskDefinition(taskExecuteStartCommand.getTaskDefinitionCode(), taskExecuteStartCommand.getTaskDefinitionVersion()); + TaskDefinition taskDefinition = processService.findTaskDefinition( + taskExecuteStartCommand.getTaskDefinitionCode(), taskExecuteStartCommand.getTaskDefinitionVersion()); if (taskDefinition == null) { - logger.error("Task definition can not be found, taskDefinitionCode:{}, taskDefinitionVersion:{}", taskExecuteStartCommand.getTaskDefinitionCode(), taskExecuteStartCommand.getTaskDefinitionVersion()); + logger.error("Task definition can not be found, taskDefinitionCode:{}, taskDefinitionVersion:{}", + taskExecuteStartCommand.getTaskDefinitionCode(), + taskExecuteStartCommand.getTaskDefinitionVersion()); return; } streamTaskExecuteThreadPool.execute(new StreamTaskExecuteRunnable(taskDefinition, taskExecuteStartCommand)); diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/processor/TaskKillResponseProcessor.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/processor/TaskKillResponseProcessor.java index 6079329b9084..1ff8cd29663d 100644 --- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/processor/TaskKillResponseProcessor.java +++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/processor/TaskKillResponseProcessor.java @@ -28,7 +28,6 @@ import org.springframework.stereotype.Component; import com.google.common.base.Preconditions; - import io.netty.channel.Channel; /** @@ -48,11 +47,13 @@ public class TaskKillResponseProcessor implements NettyRequestProcessor { */ @Override public void process(Channel channel, Command command) { - Preconditions.checkArgument(CommandType.TASK_KILL_RESPONSE == command.getType(), String.format("invalid command type : %s", command.getType())); + Preconditions.checkArgument(CommandType.TASK_KILL_RESPONSE == command.getType(), + String.format("invalid command type : %s", command.getType())); - TaskKillResponseCommand responseCommand = JSONUtils.parseObject(command.getBody(), TaskKillResponseCommand.class); + TaskKillResponseCommand responseCommand = + JSONUtils.parseObject(command.getBody(), TaskKillResponseCommand.class); logger.info("[TaskInstance-{}] Received task kill response command : {}", - responseCommand.getTaskInstanceId(), responseCommand); + responseCommand.getTaskInstanceId(), responseCommand); } } diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/processor/TaskRecallProcessor.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/processor/TaskRecallProcessor.java index 5ed1a9262a8d..b023d4ded689 100644 --- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/processor/TaskRecallProcessor.java +++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/processor/TaskRecallProcessor.java @@ -25,13 +25,13 @@ import org.apache.dolphinscheduler.server.master.processor.queue.TaskEvent; import org.apache.dolphinscheduler.server.master.processor.queue.TaskEventService; import org.apache.dolphinscheduler.service.utils.LoggerUtils; + import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; import com.google.common.base.Preconditions; - import io.netty.channel.Channel; /** @@ -53,11 +53,13 @@ public class TaskRecallProcessor implements NettyRequestProcessor { */ @Override public void process(Channel channel, Command command) { - Preconditions.checkArgument(CommandType.TASK_REJECT == command.getType(), String.format("invalid command type : %s", command.getType())); + Preconditions.checkArgument(CommandType.TASK_REJECT == command.getType(), + String.format("invalid command type : %s", command.getType())); TaskRejectCommand recallCommand = JSONUtils.parseObject(command.getBody(), TaskRejectCommand.class); TaskEvent taskEvent = TaskEvent.newRecallEvent(recallCommand, channel); try { - LoggerUtils.setWorkflowAndTaskInstanceIDMDC(recallCommand.getProcessInstanceId(), recallCommand.getTaskInstanceId()); + LoggerUtils.setWorkflowAndTaskInstanceIDMDC(recallCommand.getProcessInstanceId(), + recallCommand.getTaskInstanceId()); logger.info("Receive task recall command: {}", recallCommand); taskEventService.addEvent(taskEvent); } finally { diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/processor/WorkflowExecutingDataRequestProcessor.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/processor/WorkflowExecutingDataRequestProcessor.java index c8f70d96d0f8..1408b8d42fa9 100644 --- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/processor/WorkflowExecutingDataRequestProcessor.java +++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/processor/WorkflowExecutingDataRequestProcessor.java @@ -34,7 +34,6 @@ import org.springframework.stereotype.Component; import com.google.common.base.Preconditions; - import io.netty.channel.Channel; /** @@ -50,13 +49,16 @@ public class WorkflowExecutingDataRequestProcessor implements NettyRequestProces @Override public void process(Channel channel, Command command) { - Preconditions.checkArgument(CommandType.WORKFLOW_EXECUTING_DATA_REQUEST == command.getType(), String.format("invalid command type: %s", command.getType())); + Preconditions.checkArgument(CommandType.WORKFLOW_EXECUTING_DATA_REQUEST == command.getType(), + String.format("invalid command type: %s", command.getType())); - WorkflowExecutingDataRequestCommand requestCommand = JSONUtils.parseObject(command.getBody(), WorkflowExecutingDataRequestCommand.class); + WorkflowExecutingDataRequestCommand requestCommand = + JSONUtils.parseObject(command.getBody(), WorkflowExecutingDataRequestCommand.class); logger.info("received command, processInstanceId:{}", requestCommand.getProcessInstanceId()); - Optional workflowExecuteDtoOptional = executingService.queryWorkflowExecutingData(requestCommand.getProcessInstanceId()); + Optional workflowExecuteDtoOptional = + executingService.queryWorkflowExecutingData(requestCommand.getProcessInstanceId()); WorkflowExecutingDataResponseCommand responseCommand = new WorkflowExecutingDataResponseCommand(); workflowExecuteDtoOptional.ifPresent(responseCommand::setWorkflowExecuteDto); diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/processor/queue/StateEventResponseService.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/processor/queue/StateEventResponseService.java index 75cf8a9d0bb8..bed95868ce5c 100644 --- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/processor/queue/StateEventResponseService.java +++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/processor/queue/StateEventResponseService.java @@ -17,7 +17,6 @@ package org.apache.dolphinscheduler.server.master.processor.queue; -import io.netty.channel.Channel; import org.apache.dolphinscheduler.common.lifecycle.ServerLifeCycleManager; import org.apache.dolphinscheduler.common.thread.BaseDaemonThread; import org.apache.dolphinscheduler.remote.command.StateEventResponseCommand; @@ -26,18 +25,22 @@ import org.apache.dolphinscheduler.server.master.runner.WorkflowExecuteRunnable; import org.apache.dolphinscheduler.server.master.runner.WorkflowExecuteThreadPool; import org.apache.dolphinscheduler.service.utils.LoggerUtils; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.stereotype.Component; -import javax.annotation.PostConstruct; -import javax.annotation.PreDestroy; import java.util.ArrayList; import java.util.List; import java.util.concurrent.BlockingQueue; import java.util.concurrent.LinkedBlockingQueue; +import javax.annotation.PostConstruct; +import javax.annotation.PreDestroy; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; + +import io.netty.channel.Channel; + @Component public class StateEventResponseService { diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/processor/queue/TaskEventService.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/processor/queue/TaskEventService.java index 878b8bc6d3a7..f7b9b033f51a 100644 --- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/processor/queue/TaskEventService.java +++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/processor/queue/TaskEventService.java @@ -20,19 +20,21 @@ import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.lifecycle.ServerLifeCycleManager; import org.apache.dolphinscheduler.common.thread.BaseDaemonThread; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.stereotype.Component; -import javax.annotation.PostConstruct; -import javax.annotation.PreDestroy; import java.util.ArrayList; import java.util.List; import java.util.concurrent.BlockingQueue; import java.util.concurrent.LinkedBlockingQueue; import java.util.concurrent.TimeUnit; +import javax.annotation.PostConstruct; +import javax.annotation.PreDestroy; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; + /** * task manager */ @@ -152,4 +154,4 @@ public void run() { } } } -} \ No newline at end of file +} diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/processor/queue/TaskExecuteRunnable.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/processor/queue/TaskExecuteRunnable.java index cad01a325b87..0a33410f2e26 100644 --- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/processor/queue/TaskExecuteRunnable.java +++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/processor/queue/TaskExecuteRunnable.java @@ -22,6 +22,7 @@ import org.apache.dolphinscheduler.server.master.event.TaskEventHandleException; import org.apache.dolphinscheduler.server.master.event.TaskEventHandler; import org.apache.dolphinscheduler.service.utils.LoggerUtils; + import java.util.Map; import java.util.concurrent.ConcurrentLinkedQueue; @@ -60,14 +61,14 @@ public void run() { } catch (TaskEventHandleException taskEventHandleException) { // we don't need to resubmit this event, since the worker will resubmit this event logger.error("Handle task event failed, this event will be retry later, event: {}", event, - taskEventHandleException); + taskEventHandleException); } catch (TaskEventHandleError taskEventHandleError) { logger.error("Handle task event error, this event will be removed, event: {}", event, - taskEventHandleError); + taskEventHandleError); events.remove(event); } catch (Exception unknownException) { logger.error("Handle task event error, get a unknown exception, this event will be removed, event: {}", - event, unknownException); + event, unknownException); events.remove(event); } finally { LoggerUtils.removeWorkflowAndTaskInstanceIdMDC(); @@ -93,7 +94,8 @@ public Integer getProcessInstanceId() { public boolean addEvent(TaskEvent event) { if (event.getProcessInstanceId() != this.processInstanceId) { - logger.warn("event would be abounded, task instance id:{}, process instance id:{}, this.processInstanceId:{}", + logger.warn( + "event would be abounded, task instance id:{}, process instance id:{}, this.processInstanceId:{}", event.getTaskInstanceId(), event.getProcessInstanceId(), this.processInstanceId); return false; } diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/processor/queue/TaskExecuteThreadPool.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/processor/queue/TaskExecuteThreadPool.java index 80d2f12d04d8..ece37e7e40f7 100644 --- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/processor/queue/TaskExecuteThreadPool.java +++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/processor/queue/TaskExecuteThreadPool.java @@ -71,13 +71,15 @@ private void init() { this.setMaxPoolSize(masterConfig.getExecThreads()); this.setCorePoolSize(masterConfig.getExecThreads()); taskEventHandlerList.forEach( - taskEventHandler -> taskEventHandlerMap.put(taskEventHandler.getHandleEventType(), taskEventHandler)); + taskEventHandler -> taskEventHandlerMap.put(taskEventHandler.getHandleEventType(), taskEventHandler)); } public void submitTaskEvent(TaskEvent taskEvent) { // stream task event handle - if (taskEvent.getProcessInstanceId() == 0 && streamTaskInstanceExecCacheManager.contains(taskEvent.getTaskInstanceId())) { - streamTaskInstanceExecCacheManager.getByTaskInstanceId(taskEvent.getTaskInstanceId()).addTaskEvent(taskEvent); + if (taskEvent.getProcessInstanceId() == 0 + && streamTaskInstanceExecCacheManager.contains(taskEvent.getTaskInstanceId())) { + streamTaskInstanceExecCacheManager.getByTaskInstanceId(taskEvent.getTaskInstanceId()) + .addTaskEvent(taskEvent); return; } if (!processInstanceExecCacheManager.contains(taskEvent.getProcessInstanceId())) { @@ -85,7 +87,7 @@ public void submitTaskEvent(TaskEvent taskEvent) { return; } TaskExecuteRunnable taskExecuteRunnable = taskExecuteThreadMap.computeIfAbsent(taskEvent.getProcessInstanceId(), - (processInstanceId) -> new TaskExecuteRunnable(processInstanceId, taskEventHandlerMap)); + (processInstanceId) -> new TaskExecuteRunnable(processInstanceId, taskEventHandlerMap)); taskExecuteRunnable.addEvent(taskEvent); } @@ -105,14 +107,16 @@ public void executeEvent(TaskExecuteRunnable taskExecuteThread) { multiThreadFilterMap.put(taskExecuteThread.getKey(), taskExecuteThread); ListenableFuture future = this.submitListenable(taskExecuteThread::run); future.addCallback(new ListenableFutureCallback() { + @Override public void onFailure(Throwable ex) { Integer processInstanceId = taskExecuteThread.getProcessInstanceId(); logger.error("[WorkflowInstance-{}] persist event failed", processInstanceId, ex); if (!processInstanceExecCacheManager.contains(processInstanceId)) { taskExecuteThreadMap.remove(processInstanceId); - logger.info("[WorkflowInstance-{}] Cannot find processInstance from cacheManager, remove process instance from threadMap", - processInstanceId); + logger.info( + "[WorkflowInstance-{}] Cannot find processInstance from cacheManager, remove process instance from threadMap", + processInstanceId); } multiThreadFilterMap.remove(taskExecuteThread.getKey()); } @@ -123,8 +127,9 @@ public void onSuccess(Object result) { logger.info("[WorkflowInstance-{}] persist events succeeded", processInstanceId); if (!processInstanceExecCacheManager.contains(processInstanceId)) { taskExecuteThreadMap.remove(processInstanceId); - logger.info("[WorkflowInstance-{}] Cannot find processInstance from cacheManager, remove process instance from threadMap", - processInstanceId); + logger.info( + "[WorkflowInstance-{}] Cannot find processInstance from cacheManager, remove process instance from threadMap", + processInstanceId); } multiThreadFilterMap.remove(taskExecuteThread.getKey()); } diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/registry/MasterConnectionStateListener.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/registry/MasterConnectionStateListener.java index 1885d824308a..545112a699bc 100644 --- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/registry/MasterConnectionStateListener.java +++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/registry/MasterConnectionStateListener.java @@ -17,12 +17,14 @@ package org.apache.dolphinscheduler.server.master.registry; -import lombok.NonNull; import org.apache.dolphinscheduler.common.lifecycle.ServerLifeCycleManager; import org.apache.dolphinscheduler.registry.api.ConnectionListener; import org.apache.dolphinscheduler.registry.api.ConnectionState; import org.apache.dolphinscheduler.server.master.config.MasterConfig; import org.apache.dolphinscheduler.service.registry.RegistryClient; + +import lombok.NonNull; + import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/registry/MasterRegistryClient.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/registry/MasterRegistryClient.java index ac8d60e20e98..fd835c1234fa 100644 --- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/registry/MasterRegistryClient.java +++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/registry/MasterRegistryClient.java @@ -17,7 +17,9 @@ package org.apache.dolphinscheduler.server.master.registry; -import org.apache.commons.lang3.StringUtils; +import static org.apache.dolphinscheduler.common.Constants.REGISTRY_DOLPHINSCHEDULER_NODE; +import static org.apache.dolphinscheduler.common.Constants.SLEEP_TIME_MILLIS; + import org.apache.dolphinscheduler.common.IStoppable; import org.apache.dolphinscheduler.common.enums.NodeType; import org.apache.dolphinscheduler.common.thread.ThreadUtils; @@ -28,14 +30,14 @@ import org.apache.dolphinscheduler.server.master.service.FailoverService; import org.apache.dolphinscheduler.server.master.task.MasterHeartBeatTask; import org.apache.dolphinscheduler.service.registry.RegistryClient; + +import org.apache.commons.lang3.StringUtils; + import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; -import static org.apache.dolphinscheduler.common.Constants.REGISTRY_DOLPHINSCHEDULER_NODE; -import static org.apache.dolphinscheduler.common.Constants.SLEEP_TIME_MILLIS; - /** *

DolphinScheduler master register client, used to connect to registry and hand the registry events. *

When the Master node startup, it will register in registry center. And start a {@link MasterHeartBeatTask} to update its metadata in registry. diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/registry/MasterRegistryDataListener.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/registry/MasterRegistryDataListener.java index 1eafb4cb54fd..c6f641bcfd24 100644 --- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/registry/MasterRegistryDataListener.java +++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/registry/MasterRegistryDataListener.java @@ -47,11 +47,11 @@ public void notify(Event event) { if (Strings.isNullOrEmpty(path)) { return; } - //monitor master + // monitor master if (path.startsWith(REGISTRY_DOLPHINSCHEDULER_MASTERS + Constants.SINGLE_SLASH)) { handleMasterEvent(event); } else if (path.startsWith(REGISTRY_DOLPHINSCHEDULER_WORKERS + Constants.SINGLE_SLASH)) { - //monitor worker + // monitor worker handleWorkerEvent(event); } } diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/registry/MasterStopStrategy.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/registry/MasterStopStrategy.java index 1b1f2c84a8f1..da2aff13772b 100644 --- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/registry/MasterStopStrategy.java +++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/registry/MasterStopStrategy.java @@ -20,6 +20,7 @@ import org.apache.dolphinscheduler.registry.api.StrategyType; import org.apache.dolphinscheduler.server.master.config.MasterConfig; import org.apache.dolphinscheduler.service.registry.RegistryClient; + import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/registry/MasterWaitingStrategy.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/registry/MasterWaitingStrategy.java index 654d96f25b68..4f88ce478780 100644 --- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/registry/MasterWaitingStrategy.java +++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/registry/MasterWaitingStrategy.java @@ -29,14 +29,15 @@ import org.apache.dolphinscheduler.server.master.rpc.MasterRPCServer; import org.apache.dolphinscheduler.server.master.runner.StateWheelExecuteThread; import org.apache.dolphinscheduler.service.registry.RegistryClient; + +import java.time.Duration; + import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.stereotype.Service; -import java.time.Duration; - /** * This strategy will change the server status to {@link ServerStatus#WAITING} when disconnect from {@link Registry}. */ diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/EventExecuteService.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/EventExecuteService.java index e1c87de8f302..0d8d4d281556 100644 --- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/EventExecuteService.java +++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/EventExecuteService.java @@ -23,6 +23,7 @@ import org.apache.dolphinscheduler.server.master.cache.ProcessInstanceExecCacheManager; import org.apache.dolphinscheduler.server.master.cache.StreamTaskInstanceExecCacheManager; import org.apache.dolphinscheduler.service.utils.LoggerUtils; + import java.util.concurrent.TimeUnit; import org.slf4j.Logger; diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/FailoverExecuteThread.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/FailoverExecuteThread.java index 5546b474d774..7a488367b025 100644 --- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/FailoverExecuteThread.java +++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/FailoverExecuteThread.java @@ -23,6 +23,7 @@ import org.apache.dolphinscheduler.common.thread.ThreadUtils; import org.apache.dolphinscheduler.server.master.config.MasterConfig; import org.apache.dolphinscheduler.server.master.service.MasterFailoverService; + import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterSchedulerBootstrap.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterSchedulerBootstrap.java index 89bc42e61fb3..377dff719130 100644 --- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterSchedulerBootstrap.java +++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterSchedulerBootstrap.java @@ -41,6 +41,7 @@ import org.apache.dolphinscheduler.service.expand.CuringParamsService; import org.apache.dolphinscheduler.service.process.ProcessService; import org.apache.dolphinscheduler.service.utils.LoggerUtils; + import org.apache.commons.collections4.CollectionUtils; import java.util.ArrayList; @@ -135,7 +136,8 @@ public void run() { try { if (!ServerLifeCycleManager.isRunning()) { // the current server is not at running status, cannot consume command. - logger.warn("The current server {} is not at running status, cannot consumes commands.", this.masterAddress); + logger.warn("The current server {} is not at running status, cannot consumes commands.", + this.masterAddress); Thread.sleep(Constants.SLEEP_TIME_MILLIS); } // todo: if the workflow event queue is much, we need to handle the back pressure diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/StateWheelExecuteThread.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/StateWheelExecuteThread.java index 28d31f060a7e..1a428ee21641 100644 --- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/StateWheelExecuteThread.java +++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/StateWheelExecuteThread.java @@ -17,12 +17,11 @@ package org.apache.dolphinscheduler.server.master.runner; -import lombok.NonNull; import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.enums.StateEventType; import org.apache.dolphinscheduler.common.enums.TimeoutFlag; -import org.apache.dolphinscheduler.common.lifecycle.ServerLifeCycleManager; import org.apache.dolphinscheduler.common.enums.WorkflowExecutionStatus; +import org.apache.dolphinscheduler.common.lifecycle.ServerLifeCycleManager; import org.apache.dolphinscheduler.common.thread.BaseDaemonThread; import org.apache.dolphinscheduler.common.utils.DateUtils; import org.apache.dolphinscheduler.dao.entity.ProcessInstance; @@ -35,16 +34,20 @@ import org.apache.dolphinscheduler.server.master.event.WorkflowStateEvent; import org.apache.dolphinscheduler.server.master.runner.task.TaskInstanceKey; import org.apache.dolphinscheduler.service.utils.LoggerUtils; + +import java.util.Optional; +import java.util.concurrent.ConcurrentLinkedQueue; + +import javax.annotation.PostConstruct; + +import lombok.NonNull; + import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.context.annotation.Lazy; import org.springframework.stereotype.Component; -import javax.annotation.PostConstruct; -import java.util.Optional; -import java.util.concurrent.ConcurrentLinkedQueue; - /** * Check thread * 1. timeout task check diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/StreamTaskExecuteThreadPool.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/StreamTaskExecuteThreadPool.java index 23d4ca85211e..29977317c422 100644 --- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/StreamTaskExecuteThreadPool.java +++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/StreamTaskExecuteThreadPool.java @@ -17,10 +17,9 @@ package org.apache.dolphinscheduler.server.master.runner; -import org.apache.dolphinscheduler.remote.processor.StateEventCallbackService; import org.apache.dolphinscheduler.server.master.config.MasterConfig; -import org.apache.dolphinscheduler.service.process.ProcessService; import org.apache.dolphinscheduler.service.utils.LoggerUtils; + import javax.annotation.PostConstruct; import org.slf4j.Logger; @@ -60,6 +59,7 @@ public void executeEvent(final StreamTaskExecuteRunnable streamTaskExecuteRunnab int taskInstanceId = streamTaskExecuteRunnable.getTaskInstance().getId(); ListenableFuture future = this.submitListenable(streamTaskExecuteRunnable::handleEvents); future.addCallback(new ListenableFutureCallback() { + @Override public void onFailure(Throwable ex) { LoggerUtils.setTaskInstanceIdMDC(taskInstanceId); diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/WorkflowEventLooper.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/WorkflowEventLooper.java index 1842885348fb..9a422970b45c 100644 --- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/WorkflowEventLooper.java +++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/WorkflowEventLooper.java @@ -28,16 +28,18 @@ import org.apache.dolphinscheduler.server.master.event.WorkflowEventQueue; import org.apache.dolphinscheduler.server.master.event.WorkflowEventType; import org.apache.dolphinscheduler.service.utils.LoggerUtils; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.stereotype.Component; -import javax.annotation.PostConstruct; import java.util.HashMap; import java.util.List; import java.util.Map; +import javax.annotation.PostConstruct; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; + @Component public class WorkflowEventLooper extends BaseDaemonThread { diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/WorkflowExecuteRunnable.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/WorkflowExecuteRunnable.java index 403135f3bd48..1a896bf9c5e1 100644 --- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/WorkflowExecuteRunnable.java +++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/WorkflowExecuteRunnable.java @@ -85,6 +85,7 @@ import org.apache.dolphinscheduler.service.queue.PeerTaskInstancePriorityQueue; import org.apache.dolphinscheduler.service.utils.DagHelper; import org.apache.dolphinscheduler.service.utils.LoggerUtils; + import org.apache.commons.collections.CollectionUtils; import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.math.NumberUtils; @@ -816,7 +817,8 @@ private void initTaskQueue() throws StateEventHandleException, CronParseExceptio task.getTaskCode(), task.getState()); if (validTaskMap.containsKey(task.getTaskCode())) { - logger.warn("Have same taskCode taskInstance when init task queue, need to check taskExecutionStatus, taskCode:{}", + logger.warn( + "Have same taskCode taskInstance when init task queue, need to check taskExecutionStatus, taskCode:{}", task.getTaskCode()); int oldTaskInstanceId = validTaskMap.get(task.getTaskCode()); TaskInstance oldTaskInstance = taskInstanceMap.get(oldTaskInstanceId); @@ -965,7 +967,8 @@ private Optional submitTaskExec(TaskInstance taskInstance) { taskInstance.getProcessInstanceId(), taskInstance.getTaskGroupPriority()); if (!acquireTaskGroup) { - logger.info("Submitted task will not be dispatch right now because the first time to try to acquire" + + logger.info( + "Submitted task will not be dispatch right now because the first time to try to acquire" + " task group failed, taskInstanceName: {}, taskGroupId: {}", taskInstance.getName(), taskGroupId); return Optional.of(taskInstance); @@ -974,7 +977,8 @@ private Optional submitTaskExec(TaskInstance taskInstance) { boolean dispatchSuccess = taskProcessor.action(TaskAction.DISPATCH); if (!dispatchSuccess) { - logger.error("Dispatch standby process {} task {} failed", processInstance.getName(), taskInstance.getName()); + logger.error("Dispatch standby process {} task {} failed", processInstance.getName(), + taskInstance.getName()); return Optional.empty(); } taskProcessor.action(TaskAction.RUN); @@ -1431,9 +1435,10 @@ private boolean dependTaskSuccess(String dependNodeName, String nextNodeName) { List nextTaskList = DagHelper.parseConditionTask(dependNodeName, skipTaskNodeMap, dag, getCompleteTaskInstanceMap()); if (!nextTaskList.contains(nextNodeName)) { - logger.info("DependTask is a condition task, and its next condition branch does not hava current task, " + - "dependTaskCode: {}, currentTaskCode: {}", dependNodeName, nextNodeName - ); + logger.info( + "DependTask is a condition task, and its next condition branch does not hava current task, " + + "dependTaskCode: {}, currentTaskCode: {}", + dependNodeName, nextNodeName); return false; } } else { @@ -1804,7 +1809,8 @@ public void submitStandByTask() throws StateEventHandleException { TaskInstance retryTask = processService.findTaskInstanceById(task.getId()); if (retryTask != null && retryTask.getState().isForceSuccess()) { task.setState(retryTask.getState()); - logger.info("Task {} has been forced success, put it into complete task list and stop retrying, taskInstanceId: {}", + logger.info( + "Task {} has been forced success, put it into complete task list and stop retrying, taskInstanceId: {}", task.getName(), task.getId()); removeTaskFromStandbyList(task); completeTaskMap.put(task.getTaskCode(), task.getId()); diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/WorkflowExecuteThreadPool.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/WorkflowExecuteThreadPool.java index 388637a567a5..20fe003fc376 100644 --- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/WorkflowExecuteThreadPool.java +++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/WorkflowExecuteThreadPool.java @@ -17,8 +17,6 @@ package org.apache.dolphinscheduler.server.master.runner; -import com.google.common.base.Strings; -import lombok.NonNull; import org.apache.dolphinscheduler.common.enums.Flag; import org.apache.dolphinscheduler.common.enums.StateEventType; import org.apache.dolphinscheduler.common.utils.NetUtils; @@ -34,6 +32,14 @@ import org.apache.dolphinscheduler.server.master.event.TaskStateEvent; import org.apache.dolphinscheduler.service.process.ProcessService; import org.apache.dolphinscheduler.service.utils.LoggerUtils; + +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; + +import javax.annotation.PostConstruct; + +import lombok.NonNull; + import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; @@ -42,9 +48,7 @@ import org.springframework.util.concurrent.ListenableFuture; import org.springframework.util.concurrent.ListenableFutureCallback; -import javax.annotation.PostConstruct; -import java.util.Map; -import java.util.concurrent.ConcurrentHashMap; +import com.google.common.base.Strings; /** * Used to execute {@link WorkflowExecuteRunnable}. diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/task/CommonTaskProcessor.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/task/CommonTaskProcessor.java index fc4c4c337d8a..170f0ac4821d 100644 --- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/task/CommonTaskProcessor.java +++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/task/CommonTaskProcessor.java @@ -17,7 +17,6 @@ package org.apache.dolphinscheduler.server.master.runner.task; -import com.fasterxml.jackson.core.type.TypeReference; import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.utils.JSONUtils; import org.apache.dolphinscheduler.plugin.task.api.TaskConstants; @@ -39,6 +38,7 @@ import java.util.Date; import java.util.Map; +import com.fasterxml.jackson.core.type.TypeReference; import com.google.auto.service.AutoService; /** @@ -183,13 +183,15 @@ private void killRemoteTask() throws ExecuteException { } protected void convertExeEnvironmentOnlineToTest() { - //SQL taskType + // SQL taskType if (TaskConstants.TASK_TYPE_SQL.equals(taskInstance.getTaskType())) { - //replace test data source - Map taskDefinitionParams = JSONUtils.parseObject(taskInstance.getTaskDefine().getTaskParams(), new TypeReference>() { - }); - Map taskInstanceParams = JSONUtils.parseObject(taskInstance.getTaskParams(), new TypeReference>() { - }); + // replace test data source + Map taskDefinitionParams = JSONUtils.parseObject( + taskInstance.getTaskDefine().getTaskParams(), new TypeReference>() { + }); + Map taskInstanceParams = + JSONUtils.parseObject(taskInstance.getTaskParams(), new TypeReference>() { + }); Integer onlineDataSourceId = (Integer) taskDefinitionParams.get(Constants.DATASOUCE); Integer testDataSourceId = processService.queryTestDataSourceId(onlineDataSourceId); taskDefinitionParams.put(Constants.DATASOUCE, testDataSourceId); diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/task/TaskInstanceKey.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/task/TaskInstanceKey.java index d9ca82a45cbf..8bfbde68489d 100644 --- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/task/TaskInstanceKey.java +++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/task/TaskInstanceKey.java @@ -30,12 +30,15 @@ @Data @AllArgsConstructor public class TaskInstanceKey { + private final int processInstanceId; private final long taskCode; private final int taskVersion; - public static TaskInstanceKey getTaskInstanceKey(@NonNull ProcessInstance processInstance, @NonNull TaskInstance taskInstance) { - return new TaskInstanceKey(processInstance.getId(), taskInstance.getTaskCode(), taskInstance.getTaskDefinitionVersion()); + public static TaskInstanceKey getTaskInstanceKey(@NonNull ProcessInstance processInstance, + @NonNull TaskInstance taskInstance) { + return new TaskInstanceKey(processInstance.getId(), taskInstance.getTaskCode(), + taskInstance.getTaskDefinitionVersion()); } } diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/task/TaskProcessorFactory.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/task/TaskProcessorFactory.java index f585c94bbfe7..793cfacb0e73 100644 --- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/task/TaskProcessorFactory.java +++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/task/TaskProcessorFactory.java @@ -17,18 +17,21 @@ package org.apache.dolphinscheduler.server.master.runner.task; -import lombok.experimental.UtilityClass; -import org.apache.commons.lang3.StringUtils; +import static org.apache.dolphinscheduler.common.Constants.COMMON_TASK_TYPE; + import org.apache.dolphinscheduler.spi.plugin.PrioritySPIFactory; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; + +import org.apache.commons.lang3.StringUtils; import java.lang.reflect.Constructor; import java.lang.reflect.InvocationTargetException; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; -import static org.apache.dolphinscheduler.common.Constants.COMMON_TASK_TYPE; +import lombok.experimental.UtilityClass; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; /** * the factory to create task processor @@ -47,10 +50,12 @@ public final class TaskProcessorFactory { for (Map.Entry entry : prioritySPIFactory.getSPIMap().entrySet()) { try { logger.info("Registering task processor: {} - {}", entry.getKey(), entry.getValue().getClass()); - PROCESS_MAP.put(entry.getKey(), (Constructor) entry.getValue().getClass().getConstructor()); + PROCESS_MAP.put(entry.getKey(), + (Constructor) entry.getValue().getClass().getConstructor()); logger.info("Registered task processor: {} - {}", entry.getKey(), entry.getValue().getClass()); } catch (NoSuchMethodException e) { - throw new IllegalArgumentException(String.format("The task processor: %s should has a no args constructor", entry.getKey())); + throw new IllegalArgumentException( + String.format("The task processor: %s should has a no args constructor", entry.getKey())); } } } diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/service/ExecutingService.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/service/ExecutingService.java index fa85b66f828d..90be9f6d3ced 100644 --- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/service/ExecutingService.java +++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/service/ExecutingService.java @@ -49,7 +49,8 @@ public class ExecutingService { private ProcessInstanceExecCacheManager processInstanceExecCacheManager; public Optional queryWorkflowExecutingData(Integer processInstanceId) { - WorkflowExecuteRunnable workflowExecuteRunnable = processInstanceExecCacheManager.getByProcessInstanceId(processInstanceId); + WorkflowExecuteRunnable workflowExecuteRunnable = + processInstanceExecCacheManager.getByProcessInstanceId(processInstanceId); if (workflowExecuteRunnable == null) { logger.info("workflow execute data not found, maybe it has finished, workflow id:{}", processInstanceId); return Optional.empty(); diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/service/FailoverService.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/service/FailoverService.java index 05a54e8529c1..01d9e1968ab5 100644 --- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/service/FailoverService.java +++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/service/FailoverService.java @@ -19,17 +19,18 @@ import org.apache.dolphinscheduler.common.enums.NodeType; +import lombok.NonNull; + import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.stereotype.Component; -import lombok.NonNull; - /** * failover service */ @Component public class FailoverService { + private static final Logger LOGGER = LoggerFactory.getLogger(FailoverService.class); private final MasterFailoverService masterFailoverService; diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/task/MasterHeartBeatTask.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/task/MasterHeartBeatTask.java index a4c89b210a6d..45bf0d74cc14 100644 --- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/task/MasterHeartBeatTask.java +++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/task/MasterHeartBeatTask.java @@ -17,8 +17,6 @@ package org.apache.dolphinscheduler.server.master.task; -import lombok.NonNull; -import lombok.extern.slf4j.Slf4j; import org.apache.dolphinscheduler.common.lifecycle.ServerLifeCycleManager; import org.apache.dolphinscheduler.common.model.BaseHeartBeatTask; import org.apache.dolphinscheduler.common.model.MasterHeartBeat; @@ -27,6 +25,9 @@ import org.apache.dolphinscheduler.server.master.config.MasterConfig; import org.apache.dolphinscheduler.service.registry.RegistryClient; +import lombok.NonNull; +import lombok.extern.slf4j.Slf4j; + @Slf4j public class MasterHeartBeatTask extends BaseHeartBeatTask { diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/utils/DependentExecute.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/utils/DependentExecute.java index 69c2318e0830..e318e4094ac8 100644 --- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/utils/DependentExecute.java +++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/utils/DependentExecute.java @@ -153,7 +153,8 @@ private DependResult dependResultByProcessInstance(ProcessInstance processInstan private DependResult getDependTaskResult(long taskCode, ProcessInstance processInstance, int testFlag) { DependResult result; TaskInstance taskInstance = null; - List taskInstanceList = processService.findValidTaskListByProcessId(processInstance.getId(), testFlag); + List taskInstanceList = + processService.findValidTaskListByProcessId(processInstance.getId(), testFlag); for (TaskInstance task : taskInstanceList) { if (task.getTaskCode() == taskCode) { @@ -191,7 +192,8 @@ private ProcessInstance findLastProcessInterval(Long definitionCode, DateInterva ProcessInstance lastSchedulerProcess = processService.findLastSchedulerProcessInterval(definitionCode, dateInterval, testFlag); - ProcessInstance lastManualProcess = processService.findLastManualProcessInterval(definitionCode, dateInterval, testFlag); + ProcessInstance lastManualProcess = + processService.findLastManualProcessInterval(definitionCode, dateInterval, testFlag); if (lastManualProcess == null) { return lastSchedulerProcess; diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/utils/SwitchTaskUtils.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/utils/SwitchTaskUtils.java index d23ebacf4c05..f63f99d5c6f8 100644 --- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/utils/SwitchTaskUtils.java +++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/utils/SwitchTaskUtils.java @@ -22,6 +22,7 @@ import javax.script.ScriptException; public class SwitchTaskUtils { + private static ScriptEngineManager manager; private static ScriptEngine engine; @@ -35,4 +36,4 @@ public static boolean evaluate(String expression) throws ScriptException { return (Boolean) result; } -} \ No newline at end of file +} diff --git a/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/ConditionsTaskTest.java b/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/ConditionsTaskTest.java index d6a10f97d3b1..035a811f761c 100644 --- a/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/ConditionsTaskTest.java +++ b/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/ConditionsTaskTest.java @@ -34,6 +34,12 @@ import org.apache.dolphinscheduler.service.bean.SpringApplicationContext; import org.apache.dolphinscheduler.service.model.TaskNode; import org.apache.dolphinscheduler.service.process.ProcessService; + +import java.time.Duration; +import java.util.List; +import java.util.stream.Collectors; +import java.util.stream.Stream; + import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.extension.ExtendWith; import org.mockito.Mockito; @@ -42,11 +48,6 @@ import org.mockito.quality.Strictness; import org.springframework.context.ApplicationContext; -import java.time.Duration; -import java.util.List; -import java.util.stream.Collectors; -import java.util.stream.Stream; - @ExtendWith(MockitoExtension.class) @MockitoSettings(strictness = Strictness.LENIENT) public class ConditionsTaskTest { diff --git a/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/DependentTaskTest.java b/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/DependentTaskTest.java index e2b14714a69a..31d73ed12a61 100644 --- a/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/DependentTaskTest.java +++ b/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/DependentTaskTest.java @@ -35,6 +35,11 @@ import org.apache.dolphinscheduler.service.bean.SpringApplicationContext; import org.apache.dolphinscheduler.service.model.TaskNode; import org.apache.dolphinscheduler.service.process.ProcessService; + +import java.time.Duration; +import java.util.stream.Collectors; +import java.util.stream.Stream; + import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; @@ -44,10 +49,6 @@ import org.mockito.quality.Strictness; import org.springframework.context.ApplicationContext; -import java.time.Duration; -import java.util.stream.Collectors; -import java.util.stream.Stream; - /** * DependentTaskTest */ diff --git a/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/ParamsTest.java b/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/ParamsTest.java index 88518281cb83..ab4e1f9626c5 100644 --- a/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/ParamsTest.java +++ b/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/ParamsTest.java @@ -18,8 +18,8 @@ package org.apache.dolphinscheduler.server.master; import org.apache.dolphinscheduler.common.enums.CommandType; -import org.apache.dolphinscheduler.plugin.task.api.utils.ParameterUtils; import org.apache.dolphinscheduler.common.utils.placeholder.BusinessTimeUtils; +import org.apache.dolphinscheduler.plugin.task.api.utils.ParameterUtils; import java.util.Calendar; import java.util.Date; @@ -34,20 +34,20 @@ */ public class ParamsTest { - private static final Logger logger = LoggerFactory.getLogger(ParamsTest.class); + private static final Logger logger = LoggerFactory.getLogger(ParamsTest.class); @Test public void systemParamsTest() throws Exception { String command = "${system.biz.date}"; // start process - Map timeParams = BusinessTimeUtils + Map timeParams = BusinessTimeUtils .getBusinessTime(CommandType.START_PROCESS, new Date(), null); command = ParameterUtils.convertParameterPlaceholders(command, timeParams); - logger.info("start process : {}",command); + logger.info("start process : {}", command); Calendar calendar = Calendar.getInstance(); calendar.setTime(new Date()); @@ -59,7 +59,7 @@ public void systemParamsTest() throws Exception { .getBusinessTime(CommandType.COMPLEMENT_DATA, calendar.getTime(), null); command = ParameterUtils.convertParameterPlaceholders(command, timeParams); - logger.info("complement data : {}",command); + logger.info("complement data : {}", command); } } diff --git a/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/SwitchTaskTest.java b/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/SwitchTaskTest.java index 902ad406b71e..31e55c0af035 100644 --- a/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/SwitchTaskTest.java +++ b/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/SwitchTaskTest.java @@ -30,6 +30,13 @@ import org.apache.dolphinscheduler.server.master.config.MasterConfig; import org.apache.dolphinscheduler.service.bean.SpringApplicationContext; import org.apache.dolphinscheduler.service.process.ProcessService; + +import java.time.Duration; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.extension.ExtendWith; import org.mockito.Mockito; @@ -38,12 +45,6 @@ import org.mockito.quality.Strictness; import org.springframework.context.ApplicationContext; -import java.time.Duration; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - @ExtendWith(MockitoExtension.class) @MockitoSettings(strictness = Strictness.LENIENT) public class SwitchTaskTest { diff --git a/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/cache/impl/ProcessInstanceExecCacheManagerImplTest.java b/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/cache/impl/ProcessInstanceExecCacheManagerImplTest.java index 1d33316eb2e4..55b54fd75a1d 100644 --- a/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/cache/impl/ProcessInstanceExecCacheManagerImplTest.java +++ b/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/cache/impl/ProcessInstanceExecCacheManagerImplTest.java @@ -80,4 +80,4 @@ public void testGetAll() { Collection workflowExecuteThreads = processInstanceExecCacheManager.getAll(); Assertions.assertEquals(1, workflowExecuteThreads.size()); } -} \ No newline at end of file +} diff --git a/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/consumer/TaskPriorityQueueConsumerTest.java b/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/consumer/TaskPriorityQueueConsumerTest.java index 066a4686cf09..bbd4a321cdf9 100644 --- a/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/consumer/TaskPriorityQueueConsumerTest.java +++ b/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/consumer/TaskPriorityQueueConsumerTest.java @@ -37,11 +37,11 @@ import java.util.Date; import java.util.concurrent.TimeUnit; +import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.extension.ExtendWith; import org.mockito.Mockito; import org.springframework.beans.factory.annotation.Autowired; diff --git a/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/dispatch/ExecutorDispatcherTest.java b/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/dispatch/ExecutorDispatcherTest.java index f14ef4f8be2d..20a8d2a9f717 100644 --- a/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/dispatch/ExecutorDispatcherTest.java +++ b/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/dispatch/ExecutorDispatcherTest.java @@ -54,7 +54,7 @@ public class ExecutorDispatcherTest { public void testDispatchWithException() throws ExecuteException { ExecutionContext executionContext = ExecutionContextTestUtils.getExecutionContext(10000); Assertions.assertThrows(ExecuteException.class, () -> { - executorDispatcher.dispatch(executionContext); + executorDispatcher.dispatch(executionContext); }); } diff --git a/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/dispatch/executor/NettyExecutorManagerTest.java b/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/dispatch/executor/NettyExecutorManagerTest.java index 4eba1c5d6d2d..9fa680fb2403 100644 --- a/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/dispatch/executor/NettyExecutorManagerTest.java +++ b/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/dispatch/executor/NettyExecutorManagerTest.java @@ -48,6 +48,7 @@ @ExtendWith(SpringExtension.class) @Disabled public class NettyExecutorManagerTest { + @Autowired private NettyExecutorManager nettyExecutorManager; @Test @@ -55,8 +56,9 @@ public void testExecute() throws ExecuteException { final NettyServerConfig serverConfig = new NettyServerConfig(); serverConfig.setListenPort(30000); NettyRemotingServer nettyRemotingServer = new NettyRemotingServer(serverConfig); - nettyRemotingServer.registerProcessor(org.apache.dolphinscheduler.remote.command.CommandType.TASK_DISPATCH_REQUEST, - new TaskDispatchProcessor()); + nettyRemotingServer.registerProcessor( + org.apache.dolphinscheduler.remote.command.CommandType.TASK_DISPATCH_REQUEST, + new TaskDispatchProcessor()); nettyRemotingServer.start(); TaskInstance taskInstance = Mockito.mock(TaskInstance.class); ProcessDefinition processDefinition = Mockito.mock(ProcessDefinition.class); @@ -64,10 +66,10 @@ public void testExecute() throws ExecuteException { processInstance.setCommandType(CommandType.COMPLEMENT_DATA); taskInstance.setProcessInstance(processInstance); TaskExecutionContext context = TaskExecutionContextBuilder.get() - .buildTaskInstanceRelatedInfo(taskInstance) - .buildProcessInstanceRelatedInfo(processInstance) - .buildProcessDefinitionRelatedInfo(processDefinition) - .create(); + .buildTaskInstanceRelatedInfo(taskInstance) + .buildProcessInstanceRelatedInfo(processInstance) + .buildProcessDefinitionRelatedInfo(processDefinition) + .create(); ExecutionContext executionContext = new ExecutionContext(toCommand(context), ExecutorType.WORKER, taskInstance); executionContext.setHost(Host.of(NetUtils.getAddr(serverConfig.getListenPort()))); Boolean execute = nettyExecutorManager.execute(executionContext); @@ -96,9 +98,9 @@ public void testExecuteWithException() throws ExecuteException { } private Command toCommand(TaskExecutionContext taskExecutionContext) { TaskDispatchCommand requestCommand = new TaskDispatchCommand(taskExecutionContext, - "127.0.0.1:5678", - "127.0.0.1:1234", - System.currentTimeMillis()); + "127.0.0.1:5678", + "127.0.0.1:1234", + System.currentTimeMillis()); return requestCommand.convert2Command(); } } diff --git a/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/dispatch/host/RoundRobinHostManagerTest.java b/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/dispatch/host/RoundRobinHostManagerTest.java index a325034c5137..4862d28d493e 100644 --- a/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/dispatch/host/RoundRobinHostManagerTest.java +++ b/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/dispatch/host/RoundRobinHostManagerTest.java @@ -22,6 +22,7 @@ import org.apache.dolphinscheduler.server.master.dispatch.ExecutionContextTestUtils; import org.apache.dolphinscheduler.server.master.dispatch.context.ExecutionContext; import org.apache.dolphinscheduler.server.master.registry.ServerNodeManager; + import org.assertj.core.util.Strings; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; @@ -29,9 +30,9 @@ import org.mockito.InjectMocks; import org.mockito.Mock; import org.mockito.Mockito; +import org.mockito.junit.jupiter.MockitoExtension; import com.google.common.collect.Sets; -import org.mockito.junit.jupiter.MockitoExtension; /** * round robin host manager test diff --git a/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/dispatch/host/assign/LowerWeightRoundRobinTest.java b/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/dispatch/host/assign/LowerWeightRoundRobinTest.java index 12586c5fd71d..8159603544a0 100644 --- a/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/dispatch/host/assign/LowerWeightRoundRobinTest.java +++ b/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/dispatch/host/assign/LowerWeightRoundRobinTest.java @@ -28,9 +28,12 @@ public class LowerWeightRoundRobinTest { @Test public void testSelect() { Collection sources = new ArrayList<>(); - sources.add(new HostWeight(HostWorker.of("192.158.2.1:11", 100, "default"), 0.06, 0.44, 3.84, 1, System.currentTimeMillis() - 60 * 8 * 1000)); - sources.add(new HostWeight(HostWorker.of("192.158.2.2:22", 100, "default"), 0.06, 0.56, 3.24, 2, System.currentTimeMillis() - 60 * 5 * 1000)); - sources.add(new HostWeight(HostWorker.of("192.158.2.3:33", 100, "default"), 0.06, 0.80, 3.15, 1, System.currentTimeMillis() - 60 * 2 * 1000)); + sources.add(new HostWeight(HostWorker.of("192.158.2.1:11", 100, "default"), 0.06, 0.44, 3.84, 1, + System.currentTimeMillis() - 60 * 8 * 1000)); + sources.add(new HostWeight(HostWorker.of("192.158.2.2:22", 100, "default"), 0.06, 0.56, 3.24, 2, + System.currentTimeMillis() - 60 * 5 * 1000)); + sources.add(new HostWeight(HostWorker.of("192.158.2.3:33", 100, "default"), 0.06, 0.80, 3.15, 1, + System.currentTimeMillis() - 60 * 2 * 1000)); LowerWeightRoundRobin roundRobin = new LowerWeightRoundRobin(); HostWeight result; @@ -54,10 +57,14 @@ public void testSelect() { @Test public void testWarmUpSelect() { Collection sources = new ArrayList<>(); - sources.add(new HostWeight(HostWorker.of("192.158.2.1:11", 100, "default"), 0.06, 0.44, 3.84, 0, System.currentTimeMillis() - 60 * 8 * 1000)); - sources.add(new HostWeight(HostWorker.of("192.158.2.2:22", 100, "default"), 0.06, 0.44, 3.84, 0, System.currentTimeMillis() - 60 * 5 * 1000)); - sources.add(new HostWeight(HostWorker.of("192.158.2.3:33", 100, "default"), 0.06, 0.44, 3.84, 0, System.currentTimeMillis() - 60 * 3 * 1000)); - sources.add(new HostWeight(HostWorker.of("192.158.2.4:33", 100, "default"), 0.06, 0.44, 3.84, 0, System.currentTimeMillis() - 60 * 11 * 1000)); + sources.add(new HostWeight(HostWorker.of("192.158.2.1:11", 100, "default"), 0.06, 0.44, 3.84, 0, + System.currentTimeMillis() - 60 * 8 * 1000)); + sources.add(new HostWeight(HostWorker.of("192.158.2.2:22", 100, "default"), 0.06, 0.44, 3.84, 0, + System.currentTimeMillis() - 60 * 5 * 1000)); + sources.add(new HostWeight(HostWorker.of("192.158.2.3:33", 100, "default"), 0.06, 0.44, 3.84, 0, + System.currentTimeMillis() - 60 * 3 * 1000)); + sources.add(new HostWeight(HostWorker.of("192.158.2.4:33", 100, "default"), 0.06, 0.44, 3.84, 0, + System.currentTimeMillis() - 60 * 11 * 1000)); LowerWeightRoundRobin roundRobin = new LowerWeightRoundRobin(); HostWeight result; @@ -81,9 +88,12 @@ public void testDoSelect() { result = roundRobin.doSelect(sources); Assertions.assertNull(result); - sources.add(new HostWeight(HostWorker.of("192.158.2.1:11", 100, "default"), 0.06, 0.44, 3.14, 1, System.currentTimeMillis() - 60 * 8 * 1000)); - sources.add(new HostWeight(HostWorker.of("192.158.2.2:22", 100, "default"), 0.06, 0.56, 3.24, 2, System.currentTimeMillis() - 60 * 5 * 1000)); - sources.add(new HostWeight(HostWorker.of("192.158.2.3:33", 100, "default"), 0.06, 0.80, 3.15, 1, System.currentTimeMillis() - 60 * 2 * 1000)); + sources.add(new HostWeight(HostWorker.of("192.158.2.1:11", 100, "default"), 0.06, 0.44, 3.14, 1, + System.currentTimeMillis() - 60 * 8 * 1000)); + sources.add(new HostWeight(HostWorker.of("192.158.2.2:22", 100, "default"), 0.06, 0.56, 3.24, 2, + System.currentTimeMillis() - 60 * 5 * 1000)); + sources.add(new HostWeight(HostWorker.of("192.158.2.3:33", 100, "default"), 0.06, 0.80, 3.15, 1, + System.currentTimeMillis() - 60 * 2 * 1000)); result = roundRobin.doSelect(sources); Assertions.assertEquals("192.158.2.1", result.getHost().getIp()); } diff --git a/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/dispatch/host/assign/RandomSelectorTest.java b/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/dispatch/host/assign/RandomSelectorTest.java index 6c079c71bf7d..baad4f51f4ba 100644 --- a/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/dispatch/host/assign/RandomSelectorTest.java +++ b/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/dispatch/host/assign/RandomSelectorTest.java @@ -38,14 +38,16 @@ public void testSelectWithIllegalArgumentException() { @Test public void testSelect1() { RandomSelector selector = new RandomSelector(); - HostWorker result = selector.select(Arrays.asList(new HostWorker("192.168.1.1:11", 100, "default"), new HostWorker("192.168.1.2:22", 80, "default"))); + HostWorker result = selector.select(Arrays.asList(new HostWorker("192.168.1.1:11", 100, "default"), + new HostWorker("192.168.1.2:22", 80, "default"))); Assertions.assertNotNull(result); } @Test public void testSelect() { RandomSelector selector = new RandomSelector(); - HostWorker result = selector.select(Arrays.asList(new HostWorker("192.168.1.1", 11, 100, "default"), new HostWorker("192.168.1.2:", 22, 20, "default"))); + HostWorker result = selector.select(Arrays.asList(new HostWorker("192.168.1.1", 11, 100, "default"), + new HostWorker("192.168.1.2:", 22, 20, "default"))); Assertions.assertNotNull(result); } } diff --git a/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/dispatch/host/assign/RoundRobinSelectorTest.java b/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/dispatch/host/assign/RoundRobinSelectorTest.java index 2341ba9ec628..910998abf773 100644 --- a/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/dispatch/host/assign/RoundRobinSelectorTest.java +++ b/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/dispatch/host/assign/RoundRobinSelectorTest.java @@ -40,13 +40,13 @@ public void testSelectWithIllegalArgumentException() { public void testSelect1() { RoundRobinSelector selector = new RoundRobinSelector(); List hostOneList = Arrays.asList( - new HostWorker("192.168.1.1", 80, 20, "kris"), - new HostWorker("192.168.1.2", 80, 10, "kris")); + new HostWorker("192.168.1.1", 80, 20, "kris"), + new HostWorker("192.168.1.2", 80, 10, "kris")); List hostTwoList = Arrays.asList( - new HostWorker("192.168.1.1", 80, 20, "kris"), - new HostWorker("192.168.1.2", 80, 10, "kris"), - new HostWorker("192.168.1.3", 80, 10, "kris")); + new HostWorker("192.168.1.1", 80, 20, "kris"), + new HostWorker("192.168.1.2", 80, 10, "kris"), + new HostWorker("192.168.1.3", 80, 10, "kris")); HostWorker result; result = selector.select(hostOneList); @@ -98,7 +98,8 @@ public void testWeightRoundRobinSelector() { RoundRobinSelector selector = new RoundRobinSelector(); HostWorker result; result = selector.select( - Arrays.asList(new HostWorker("192.168.1.1", 11, 20, "kris"), new HostWorker("192.168.1.2", 22, 80, "kris"))); + Arrays.asList(new HostWorker("192.168.1.1", 11, 20, "kris"), + new HostWorker("192.168.1.2", 22, 80, "kris"))); Assertions.assertEquals("192.168.1.2", result.getIp()); } diff --git a/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/processor/TaskAckProcessorTest.java b/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/processor/TaskAckProcessorTest.java index 476bab2c204d..c0fe33a137d1 100644 --- a/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/processor/TaskAckProcessorTest.java +++ b/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/processor/TaskAckProcessorTest.java @@ -30,9 +30,9 @@ import org.junit.jupiter.api.extension.ExtendWith; import org.mockito.MockedStatic; import org.mockito.Mockito; +import org.mockito.junit.jupiter.MockitoExtension; import io.netty.channel.Channel; -import org.mockito.junit.jupiter.MockitoExtension; /** * task ack processor test diff --git a/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/processor/TaskResponseProcessorTestConfig.java b/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/processor/TaskResponseProcessorTestConfig.java index 79065569e5ba..df88de34e30a 100644 --- a/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/processor/TaskResponseProcessorTestConfig.java +++ b/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/processor/TaskResponseProcessorTestConfig.java @@ -33,4 +33,4 @@ public class TaskResponseProcessorTestConfig { public DataQualityResultOperator dataQualityResultOperator() { return Mockito.mock(DataQualityResultOperator.class); } -} \ No newline at end of file +} diff --git a/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/processor/queue/TaskResponseServiceTest.java b/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/processor/queue/TaskResponseServiceTest.java index 1825d521f25c..4b9e37a601c3 100644 --- a/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/processor/queue/TaskResponseServiceTest.java +++ b/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/processor/queue/TaskResponseServiceTest.java @@ -33,9 +33,9 @@ import org.junit.jupiter.api.extension.ExtendWith; import org.mockito.InjectMocks; import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; import io.netty.channel.Channel; -import org.mockito.junit.jupiter.MockitoExtension; @ExtendWith(MockitoExtension.class) public class TaskResponseServiceTest { diff --git a/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/runner/task/CommonTaskProcessorTest.java b/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/runner/task/CommonTaskProcessorTest.java index 77091111ad9a..0217748fa439 100644 --- a/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/runner/task/CommonTaskProcessorTest.java +++ b/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/runner/task/CommonTaskProcessorTest.java @@ -17,16 +17,29 @@ package org.apache.dolphinscheduler.server.master.runner.task; +import static org.mockito.ArgumentMatchers.any; + import org.apache.dolphinscheduler.common.enums.CommandType; import org.apache.dolphinscheduler.common.enums.Priority; import org.apache.dolphinscheduler.common.enums.TimeoutFlag; -import org.apache.dolphinscheduler.dao.entity.*; +import org.apache.dolphinscheduler.dao.entity.ProcessDefinition; +import org.apache.dolphinscheduler.dao.entity.ProcessInstance; +import org.apache.dolphinscheduler.dao.entity.Resource; +import org.apache.dolphinscheduler.dao.entity.TaskDefinition; +import org.apache.dolphinscheduler.dao.entity.TaskInstance; +import org.apache.dolphinscheduler.dao.entity.Tenant; import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext; import org.apache.dolphinscheduler.plugin.task.api.enums.TaskExecutionStatus; import org.apache.dolphinscheduler.plugin.task.api.enums.TaskTimeoutStrategy; import org.apache.dolphinscheduler.service.bean.SpringApplicationContext; import org.apache.dolphinscheduler.service.process.ProcessService; import org.apache.dolphinscheduler.spi.enums.ResourceType; + +import java.util.ArrayList; +import java.util.Date; +import java.util.List; +import java.util.Map; + import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -37,14 +50,6 @@ import org.mockito.quality.Strictness; import org.springframework.context.ApplicationContext; -import java.util.ArrayList; -import java.util.Date; -import java.util.List; -import java.util.Map; - -import static org.mockito.ArgumentMatchers.any; - - @ExtendWith(MockitoExtension.class) @MockitoSettings(strictness = Strictness.LENIENT) public class CommonTaskProcessorTest { @@ -126,7 +131,8 @@ public void testGetResourceFullNames() { resource.setFileName("fileName"); resourcesList.add(resource); Mockito.doReturn(resourcesList).when(processService).listResourceByIds(new Integer[]{123}); - Mockito.doReturn("tenantCode").when(processService).queryTenantCodeByResName(resource.getFullName(), ResourceType.FILE); + Mockito.doReturn("tenantCode").when(processService).queryTenantCodeByResName(resource.getFullName(), + ResourceType.FILE); Assertions.assertNotNull(map); } @@ -166,17 +172,19 @@ public void testReplaceTestDatSource() { TaskInstance taskInstance = new TaskInstance(); taskInstance.setTestFlag(1); taskInstance.setTaskType("SQL"); - taskInstance.setTaskParams("{\"localParams\":[],\"resourceList\":[],\"type\":\"MYSQL\",\"datasource\":1,\"sql\":\"select * from 'order'\",\"sqlType\":\"0\",\"preStatements\":[],\"postStatements\":[],\"segmentSeparator\":\"\",\"displayRows\":10}"); + taskInstance.setTaskParams( + "{\"localParams\":[],\"resourceList\":[],\"type\":\"MYSQL\",\"datasource\":1,\"sql\":\"select * from 'order'\",\"sqlType\":\"0\",\"preStatements\":[],\"postStatements\":[],\"segmentSeparator\":\"\",\"displayRows\":10}"); TaskDefinition taskDefinition = new TaskDefinition(); - taskDefinition.setTaskParams("{\"localParams\":[],\"resourceList\":[],\"type\":\"MYSQL\",\"datasource\":1,\"sql\":\"select * from 'order'\",\"sqlType\":\"0\",\"preStatements\":[],\"postStatements\":[],\"segmentSeparator\":\"\",\"displayRows\":10}"); + taskDefinition.setTaskParams( + "{\"localParams\":[],\"resourceList\":[],\"type\":\"MYSQL\",\"datasource\":1,\"sql\":\"select * from 'order'\",\"sqlType\":\"0\",\"preStatements\":[],\"postStatements\":[],\"segmentSeparator\":\"\",\"displayRows\":10}"); taskInstance.setTaskDefine(taskDefinition); commonTaskProcessor1.taskInstance = taskInstance; - //The data source instance has no bound test data source + // The data source instance has no bound test data source Mockito.when(processService.queryTestDataSourceId(any(Integer.class))).thenReturn(null); commonTaskProcessor1.convertExeEnvironmentOnlineToTest(); - //The data source instance has bound test data source + // The data source instance has bound test data source Mockito.when(processService.queryTestDataSourceId(any(Integer.class))).thenReturn(2); commonTaskProcessor1.convertExeEnvironmentOnlineToTest(); } diff --git a/dolphinscheduler-meter/pom.xml b/dolphinscheduler-meter/pom.xml index acaee02dca8f..9fbb578f8cbb 100644 --- a/dolphinscheduler-meter/pom.xml +++ b/dolphinscheduler-meter/pom.xml @@ -18,18 +18,29 @@ ~ under the License. ~ --> - + 4.0.0 - dolphinscheduler org.apache.dolphinscheduler + dolphinscheduler dev-SNAPSHOT - 4.0.0 dolphinscheduler-meter + + + + org.apache.dolphinscheduler + dolphinscheduler-bom + ${project.version} + pom + import + + + + org.springframework.boot @@ -50,8 +61,8 @@ spring-boot-starter-tomcat - log4j-to-slf4j org.apache.logging.log4j + log4j-to-slf4j @@ -83,18 +94,6 @@ - - - - org.apache.dolphinscheduler - dolphinscheduler-bom - ${project.version} - pom - import - - - - diff --git a/dolphinscheduler-meter/src/main/java/org/apache/dolphinscheduler/meter/MeterConfiguration.java b/dolphinscheduler-meter/src/main/java/org/apache/dolphinscheduler/meter/MeterConfiguration.java index 64259f2f4f75..e3b140a57842 100644 --- a/dolphinscheduler-meter/src/main/java/org/apache/dolphinscheduler/meter/MeterConfiguration.java +++ b/dolphinscheduler-meter/src/main/java/org/apache/dolphinscheduler/meter/MeterConfiguration.java @@ -47,6 +47,7 @@ @EnableAutoConfiguration @ConditionalOnProperty(prefix = "metrics", name = "enabled", havingValue = "true") public class MeterConfiguration { + @Bean public TimedAspect timedAspect(MeterRegistry registry) { return new TimedAspect(registry); diff --git a/dolphinscheduler-microbench/pom.xml b/dolphinscheduler-microbench/pom.xml index 9ee35e09d08b..f3447b319ca2 100644 --- a/dolphinscheduler-microbench/pom.xml +++ b/dolphinscheduler-microbench/pom.xml @@ -15,15 +15,14 @@ ~ See the License for the specific language governing permissions and ~ limitations under the License. --> - + 4.0.0 - dolphinscheduler org.apache.dolphinscheduler + dolphinscheduler dev-SNAPSHOT - 4.0.0 dolphinscheduler-microbench jar @@ -35,6 +34,18 @@ benchmarks + + + + org.apache.dolphinscheduler + dolphinscheduler-bom + ${project.version} + pom + import + + + + @@ -67,18 +78,6 @@ - - - - org.apache.dolphinscheduler - dolphinscheduler-bom - ${project.version} - pom - import - - - - @@ -112,5 +111,4 @@ - diff --git a/dolphinscheduler-microbench/src/main/java/org/apache/dolphinscheduler/microbench/base/AbstractBaseBenchmark.java b/dolphinscheduler-microbench/src/main/java/org/apache/dolphinscheduler/microbench/base/AbstractBaseBenchmark.java index 2059f3f26606..392ad3e10088 100644 --- a/dolphinscheduler-microbench/src/main/java/org/apache/dolphinscheduler/microbench/base/AbstractBaseBenchmark.java +++ b/dolphinscheduler-microbench/src/main/java/org/apache/dolphinscheduler/microbench/base/AbstractBaseBenchmark.java @@ -122,4 +122,3 @@ private static int getForks() { } } - diff --git a/dolphinscheduler-microbench/src/main/java/org/apache/dolphinscheduler/microbench/common/EnumBenchMark.java b/dolphinscheduler-microbench/src/main/java/org/apache/dolphinscheduler/microbench/common/EnumBenchMark.java index dcce5368e3e8..b2299458f522 100644 --- a/dolphinscheduler-microbench/src/main/java/org/apache/dolphinscheduler/microbench/common/EnumBenchMark.java +++ b/dolphinscheduler-microbench/src/main/java/org/apache/dolphinscheduler/microbench/common/EnumBenchMark.java @@ -16,13 +16,22 @@ */ package org.apache.dolphinscheduler.microbench.common; - import org.apache.dolphinscheduler.microbench.base.AbstractBaseBenchmark; -import org.openjdk.jmh.annotations.*; + import java.util.HashMap; import java.util.Map; import java.util.concurrent.TimeUnit; +import org.openjdk.jmh.annotations.Benchmark; +import org.openjdk.jmh.annotations.BenchmarkMode; +import org.openjdk.jmh.annotations.Measurement; +import org.openjdk.jmh.annotations.Mode; +import org.openjdk.jmh.annotations.OutputTimeUnit; +import org.openjdk.jmh.annotations.Param; +import org.openjdk.jmh.annotations.Scope; +import org.openjdk.jmh.annotations.State; +import org.openjdk.jmh.annotations.Warmup; + /** *Enum values JMH test */ @@ -32,13 +41,12 @@ public class EnumBenchMark extends AbstractBaseBenchmark { @Benchmark - public boolean simpleTest(){ + public boolean simpleTest() { return Boolean.TRUE; } @Param({"101", "108", "103", "104", "105", "103"}) private int testNum; - @Benchmark @BenchmarkMode(Mode.AverageTime) @OutputTimeUnit(TimeUnit.MICROSECONDS) @@ -53,8 +61,7 @@ public void enumStaticMapTest() { TestTypeEnum.newGetNameByType(testNum); } - - public enum TestTypeEnum { + public enum TestTypeEnum { TYPE_101(101, "TYPE101"), TYPE_102(102, "TYPE102"), @@ -72,12 +79,10 @@ public int getCode() { return code; } - public String getName() { return name; } - TestTypeEnum(int code, String name) { this.code = code; this.name = name; @@ -87,7 +92,7 @@ public String getName() { static { for (TestTypeEnum testTypeEnum : TestTypeEnum.values()) { - TEST_TYPE_MAP.put(testTypeEnum.code,testTypeEnum); + TEST_TYPE_MAP.put(testTypeEnum.code, testTypeEnum); } } diff --git a/dolphinscheduler-microbench/src/main/java/org/apache/dolphinscheduler/microbench/common/RpcTest.java b/dolphinscheduler-microbench/src/main/java/org/apache/dolphinscheduler/microbench/common/RpcTest.java index ecc54f8f2613..1f3f3213d1c4 100644 --- a/dolphinscheduler-microbench/src/main/java/org/apache/dolphinscheduler/microbench/common/RpcTest.java +++ b/dolphinscheduler-microbench/src/main/java/org/apache/dolphinscheduler/microbench/common/RpcTest.java @@ -43,6 +43,7 @@ @State(Scope.Benchmark) @BenchmarkMode({Mode.Throughput, Mode.AverageTime, Mode.SampleTime}) public class RpcTest extends AbstractBaseBenchmark { + private NettyServer nettyServer; private IUserService userService; diff --git a/dolphinscheduler-microbench/src/main/java/org/apache/dolphinscheduler/microbench/common/UserCallback.java b/dolphinscheduler-microbench/src/main/java/org/apache/dolphinscheduler/microbench/common/UserCallback.java index bb32093f91c7..130bcf746988 100644 --- a/dolphinscheduler-microbench/src/main/java/org/apache/dolphinscheduler/microbench/common/UserCallback.java +++ b/dolphinscheduler-microbench/src/main/java/org/apache/dolphinscheduler/microbench/common/UserCallback.java @@ -23,6 +23,7 @@ * UserCallback */ public class UserCallback extends AbstractRpcCallBack { + @Override public void run(Object object) { diff --git a/dolphinscheduler-python/pom.xml b/dolphinscheduler-python/pom.xml index 410180e847a8..a3133a52e7ff 100644 --- a/dolphinscheduler-python/pom.xml +++ b/dolphinscheduler-python/pom.xml @@ -15,7 +15,6 @@ ~ See the License for the specific language governing permissions and ~ limitations under the License. --> - 4.0.0 @@ -25,8 +24,8 @@ dev-SNAPSHOT dolphinscheduler-python - ${project.artifactId} jar + ${project.artifactId} @@ -45,10 +44,10 @@ python-api-prepare - prepare-package exec + prepare-package python ${project.basedir}/pydolphinscheduler @@ -64,10 +63,10 @@ python-api-clean - prepare-package exec + prepare-package python ${project.basedir}/pydolphinscheduler @@ -79,10 +78,10 @@ python-api-build - prepare-package exec + prepare-package python ${project.basedir}/pydolphinscheduler @@ -95,10 +94,10 @@ python-pkg-rename-tar - prepare-package exec + prepare-package bash ${project.basedir}/pydolphinscheduler @@ -110,10 +109,10 @@ python-pkg-rename-whl - prepare-package exec + prepare-package bash ${project.basedir}/pydolphinscheduler @@ -125,10 +124,10 @@ sign-source - prepare-package exec + prepare-package ${python.sign.skip} bash @@ -142,10 +141,10 @@ sign-wheel - prepare-package exec + prepare-package ${python.sign.skip} bash diff --git a/dolphinscheduler-python/pydolphinscheduler/DEVELOP.md b/dolphinscheduler-python/pydolphinscheduler/DEVELOP.md index fbc1e935324e..eac4b3678a04 100644 --- a/dolphinscheduler-python/pydolphinscheduler/DEVELOP.md +++ b/dolphinscheduler-python/pydolphinscheduler/DEVELOP.md @@ -249,7 +249,8 @@ When you add a new package in pydolphinscheduler, you should also add the packag When you change public class, method or interface, you should change the [UPDATING.md](./UPDATING.md) to notice users who may use it in other way. - +## Reference + [py4j]: https://www.py4j.org/index.html [pycharm]: https://www.jetbrains.com/pycharm [idea]: https://www.jetbrains.com/idea/ @@ -261,3 +262,4 @@ users who may use it in other way. [coverage]: https://coverage.readthedocs.io/en/stable/ [isort]: https://pycqa.github.io/isort/index.html [sphinx]: https://www.sphinx-doc.org/en/master + diff --git a/dolphinscheduler-registry/dolphinscheduler-registry-all/pom.xml b/dolphinscheduler-registry/dolphinscheduler-registry-all/pom.xml index 682e66830136..bc55198b257b 100644 --- a/dolphinscheduler-registry/dolphinscheduler-registry-all/pom.xml +++ b/dolphinscheduler-registry/dolphinscheduler-registry-all/pom.xml @@ -15,15 +15,14 @@ ~ See the License for the specific language governing permissions and ~ limitations under the License. --> - + 4.0.0 org.apache.dolphinscheduler dolphinscheduler-registry dev-SNAPSHOT - 4.0.0 dolphinscheduler-registry-all @@ -44,4 +43,4 @@ ${project.version} - \ No newline at end of file + diff --git a/dolphinscheduler-registry/dolphinscheduler-registry-api/pom.xml b/dolphinscheduler-registry/dolphinscheduler-registry-api/pom.xml index b704c4131689..e798920cebd0 100644 --- a/dolphinscheduler-registry/dolphinscheduler-registry-api/pom.xml +++ b/dolphinscheduler-registry/dolphinscheduler-registry-api/pom.xml @@ -17,16 +17,14 @@ ~ specific language governing permissions and limitations ~ under the License. --> - - + 4.0.0 - dolphinscheduler-registry org.apache.dolphinscheduler + dolphinscheduler-registry dev-SNAPSHOT - 4.0.0 dolphinscheduler-registry-api diff --git a/dolphinscheduler-registry/dolphinscheduler-registry-api/src/main/java/org/apache/dolphinscheduler/registry/api/ConnectStrategyProperties.java b/dolphinscheduler-registry/dolphinscheduler-registry-api/src/main/java/org/apache/dolphinscheduler/registry/api/ConnectStrategyProperties.java index e76cd57181f6..5457a9e44ab5 100644 --- a/dolphinscheduler-registry/dolphinscheduler-registry-api/src/main/java/org/apache/dolphinscheduler/registry/api/ConnectStrategyProperties.java +++ b/dolphinscheduler-registry/dolphinscheduler-registry-api/src/main/java/org/apache/dolphinscheduler/registry/api/ConnectStrategyProperties.java @@ -17,10 +17,10 @@ package org.apache.dolphinscheduler.registry.api; -import lombok.Data; - import java.time.Duration; +import lombok.Data; + @Data public class ConnectStrategyProperties { diff --git a/dolphinscheduler-registry/dolphinscheduler-registry-api/src/main/java/org/apache/dolphinscheduler/registry/api/ConnectionListener.java b/dolphinscheduler-registry/dolphinscheduler-registry-api/src/main/java/org/apache/dolphinscheduler/registry/api/ConnectionListener.java index 1a3d051091c8..d854eaf8e1e6 100644 --- a/dolphinscheduler-registry/dolphinscheduler-registry-api/src/main/java/org/apache/dolphinscheduler/registry/api/ConnectionListener.java +++ b/dolphinscheduler-registry/dolphinscheduler-registry-api/src/main/java/org/apache/dolphinscheduler/registry/api/ConnectionListener.java @@ -25,5 +25,6 @@ */ @FunctionalInterface public interface ConnectionListener { + void onUpdate(ConnectionState newState); } diff --git a/dolphinscheduler-registry/dolphinscheduler-registry-api/src/main/java/org/apache/dolphinscheduler/registry/api/Event.java b/dolphinscheduler-registry/dolphinscheduler-registry-api/src/main/java/org/apache/dolphinscheduler/registry/api/Event.java index 32fe933fa2bb..9d6823819764 100644 --- a/dolphinscheduler-registry/dolphinscheduler-registry-api/src/main/java/org/apache/dolphinscheduler/registry/api/Event.java +++ b/dolphinscheduler-registry/dolphinscheduler-registry-api/src/main/java/org/apache/dolphinscheduler/registry/api/Event.java @@ -20,6 +20,7 @@ package org.apache.dolphinscheduler.registry.api; public class Event { + // The prefix which is watched private String key; // The full path where the event was generated @@ -80,7 +81,8 @@ public Event type(Type type) { } public String toString() { - return "Event(key=" + this.key() + ", path=" + this.path() + ", data=" + this.data() + ", type=" + this.type() + ")"; + return "Event(key=" + this.key() + ", path=" + this.path() + ", data=" + this.data() + ", type=" + this.type() + + ")"; } public enum Type { @@ -90,6 +92,7 @@ public enum Type { } public static class EventBuilder { + private String key; private String path; private String data; @@ -123,7 +126,8 @@ public Event build() { } public String toString() { - return "Event.EventBuilder(key=" + this.key + ", path=" + this.path + ", data=" + this.data + ", type=" + this.type + ")"; + return "Event.EventBuilder(key=" + this.key + ", path=" + this.path + ", data=" + this.data + ", type=" + + this.type + ")"; } } } diff --git a/dolphinscheduler-registry/dolphinscheduler-registry-api/src/main/java/org/apache/dolphinscheduler/registry/api/SubscribeListener.java b/dolphinscheduler-registry/dolphinscheduler-registry-api/src/main/java/org/apache/dolphinscheduler/registry/api/SubscribeListener.java index 2432eb1e2532..164278123494 100644 --- a/dolphinscheduler-registry/dolphinscheduler-registry-api/src/main/java/org/apache/dolphinscheduler/registry/api/SubscribeListener.java +++ b/dolphinscheduler-registry/dolphinscheduler-registry-api/src/main/java/org/apache/dolphinscheduler/registry/api/SubscribeListener.java @@ -20,5 +20,6 @@ package org.apache.dolphinscheduler.registry.api; public interface SubscribeListener { + void notify(Event event); } diff --git a/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-etcd/pom.xml b/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-etcd/pom.xml index f55aa022e32d..b084db1ccfee 100644 --- a/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-etcd/pom.xml +++ b/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-etcd/pom.xml @@ -15,15 +15,14 @@ ~ See the License for the specific language governing permissions and ~ limitations under the License. --> - + 4.0.0 - dolphinscheduler-registry-plugins org.apache.dolphinscheduler + dolphinscheduler-registry-plugins dev-SNAPSHOT - 4.0.0 dolphinscheduler-registry-etcd @@ -64,4 +63,4 @@ slf4j-api - \ No newline at end of file + diff --git a/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-etcd/src/main/java/org/apache/dolphinscheduler/plugin/registry/etcd/EtcdConnectionStateListener.java b/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-etcd/src/main/java/org/apache/dolphinscheduler/plugin/registry/etcd/EtcdConnectionStateListener.java index 31c77e46eb9f..56c60c0e4e15 100644 --- a/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-etcd/src/main/java/org/apache/dolphinscheduler/plugin/registry/etcd/EtcdConnectionStateListener.java +++ b/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-etcd/src/main/java/org/apache/dolphinscheduler/plugin/registry/etcd/EtcdConnectionStateListener.java @@ -29,10 +29,10 @@ import java.util.concurrent.TimeUnit; import com.google.common.util.concurrent.ThreadFactoryBuilder; - import io.etcd.jetcd.Client; public class EtcdConnectionStateListener implements AutoCloseable { + private final List connectionListeners = Collections.synchronizedList(new ArrayList<>()); // A thread pool that periodically obtains connection status private final ScheduledExecutorService scheduledExecutorService; diff --git a/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-etcd/src/main/java/org/apache/dolphinscheduler/plugin/registry/etcd/EtcdRegistry.java b/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-etcd/src/main/java/org/apache/dolphinscheduler/plugin/registry/etcd/EtcdRegistry.java index 9348d9e02765..55be08784fcc 100644 --- a/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-etcd/src/main/java/org/apache/dolphinscheduler/plugin/registry/etcd/EtcdRegistry.java +++ b/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-etcd/src/main/java/org/apache/dolphinscheduler/plugin/registry/etcd/EtcdRegistry.java @@ -37,6 +37,8 @@ import javax.annotation.PostConstruct; +import lombok.NonNull; + import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; @@ -59,7 +61,6 @@ import io.etcd.jetcd.options.WatchOption; import io.etcd.jetcd.support.Observers; import io.etcd.jetcd.watch.WatchEvent; -import lombok.NonNull; /** * This is one of the implementation of {@link Registry}, with this implementation, you need to rely on Etcd cluster to @@ -68,6 +69,7 @@ @Component @ConditionalOnProperty(prefix = "registry", name = "type", havingValue = "etcd") public class EtcdRegistry implements Registry { + private static Logger LOGGER = LoggerFactory.getLogger(EtcdRegistry.class); private final Client client; private EtcdConnectionStateListener etcdConnectionStateListener; @@ -88,7 +90,8 @@ public EtcdRegistry(EtcdRegistryProperties registryProperties) { .retryDelay(registryProperties.getRetryDelay().toMillis()) .retryMaxDelay(registryProperties.getRetryMaxDelay().toMillis()) .retryMaxDuration(registryProperties.getRetryMaxDuration()); - if (StringUtils.hasLength(registryProperties.getUser()) && StringUtils.hasLength(registryProperties.getPassword())) { + if (StringUtils.hasLength(registryProperties.getUser()) + && StringUtils.hasLength(registryProperties.getPassword())) { clientBuilder.user(byteSequence(registryProperties.getUser())); clientBuilder.password(byteSequence(registryProperties.getPassword())); } @@ -129,11 +132,12 @@ public boolean subscribe(String path, SubscribeListener listener) { try { ByteSequence watchKey = byteSequence(path); WatchOption watchOption = WatchOption.newBuilder().isPrefix(true).build(); - watcherMap.computeIfAbsent(path, $ -> client.getWatchClient().watch(watchKey, watchOption,watchResponse -> { - for (WatchEvent event : watchResponse.getEvents()) { - listener.notify(new EventAdaptor(event, path)); - } - })); + watcherMap.computeIfAbsent(path, + $ -> client.getWatchClient().watch(watchKey, watchOption, watchResponse -> { + for (WatchEvent event : watchResponse.getEvents()) { + listener.notify(new EventAdaptor(event, path)); + } + })); } catch (Exception e) { throw new RegistryException("Failed to subscribe listener for key: " + path, e); } @@ -190,7 +194,7 @@ public void put(String key, String value, boolean deleteOnDisconnect) { client.getLeaseClient().keepAlive(leaseId, Observers.observer(response -> { })); PutOption putOption = PutOption.newBuilder().withLeaseId(leaseId).build(); - client.getKVClient().put(byteSequence(key), byteSequence(value),putOption).get(); + client.getKVClient().put(byteSequence(key), byteSequence(value), putOption).get(); } else { client.getKVClient().put(byteSequence(key), byteSequence(value)).get(); } @@ -210,10 +214,10 @@ public void delete(String key) { try { DeleteOption deleteOption = DeleteOption.newBuilder().isPrefix(true).build(); client.getKVClient().delete(byteSequence(key), deleteOption).get(); - } catch (InterruptedException e) { + } catch (InterruptedException e) { Thread.currentThread().interrupt(); throw new RegistryException("Failed to delete registry key: " + key, e); - } catch (ExecutionException e) { + } catch (ExecutionException e) { throw new RegistryException("Failed to delete registry key: " + key, e); } } @@ -226,10 +230,12 @@ public Collection children(String key) { // Make sure the string end with '/' // eg:change key = /nodes to /nodes/ String prefix = key.endsWith(FOLDER_SEPARATOR) ? key : key + FOLDER_SEPARATOR; - GetOption getOption = GetOption.newBuilder().isPrefix(true).withSortField(GetOption.SortTarget.KEY).withSortOrder(GetOption.SortOrder.ASCEND).build(); + GetOption getOption = GetOption.newBuilder().isPrefix(true).withSortField(GetOption.SortTarget.KEY) + .withSortOrder(GetOption.SortOrder.ASCEND).build(); try { - List keyValues = client.getKVClient().get(byteSequence(prefix),getOption).get().getKvs(); - return keyValues.stream().map(e -> getSubNodeKeyName(prefix, e.getKey().toString(StandardCharsets.UTF_8))).distinct().collect(Collectors.toList()); + List keyValues = client.getKVClient().get(byteSequence(prefix), getOption).get().getKvs(); + return keyValues.stream().map(e -> getSubNodeKeyName(prefix, e.getKey().toString(StandardCharsets.UTF_8))) + .distinct().collect(Collectors.toList()); } catch (InterruptedException e) { Thread.currentThread().interrupt(); throw new RegistryException("etcd get children error", e); @@ -243,14 +249,16 @@ public Collection children(String key) { */ private String getSubNodeKeyName(final String prefix, final String fullPath) { String pathWithoutPrefix = fullPath.substring(prefix.length()); - return pathWithoutPrefix.contains(FOLDER_SEPARATOR) ? pathWithoutPrefix.substring(0, pathWithoutPrefix.indexOf(FOLDER_SEPARATOR)) : pathWithoutPrefix; + return pathWithoutPrefix.contains(FOLDER_SEPARATOR) + ? pathWithoutPrefix.substring(0, pathWithoutPrefix.indexOf(FOLDER_SEPARATOR)) + : pathWithoutPrefix; } @Override public boolean exists(String key) { GetOption getOption = GetOption.newBuilder().withCountOnly(true).build(); try { - if (client.getKVClient().get(byteSequence(key),getOption).get().getCount() >= 1) { + if (client.getKVClient().get(byteSequence(key), getOption).get().getCount() >= 1) { return true; } } catch (InterruptedException e) { @@ -275,13 +283,13 @@ public boolean acquireLock(String key) { // keep the lease client.getLeaseClient().keepAlive(leaseId, Observers.observer(response -> { })); - lockClient.lock(byteSequence(key),leaseId).get(); + lockClient.lock(byteSequence(key), leaseId).get(); // save the leaseId for release Lock if (null == threadLocalLockMap.get()) { threadLocalLockMap.set(new HashMap<>()); } - threadLocalLockMap.get().put(key,leaseId); + threadLocalLockMap.get().put(key, leaseId); return true; } catch (InterruptedException e) { Thread.currentThread().interrupt(); @@ -320,6 +328,7 @@ private static ByteSequence byteSequence(String val) { } static final class EventAdaptor extends Event { + public EventAdaptor(WatchEvent event, String key) { key(key); @@ -341,4 +350,3 @@ public EventAdaptor(WatchEvent event, String key) { } } } - diff --git a/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-etcd/src/main/java/org/apache/dolphinscheduler/plugin/registry/etcd/EtcdRegistryProperties.java b/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-etcd/src/main/java/org/apache/dolphinscheduler/plugin/registry/etcd/EtcdRegistryProperties.java index 96be83c5d5fc..593752a7c079 100644 --- a/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-etcd/src/main/java/org/apache/dolphinscheduler/plugin/registry/etcd/EtcdRegistryProperties.java +++ b/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-etcd/src/main/java/org/apache/dolphinscheduler/plugin/registry/etcd/EtcdRegistryProperties.java @@ -19,17 +19,18 @@ import java.time.Duration; +import lombok.Data; + import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.boot.context.properties.ConfigurationProperties; import org.springframework.context.annotation.Configuration; -import lombok.Data; - @Data @Configuration @ConditionalOnProperty(prefix = "registry", name = "type", havingValue = "etcd") @ConfigurationProperties(prefix = "registry") public class EtcdRegistryProperties { + private String endpoints; private String namespace = "dolphinscheduler"; private Duration connectionTimeout = Duration.ofSeconds(9); diff --git a/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-mysql/pom.xml b/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-mysql/pom.xml index 150b058354f4..fd7f0925e2da 100644 --- a/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-mysql/pom.xml +++ b/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-mysql/pom.xml @@ -17,15 +17,14 @@ ~ specific language governing permissions and limitations ~ under the License. --> - + 4.0.0 org.apache.dolphinscheduler dolphinscheduler-registry-plugins dev-SNAPSHOT - 4.0.0 dolphinscheduler-registry-mysql @@ -53,4 +52,4 @@ - \ No newline at end of file + diff --git a/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-mysql/src/main/java/org/apache/dolphinscheduler/plugin/registry/mysql/MysqlRegistry.java b/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-mysql/src/main/java/org/apache/dolphinscheduler/plugin/registry/mysql/MysqlRegistry.java index d475a1ae4f60..d5caae9f0066 100644 --- a/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-mysql/src/main/java/org/apache/dolphinscheduler/plugin/registry/mysql/MysqlRegistry.java +++ b/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-mysql/src/main/java/org/apache/dolphinscheduler/plugin/registry/mysql/MysqlRegistry.java @@ -17,7 +17,6 @@ package org.apache.dolphinscheduler.plugin.registry.mysql; -import lombok.NonNull; import org.apache.dolphinscheduler.plugin.registry.mysql.task.EphemeralDateManager; import org.apache.dolphinscheduler.plugin.registry.mysql.task.RegistryLockManager; import org.apache.dolphinscheduler.plugin.registry.mysql.task.SubscribeDataManager; @@ -27,16 +26,19 @@ import org.apache.dolphinscheduler.registry.api.RegistryException; import org.apache.dolphinscheduler.registry.api.SubscribeListener; +import java.sql.SQLException; +import java.time.Duration; +import java.util.Collection; + +import javax.annotation.PostConstruct; + +import lombok.NonNull; + import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.stereotype.Component; -import javax.annotation.PostConstruct; -import java.sql.SQLException; -import java.time.Duration; -import java.util.Collection; - /** * This is one of the implementation of {@link Registry}, with this implementation, you need to rely on mysql database to * store the DolphinScheduler master/worker's metadata and do the server registry/unRegistry. diff --git a/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-mysql/src/main/java/org/apache/dolphinscheduler/plugin/registry/mysql/MysqlRegistryProperties.java b/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-mysql/src/main/java/org/apache/dolphinscheduler/plugin/registry/mysql/MysqlRegistryProperties.java index d80b8ead0d28..9690418bfb5a 100644 --- a/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-mysql/src/main/java/org/apache/dolphinscheduler/plugin/registry/mysql/MysqlRegistryProperties.java +++ b/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-mysql/src/main/java/org/apache/dolphinscheduler/plugin/registry/mysql/MysqlRegistryProperties.java @@ -19,14 +19,14 @@ import java.time.Duration; +import lombok.Data; + import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.boot.context.properties.ConfigurationProperties; import org.springframework.context.annotation.Configuration; import com.zaxxer.hikari.HikariConfig; -import lombok.Data; - @Data @Configuration @ConditionalOnProperty(prefix = "registry", name = "type", havingValue = "mysql") diff --git a/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-mysql/src/main/java/org/apache/dolphinscheduler/plugin/registry/mysql/model/DataType.java b/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-mysql/src/main/java/org/apache/dolphinscheduler/plugin/registry/mysql/model/DataType.java index 657c80b716dc..bacc8c9a8da1 100644 --- a/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-mysql/src/main/java/org/apache/dolphinscheduler/plugin/registry/mysql/model/DataType.java +++ b/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-mysql/src/main/java/org/apache/dolphinscheduler/plugin/registry/mysql/model/DataType.java @@ -18,6 +18,7 @@ package org.apache.dolphinscheduler.plugin.registry.mysql.model; public enum DataType { + EPHEMERAL(1), PERSISTENT(2), ; diff --git a/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-mysql/src/main/java/org/apache/dolphinscheduler/plugin/registry/mysql/task/RegistryLockManager.java b/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-mysql/src/main/java/org/apache/dolphinscheduler/plugin/registry/mysql/task/RegistryLockManager.java index 478727545b6a..9c6ca6eeeb02 100644 --- a/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-mysql/src/main/java/org/apache/dolphinscheduler/plugin/registry/mysql/task/RegistryLockManager.java +++ b/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-mysql/src/main/java/org/apache/dolphinscheduler/plugin/registry/mysql/task/RegistryLockManager.java @@ -33,14 +33,14 @@ import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; +import lombok.AccessLevel; +import lombok.RequiredArgsConstructor; + import org.slf4j.Logger; import org.slf4j.LoggerFactory; import com.google.common.util.concurrent.ThreadFactoryBuilder; -import lombok.AccessLevel; -import lombok.RequiredArgsConstructor; - public class RegistryLockManager implements AutoCloseable { private static final Logger logger = LoggerFactory.getLogger(RegistryLockManager.class); @@ -112,6 +112,7 @@ public void close() { */ @RequiredArgsConstructor(access = AccessLevel.PRIVATE) static class LockTermRefreshTask implements Runnable { + private final Map lockHoldMap; private final MysqlOperator mysqlOperator; @@ -134,4 +135,3 @@ public void run() { } } } - diff --git a/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-mysql/src/main/java/org/apache/dolphinscheduler/plugin/registry/mysql/task/SubscribeDataManager.java b/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-mysql/src/main/java/org/apache/dolphinscheduler/plugin/registry/mysql/task/SubscribeDataManager.java index 4c8d7190f378..bbcc25d7be12 100644 --- a/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-mysql/src/main/java/org/apache/dolphinscheduler/plugin/registry/mysql/task/SubscribeDataManager.java +++ b/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-mysql/src/main/java/org/apache/dolphinscheduler/plugin/registry/mysql/task/SubscribeDataManager.java @@ -33,13 +33,13 @@ import java.util.function.Function; import java.util.stream.Collectors; +import lombok.RequiredArgsConstructor; + import org.slf4j.Logger; import org.slf4j.LoggerFactory; import com.google.common.util.concurrent.ThreadFactoryBuilder; -import lombok.RequiredArgsConstructor; - /** * Used to refresh if the subscribe path has been changed. */ @@ -58,7 +58,8 @@ public SubscribeDataManager(MysqlRegistryProperties registryProperties, MysqlOpe this.mysqlOperator = mysqlOperator; this.dataSubscribeCheckThreadPool = Executors.newScheduledThreadPool( 1, - new ThreadFactoryBuilder().setNameFormat("MysqlRegistrySubscribeDataCheckThread").setDaemon(true).build()); + new ThreadFactoryBuilder().setNameFormat("MysqlRegistrySubscribeDataCheckThread").setDaemon(true) + .build()); } public void start() { @@ -146,8 +147,8 @@ private void triggerListener(List dataList, Event.Type type) { for (MysqlRegistryData data : dataList) { if (data.getKey().startsWith(subscribeKey)) { - subscribeListeners.forEach(subscribeListener -> - subscribeListener.notify(new Event(data.getKey(), data.getKey(), data.getData(), type))); + subscribeListeners.forEach(subscribeListener -> subscribeListener + .notify(new Event(data.getKey(), data.getKey(), data.getData(), type))); } } } diff --git a/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-zookeeper/src/main/java/org/apache/dolphinscheduler/plugin/registry/zookeeper/ZookeeperConnectionStateListener.java b/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-zookeeper/src/main/java/org/apache/dolphinscheduler/plugin/registry/zookeeper/ZookeeperConnectionStateListener.java index c39f6a54b058..ca22f68bf25c 100644 --- a/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-zookeeper/src/main/java/org/apache/dolphinscheduler/plugin/registry/zookeeper/ZookeeperConnectionStateListener.java +++ b/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-zookeeper/src/main/java/org/apache/dolphinscheduler/plugin/registry/zookeeper/ZookeeperConnectionStateListener.java @@ -27,7 +27,7 @@ import org.slf4j.LoggerFactory; public final class ZookeeperConnectionStateListener implements ConnectionStateListener { - + private static final Logger logger = LoggerFactory.getLogger(ZookeeperConnectionStateListener.class); private final ConnectionListener listener; diff --git a/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-zookeeper/src/main/java/org/apache/dolphinscheduler/plugin/registry/zookeeper/ZookeeperRegistry.java b/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-zookeeper/src/main/java/org/apache/dolphinscheduler/plugin/registry/zookeeper/ZookeeperRegistry.java index 5aa99b51985d..a2741d4d9a17 100644 --- a/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-zookeeper/src/main/java/org/apache/dolphinscheduler/plugin/registry/zookeeper/ZookeeperRegistry.java +++ b/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-zookeeper/src/main/java/org/apache/dolphinscheduler/plugin/registry/zookeeper/ZookeeperRegistry.java @@ -17,8 +17,14 @@ package org.apache.dolphinscheduler.plugin.registry.zookeeper; -import com.google.common.base.Strings; -import lombok.NonNull; +import static java.util.concurrent.TimeUnit.MILLISECONDS; + +import org.apache.dolphinscheduler.registry.api.ConnectionListener; +import org.apache.dolphinscheduler.registry.api.Event; +import org.apache.dolphinscheduler.registry.api.Registry; +import org.apache.dolphinscheduler.registry.api.RegistryException; +import org.apache.dolphinscheduler.registry.api.SubscribeListener; + import org.apache.curator.framework.CuratorFramework; import org.apache.curator.framework.CuratorFrameworkFactory; import org.apache.curator.framework.api.ACLProvider; @@ -28,19 +34,11 @@ import org.apache.curator.framework.recipes.locks.InterProcessMutex; import org.apache.curator.retry.ExponentialBackoffRetry; import org.apache.curator.utils.CloseableUtils; -import org.apache.dolphinscheduler.registry.api.ConnectionListener; -import org.apache.dolphinscheduler.registry.api.Event; -import org.apache.dolphinscheduler.registry.api.Registry; -import org.apache.dolphinscheduler.registry.api.RegistryException; -import org.apache.dolphinscheduler.registry.api.SubscribeListener; import org.apache.zookeeper.CreateMode; import org.apache.zookeeper.KeeperException; import org.apache.zookeeper.ZooDefs; import org.apache.zookeeper.data.ACL; -import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; -import org.springframework.stereotype.Component; -import javax.annotation.PostConstruct; import java.nio.charset.StandardCharsets; import java.time.Duration; import java.util.Comparator; @@ -49,7 +47,14 @@ import java.util.Map; import java.util.concurrent.ConcurrentHashMap; -import static java.util.concurrent.TimeUnit.MILLISECONDS; +import javax.annotation.PostConstruct; + +import lombok.NonNull; + +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.stereotype.Component; + +import com.google.common.base.Strings; @Component @ConditionalOnProperty(prefix = "registry", name = "type", havingValue = "zookeeper") diff --git a/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-zookeeper/src/main/java/org/apache/dolphinscheduler/plugin/registry/zookeeper/ZookeeperRegistryProperties.java b/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-zookeeper/src/main/java/org/apache/dolphinscheduler/plugin/registry/zookeeper/ZookeeperRegistryProperties.java index 84026a34f05b..42dbc5d256f0 100644 --- a/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-zookeeper/src/main/java/org/apache/dolphinscheduler/plugin/registry/zookeeper/ZookeeperRegistryProperties.java +++ b/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-zookeeper/src/main/java/org/apache/dolphinscheduler/plugin/registry/zookeeper/ZookeeperRegistryProperties.java @@ -27,6 +27,7 @@ @ConditionalOnProperty(prefix = "registry", name = "type", havingValue = "zookeeper") @ConfigurationProperties(prefix = "registry") public class ZookeeperRegistryProperties { + private ZookeeperProperties zookeeper = new ZookeeperProperties(); public ZookeeperProperties getZookeeper() { @@ -38,6 +39,7 @@ public void setZookeeper(ZookeeperProperties zookeeper) { } public static final class ZookeeperProperties { + private String namespace; private String connectString; private RetryPolicy retryPolicy = new RetryPolicy(); @@ -103,6 +105,7 @@ public void setBlockUntilConnected(Duration blockUntilConnected) { } public static final class RetryPolicy { + private Duration baseSleepTime = Duration.ofMillis(60); private int maxRetries; private Duration maxSleep = Duration.ofMillis(300); diff --git a/dolphinscheduler-registry/dolphinscheduler-registry-plugins/pom.xml b/dolphinscheduler-registry/dolphinscheduler-registry-plugins/pom.xml index 8c3c108575b5..a2bcfa97756a 100644 --- a/dolphinscheduler-registry/dolphinscheduler-registry-plugins/pom.xml +++ b/dolphinscheduler-registry/dolphinscheduler-registry-plugins/pom.xml @@ -17,17 +17,15 @@ ~ specific language governing permissions and limitations ~ under the License. --> - - + 4.0.0 org.apache.dolphinscheduler dolphinscheduler-registry dev-SNAPSHOT dolphinscheduler-registry-plugins - 4.0.0 pom diff --git a/dolphinscheduler-registry/pom.xml b/dolphinscheduler-registry/pom.xml index e7119005508b..7a273dc1a8c2 100644 --- a/dolphinscheduler-registry/pom.xml +++ b/dolphinscheduler-registry/pom.xml @@ -15,35 +15,17 @@ ~ See the License for the specific language governing permissions and ~ limitations under the License. --> - + 4.0.0 - dolphinscheduler org.apache.dolphinscheduler + dolphinscheduler dev-SNAPSHOT - 4.0.0 dolphinscheduler-registry pom - - - org.springframework - spring-context - provided - - - org.springframework.boot - spring-boot-autoconfigure - provided - - - javax.annotation - javax.annotation-api - - - dolphinscheduler-registry-api dolphinscheduler-registry-plugins @@ -61,4 +43,21 @@ + + + + org.springframework + spring-context + provided + + + org.springframework.boot + spring-boot-autoconfigure + provided + + + javax.annotation + javax.annotation-api + + diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/NettyRemotingClient.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/NettyRemotingClient.java index d47d68fa3d16..5ac59180442e 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/NettyRemotingClient.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/NettyRemotingClient.java @@ -17,17 +17,6 @@ package org.apache.dolphinscheduler.remote; -import io.netty.bootstrap.Bootstrap; -import io.netty.channel.Channel; -import io.netty.channel.ChannelFuture; -import io.netty.channel.ChannelInitializer; -import io.netty.channel.ChannelOption; -import io.netty.channel.EventLoopGroup; -import io.netty.channel.epoll.Epoll; -import io.netty.channel.epoll.EpollEventLoopGroup; -import io.netty.channel.nio.NioEventLoopGroup; -import io.netty.channel.socket.SocketChannel; -import io.netty.handler.timeout.IdleStateHandler; import org.apache.dolphinscheduler.remote.codec.NettyDecoder; import org.apache.dolphinscheduler.remote.codec.NettyEncoder; import org.apache.dolphinscheduler.remote.command.Command; @@ -46,8 +35,6 @@ import org.apache.dolphinscheduler.remote.utils.Host; import org.apache.dolphinscheduler.remote.utils.NamedThreadFactory; import org.apache.dolphinscheduler.remote.utils.NettyUtils; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import java.net.InetSocketAddress; import java.util.concurrent.ConcurrentHashMap; @@ -60,6 +47,21 @@ import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import io.netty.bootstrap.Bootstrap; +import io.netty.channel.Channel; +import io.netty.channel.ChannelFuture; +import io.netty.channel.ChannelInitializer; +import io.netty.channel.ChannelOption; +import io.netty.channel.EventLoopGroup; +import io.netty.channel.epoll.Epoll; +import io.netty.channel.epoll.EpollEventLoopGroup; +import io.netty.channel.nio.NioEventLoopGroup; +import io.netty.channel.socket.SocketChannel; +import io.netty.handler.timeout.IdleStateHandler; + public class NettyRemotingClient implements AutoCloseable { private final Logger logger = LoggerFactory.getLogger(NettyRemotingClient.class); @@ -87,9 +89,11 @@ public class NettyRemotingClient implements AutoCloseable { public NettyRemotingClient(final NettyClientConfig clientConfig) { this.clientConfig = clientConfig; if (Epoll.isAvailable()) { - this.workerGroup = new EpollEventLoopGroup(clientConfig.getWorkerThreads(), new NamedThreadFactory("NettyClient")); + this.workerGroup = + new EpollEventLoopGroup(clientConfig.getWorkerThreads(), new NamedThreadFactory("NettyClient")); } else { - this.workerGroup = new NioEventLoopGroup(clientConfig.getWorkerThreads(), new NamedThreadFactory("NettyClient")); + this.workerGroup = + new NioEventLoopGroup(clientConfig.getWorkerThreads(), new NamedThreadFactory("NettyClient")); } this.callbackExecutor = new ThreadPoolExecutor( Constants.CPUS, @@ -101,7 +105,8 @@ public NettyRemotingClient(final NettyClientConfig clientConfig) { new CallerThreadExecutePolicy()); this.clientHandler = new NettyClientHandler(this, callbackExecutor); - this.responseFutureExecutor = Executors.newSingleThreadScheduledExecutor(new NamedThreadFactory("ResponseFutureExecutor")); + this.responseFutureExecutor = + Executors.newSingleThreadScheduledExecutor(new NamedThreadFactory("ResponseFutureExecutor")); this.start(); } @@ -117,14 +122,18 @@ private void start() { .option(ChannelOption.SO_RCVBUF, clientConfig.getReceiveBufferSize()) .option(ChannelOption.CONNECT_TIMEOUT_MILLIS, clientConfig.getConnectTimeoutMillis()) .handler(new ChannelInitializer() { + @Override public void initChannel(SocketChannel ch) { ch.pipeline() - .addLast("client-idle-handler", new IdleStateHandler(Constants.NETTY_CLIENT_HEART_BEAT_TIME, 0, 0, TimeUnit.MILLISECONDS)) + .addLast("client-idle-handler", + new IdleStateHandler(Constants.NETTY_CLIENT_HEART_BEAT_TIME, 0, 0, + TimeUnit.MILLISECONDS)) .addLast(new NettyDecoder(), clientHandler, encoder); } }); - this.responseFutureExecutor.scheduleAtFixedRate(ResponseFuture::scanFutureTable, 5000, 1000, TimeUnit.MILLISECONDS); + this.responseFutureExecutor.scheduleAtFixedRate(ResponseFuture::scanFutureTable, 5000, 1000, + TimeUnit.MILLISECONDS); isStarted.compareAndSet(false, true); } @@ -148,14 +157,14 @@ public void sendAsync(final Host host, final Command command, */ final long opaque = command.getOpaque(); /* - * control concurrency number + * control concurrency number */ boolean acquired = this.asyncSemaphore.tryAcquire(timeoutMillis, TimeUnit.MILLISECONDS); if (acquired) { final ReleaseSemaphore releaseSemaphore = new ReleaseSemaphore(this.asyncSemaphore); /* - * response future + * response future */ final ResponseFuture responseFuture = new ResponseFuture(opaque, timeoutMillis, @@ -184,7 +193,8 @@ public void sendAsync(final Host host, final Command command, throw new RemotingException(String.format("send command to host: %s failed", host), ex); } } else { - String message = String.format("try to acquire async semaphore timeout: %d, waiting thread num: %d, total permits: %d", + String message = String.format( + "try to acquire async semaphore timeout: %d, waiting thread num: %d, total permits: %d", timeoutMillis, asyncSemaphore.getQueueLength(), asyncSemaphore.availablePermits()); throw new RemotingTooMuchRequestException(message); } @@ -198,7 +208,8 @@ public void sendAsync(final Host host, final Command command, * @param timeoutMillis timeoutMillis * @return command */ - public Command sendSync(final Host host, final Command command, final long timeoutMillis) throws InterruptedException, RemotingException { + public Command sendSync(final Host host, final Command command, + final long timeoutMillis) throws InterruptedException, RemotingException { final Channel channel = getChannel(host); if (channel == null) { throw new RemotingException(String.format("connect to : %s fail", host)); @@ -254,7 +265,8 @@ public void send(final Host host, final Command command) throws RemotingExceptio throw remotingException; } catch (Exception e) { logger.error("Send command {} to address {} encounter error.", command, host.getAddress()); - throw new RemotingException(String.format("Send command : %s , to :%s encounter error", command, host.getAddress()), e); + throw new RemotingException( + String.format("Send command : %s , to :%s encounter error", command, host.getAddress()), e); } } @@ -275,7 +287,8 @@ public void registerProcessor(final CommandType commandType, final NettyRequestP * @param processor processor * @param executor thread executor */ - public void registerProcessor(final CommandType commandType, final NettyRequestProcessor processor, final ExecutorService executor) { + public void registerProcessor(final CommandType commandType, final NettyRequestProcessor processor, + final ExecutorService executor) { this.clientHandler.registerProcessor(commandType, processor, executor); } diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/NettyRemotingServer.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/NettyRemotingServer.java index 098751ab6549..63c1392a06aa 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/NettyRemotingServer.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/NettyRemotingServer.java @@ -103,8 +103,10 @@ public class NettyRemotingServer { */ public NettyRemotingServer(final NettyServerConfig serverConfig) { this.serverConfig = serverConfig; - ThreadFactory bossThreadFactory = new ThreadFactoryBuilder().setDaemon(true).setNameFormat("NettyServerBossThread_%s").build(); - ThreadFactory workerThreadFactory = new ThreadFactoryBuilder().setDaemon(true).setNameFormat("NettyServerWorkerThread_%s").build(); + ThreadFactory bossThreadFactory = + new ThreadFactoryBuilder().setDaemon(true).setNameFormat("NettyServerBossThread_%s").build(); + ThreadFactory workerThreadFactory = + new ThreadFactoryBuilder().setDaemon(true).setNameFormat("NettyServerWorkerThread_%s").build(); if (Epoll.isAvailable()) { this.bossGroup = new EpollEventLoopGroup(1, bossThreadFactory); this.workGroup = new EpollEventLoopGroup(serverConfig.getWorkerThread(), workerThreadFactory); @@ -146,7 +148,8 @@ protected void initChannel(SocketChannel ch) { if (future.isSuccess()) { logger.info("NettyRemotingServer bind success at port : {}", serverConfig.getListenPort()); } else if (future.cause() != null) { - throw new RemoteException(String.format(NETTY_BIND_FAILURE_MSG, serverConfig.getListenPort()), future.cause()); + throw new RemoteException(String.format(NETTY_BIND_FAILURE_MSG, serverConfig.getListenPort()), + future.cause()); } else { throw new RemoteException(String.format(NETTY_BIND_FAILURE_MSG, serverConfig.getListenPort())); } @@ -162,7 +165,8 @@ private void initNettyChannel(SocketChannel ch) { ch.pipeline() .addLast("encoder", new NettyEncoder()) .addLast("decoder", new NettyDecoder()) - .addLast("server-idle-handle", new IdleStateHandler(0, 0, Constants.NETTY_SERVER_HEART_BEAT_TIME, TimeUnit.MILLISECONDS)) + .addLast("server-idle-handle", + new IdleStateHandler(0, 0, Constants.NETTY_SERVER_HEART_BEAT_TIME, TimeUnit.MILLISECONDS)) .addLast("handler", serverHandler); } @@ -183,7 +187,8 @@ public void registerProcessor(final CommandType commandType, final NettyRequestP * @param processor processor * @param executor thread executor */ - public void registerProcessor(final CommandType commandType, final NettyRequestProcessor processor, final ExecutorService executor) { + public void registerProcessor(final CommandType commandType, final NettyRequestProcessor processor, + final ExecutorService executor) { this.serverHandler.registerProcessor(commandType, processor, executor); } diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/codec/NettyDecoder.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/codec/NettyDecoder.java index 84b5c1f90b6e..4066b6ed10a7 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/codec/NettyDecoder.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/codec/NettyDecoder.java @@ -35,6 +35,7 @@ * netty decoder */ public class NettyDecoder extends ReplayingDecoder { + private static final Logger logger = LoggerFactory.getLogger(NettyDecoder.class); public NettyDecoder() { diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/codec/NettyEncoder.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/codec/NettyEncoder.java index 04487ff6865c..c1d6e0fcd2aa 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/codec/NettyEncoder.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/codec/NettyEncoder.java @@ -58,4 +58,3 @@ private void writeContext(Command msg, ByteBuf out) { out.writeBytes(headerBytes); } } - diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/Command.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/Command.java index 527a269411c8..28c36e56fb12 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/Command.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/Command.java @@ -33,11 +33,11 @@ public class Command implements Serializable { public static final byte MAGIC = (byte) 0xbabe; public static final byte VERSION = 0; - public Command(){ + public Command() { this.opaque = REQUEST_ID.getAndIncrement(); } - public Command(long opaque){ + public Command(long opaque) { this.opaque = opaque; } diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/CommandType.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/CommandType.java index b83c9ec53cc7..920332f4a57f 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/CommandType.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/CommandType.java @@ -38,7 +38,6 @@ public enum CommandType { GET_LOG_BYTES_RESPONSE, - WORKER_REQUEST, MASTER_RESPONSE, diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/HostUpdateCommand.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/HostUpdateCommand.java index f265b758b443..b55453b2068e 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/HostUpdateCommand.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/HostUpdateCommand.java @@ -17,11 +17,12 @@ package org.apache.dolphinscheduler.remote.command; -import lombok.Data; import org.apache.dolphinscheduler.common.utils.JSONUtils; import java.io.Serializable; +import lombok.Data; + /** * process host update */ diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/StateEventResponseCommand.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/StateEventResponseCommand.java index b98e5dad0bee..6b21e1acd69d 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/StateEventResponseCommand.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/StateEventResponseCommand.java @@ -17,12 +17,13 @@ package org.apache.dolphinscheduler.remote.command; -import lombok.AllArgsConstructor; -import lombok.Data; import org.apache.dolphinscheduler.common.utils.JSONUtils; import java.io.Serializable; +import lombok.AllArgsConstructor; +import lombok.Data; + /** * db task final result response command */ diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/TaskDispatchCommand.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/TaskDispatchCommand.java index f04c6107c1c2..bf9abe8c9689 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/TaskDispatchCommand.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/TaskDispatchCommand.java @@ -25,7 +25,6 @@ import lombok.NoArgsConstructor; import lombok.ToString; - /** * The task dispatch message, means dispatch a task to worker. */ diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/TaskEventChangeCommand.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/TaskEventChangeCommand.java index bf0ff595d66d..adfcaa105a2b 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/TaskEventChangeCommand.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/TaskEventChangeCommand.java @@ -17,12 +17,13 @@ package org.apache.dolphinscheduler.remote.command; -import lombok.Data; -import lombok.NoArgsConstructor; import org.apache.dolphinscheduler.common.utils.JSONUtils; import java.io.Serializable; +import lombok.Data; +import lombok.NoArgsConstructor; + /** * db task final result response command */ diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/TaskExecuteResultCommand.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/TaskExecuteResultCommand.java index fb24438426a8..42d99672e0d9 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/TaskExecuteResultCommand.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/TaskExecuteResultCommand.java @@ -19,8 +19,6 @@ import org.apache.dolphinscheduler.common.utils.JSONUtils; -import java.util.Date; - import lombok.Data; import lombok.EqualsAndHashCode; import lombok.NoArgsConstructor; @@ -74,13 +72,11 @@ public TaskExecuteResultCommand(String messageSenderAddress, String messageRecei */ private String executePath; - /** * end time */ private long endTime; - /** * processId */ diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/TaskExecuteRunningAckMessage.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/TaskExecuteRunningAckMessage.java index ea0da08cff83..120867977b62 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/TaskExecuteRunningAckMessage.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/TaskExecuteRunningAckMessage.java @@ -17,13 +17,14 @@ package org.apache.dolphinscheduler.remote.command; +import org.apache.dolphinscheduler.common.utils.JSONUtils; + +import java.io.Serializable; + import lombok.AllArgsConstructor; import lombok.Builder; import lombok.Data; import lombok.NoArgsConstructor; -import org.apache.dolphinscheduler.common.utils.JSONUtils; - -import java.io.Serializable; /** * task execute running ack command diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/TaskExecuteRunningCommand.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/TaskExecuteRunningCommand.java index ed0231b374a3..c02ea64849a8 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/TaskExecuteRunningCommand.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/TaskExecuteRunningCommand.java @@ -18,12 +18,12 @@ package org.apache.dolphinscheduler.remote.command; import org.apache.dolphinscheduler.common.utils.JSONUtils; +import org.apache.dolphinscheduler.plugin.task.api.enums.TaskExecutionStatus; import lombok.Data; import lombok.EqualsAndHashCode; import lombok.NoArgsConstructor; import lombok.ToString; -import org.apache.dolphinscheduler.plugin.task.api.enums.TaskExecutionStatus; /** * Task running message, means the task is running in worker. diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/TaskKillResponseCommand.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/TaskKillResponseCommand.java index ba7fad83682c..7f92b415436b 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/TaskKillResponseCommand.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/TaskKillResponseCommand.java @@ -17,16 +17,17 @@ package org.apache.dolphinscheduler.remote.command; -import lombok.AllArgsConstructor; -import lombok.Builder; -import lombok.Data; -import lombok.NoArgsConstructor; import org.apache.dolphinscheduler.common.utils.JSONUtils; import org.apache.dolphinscheduler.plugin.task.api.enums.TaskExecutionStatus; import java.io.Serializable; import java.util.List; +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; + /** * kill task response command */ diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/TaskSavePointResponseCommand.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/TaskSavePointResponseCommand.java index c126a4505579..55eb272eb3d6 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/TaskSavePointResponseCommand.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/TaskSavePointResponseCommand.java @@ -17,14 +17,14 @@ package org.apache.dolphinscheduler.remote.command; -import lombok.AllArgsConstructor; -import lombok.Data; -import lombok.NoArgsConstructor; - import org.apache.dolphinscheduler.common.utils.JSONUtils; import java.io.Serializable; +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; + /** * task savepoint response command */ diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/WorkflowStateEventChangeCommand.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/WorkflowStateEventChangeCommand.java index 5db49f85009e..def9f663edd1 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/WorkflowStateEventChangeCommand.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/WorkflowStateEventChangeCommand.java @@ -17,13 +17,14 @@ package org.apache.dolphinscheduler.remote.command; -import lombok.Data; -import lombok.NoArgsConstructor; import org.apache.dolphinscheduler.common.enums.WorkflowExecutionStatus; import org.apache.dolphinscheduler.common.utils.JSONUtils; import java.io.Serializable; +import lombok.Data; +import lombok.NoArgsConstructor; + /** * db task final result response command */ diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/alert/AlertSendRequestCommand.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/alert/AlertSendRequestCommand.java index 2acf46d2534f..5bc4154d9c14 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/alert/AlertSendRequestCommand.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/alert/AlertSendRequestCommand.java @@ -17,15 +17,16 @@ package org.apache.dolphinscheduler.remote.command.alert; -import lombok.AllArgsConstructor; -import lombok.Data; -import lombok.NoArgsConstructor; import org.apache.dolphinscheduler.remote.command.Command; import org.apache.dolphinscheduler.remote.command.CommandType; import org.apache.dolphinscheduler.remote.utils.JsonSerializer; import java.io.Serializable; +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; + @Data @NoArgsConstructor @AllArgsConstructor diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/alert/AlertSendResponseCommand.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/alert/AlertSendResponseCommand.java index 9c5e62781b03..6d729e753959 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/alert/AlertSendResponseCommand.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/alert/AlertSendResponseCommand.java @@ -17,9 +17,6 @@ package org.apache.dolphinscheduler.remote.command.alert; -import lombok.AllArgsConstructor; -import lombok.Data; -import lombok.NoArgsConstructor; import org.apache.dolphinscheduler.remote.command.Command; import org.apache.dolphinscheduler.remote.command.CommandType; import org.apache.dolphinscheduler.remote.utils.JsonSerializer; @@ -27,6 +24,10 @@ import java.io.Serializable; import java.util.List; +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; + @Data @NoArgsConstructor @AllArgsConstructor diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/alert/AlertSendResponseResult.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/alert/AlertSendResponseResult.java index c0867ee7aebc..7b2fd98edd5b 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/alert/AlertSendResponseResult.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/alert/AlertSendResponseResult.java @@ -17,12 +17,12 @@ package org.apache.dolphinscheduler.remote.command.alert; +import java.io.Serializable; + import lombok.AllArgsConstructor; import lombok.Data; import lombok.NoArgsConstructor; -import java.io.Serializable; - @Data @NoArgsConstructor @AllArgsConstructor diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/log/GetAppIdRequestCommand.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/log/GetAppIdRequestCommand.java index 26412b8283b1..91706feb8346 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/log/GetAppIdRequestCommand.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/log/GetAppIdRequestCommand.java @@ -17,15 +17,16 @@ package org.apache.dolphinscheduler.remote.command.log; -import lombok.AllArgsConstructor; -import lombok.Data; -import lombok.NoArgsConstructor; import org.apache.dolphinscheduler.common.utils.JSONUtils; import org.apache.dolphinscheduler.remote.command.Command; import org.apache.dolphinscheduler.remote.command.CommandType; import java.io.Serializable; +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; + @Data @AllArgsConstructor @NoArgsConstructor diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/log/GetAppIdResponseCommand.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/log/GetAppIdResponseCommand.java index 2e54a9008d43..0952cfb1ba46 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/log/GetAppIdResponseCommand.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/log/GetAppIdResponseCommand.java @@ -17,9 +17,6 @@ package org.apache.dolphinscheduler.remote.command.log; -import lombok.AllArgsConstructor; -import lombok.Data; -import lombok.NoArgsConstructor; import org.apache.dolphinscheduler.common.utils.JSONUtils; import org.apache.dolphinscheduler.remote.command.Command; import org.apache.dolphinscheduler.remote.command.CommandType; @@ -27,6 +24,10 @@ import java.io.Serializable; import java.util.List; +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; + @Data @AllArgsConstructor @NoArgsConstructor diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/log/GetLogBytesRequestCommand.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/log/GetLogBytesRequestCommand.java index 0022706e99cf..de73e268610b 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/log/GetLogBytesRequestCommand.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/log/GetLogBytesRequestCommand.java @@ -17,15 +17,16 @@ package org.apache.dolphinscheduler.remote.command.log; -import lombok.AllArgsConstructor; -import lombok.Data; -import lombok.NoArgsConstructor; import org.apache.dolphinscheduler.common.utils.JSONUtils; import org.apache.dolphinscheduler.remote.command.Command; import org.apache.dolphinscheduler.remote.command.CommandType; import java.io.Serializable; +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; + /** * get log bytes request command */ diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/log/GetLogBytesResponseCommand.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/log/GetLogBytesResponseCommand.java index 30072e3151fe..3ba4cf3b58dd 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/log/GetLogBytesResponseCommand.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/log/GetLogBytesResponseCommand.java @@ -17,15 +17,16 @@ package org.apache.dolphinscheduler.remote.command.log; -import lombok.AllArgsConstructor; -import lombok.Data; -import lombok.NoArgsConstructor; import org.apache.dolphinscheduler.common.utils.JSONUtils; import org.apache.dolphinscheduler.remote.command.Command; import org.apache.dolphinscheduler.remote.command.CommandType; import java.io.Serializable; +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; + /** * get log bytes response command */ diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/log/RemoveTaskLogRequestCommand.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/log/RemoveTaskLogRequestCommand.java index 93efc253186a..8dc8f274d864 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/log/RemoveTaskLogRequestCommand.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/log/RemoveTaskLogRequestCommand.java @@ -17,15 +17,16 @@ package org.apache.dolphinscheduler.remote.command.log; -import lombok.AllArgsConstructor; -import lombok.Data; -import lombok.NoArgsConstructor; import org.apache.dolphinscheduler.common.utils.JSONUtils; import org.apache.dolphinscheduler.remote.command.Command; import org.apache.dolphinscheduler.remote.command.CommandType; import java.io.Serializable; +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; + /** * remove task log request command */ diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/log/RemoveTaskLogResponseCommand.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/log/RemoveTaskLogResponseCommand.java index 431a52066d32..6e0ae320a6e9 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/log/RemoveTaskLogResponseCommand.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/log/RemoveTaskLogResponseCommand.java @@ -17,15 +17,16 @@ package org.apache.dolphinscheduler.remote.command.log; -import lombok.AllArgsConstructor; -import lombok.Data; -import lombok.NoArgsConstructor; import org.apache.dolphinscheduler.common.utils.JSONUtils; import org.apache.dolphinscheduler.remote.command.Command; import org.apache.dolphinscheduler.remote.command.CommandType; import java.io.Serializable; +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; + /** * remove task log request command */ diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/log/RollViewLogRequestCommand.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/log/RollViewLogRequestCommand.java index 4d11836f4597..e4075f6cf76b 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/log/RollViewLogRequestCommand.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/log/RollViewLogRequestCommand.java @@ -17,15 +17,16 @@ package org.apache.dolphinscheduler.remote.command.log; -import lombok.AllArgsConstructor; -import lombok.Data; -import lombok.NoArgsConstructor; import org.apache.dolphinscheduler.common.utils.JSONUtils; import org.apache.dolphinscheduler.remote.command.Command; import org.apache.dolphinscheduler.remote.command.CommandType; import java.io.Serializable; +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; + /** * roll view log request command */ diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/log/RollViewLogResponseCommand.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/log/RollViewLogResponseCommand.java index 4fea16d912b9..71be7a0cf3a5 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/log/RollViewLogResponseCommand.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/log/RollViewLogResponseCommand.java @@ -17,15 +17,16 @@ package org.apache.dolphinscheduler.remote.command.log; -import lombok.AllArgsConstructor; -import lombok.Data; -import lombok.NoArgsConstructor; import org.apache.dolphinscheduler.common.utils.JSONUtils; import org.apache.dolphinscheduler.remote.command.Command; import org.apache.dolphinscheduler.remote.command.CommandType; import java.io.Serializable; +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; + /** * roll view log response command */ diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/log/ViewLogRequestCommand.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/log/ViewLogRequestCommand.java index 6eae191128f8..a4b3c4749be0 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/log/ViewLogRequestCommand.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/log/ViewLogRequestCommand.java @@ -17,15 +17,16 @@ package org.apache.dolphinscheduler.remote.command.log; -import lombok.AllArgsConstructor; -import lombok.Data; -import lombok.NoArgsConstructor; import org.apache.dolphinscheduler.common.utils.JSONUtils; import org.apache.dolphinscheduler.remote.command.Command; import org.apache.dolphinscheduler.remote.command.CommandType; import java.io.Serializable; +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; + /** * view log request command */ diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/log/ViewLogResponseCommand.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/log/ViewLogResponseCommand.java index 52a1b174108e..e9a368ad7c33 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/log/ViewLogResponseCommand.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/log/ViewLogResponseCommand.java @@ -17,15 +17,16 @@ package org.apache.dolphinscheduler.remote.command.log; -import lombok.AllArgsConstructor; -import lombok.Data; -import lombok.NoArgsConstructor; import org.apache.dolphinscheduler.common.utils.JSONUtils; import org.apache.dolphinscheduler.remote.command.Command; import org.apache.dolphinscheduler.remote.command.CommandType; import java.io.Serializable; +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; + /** * view log response command */ diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/config/NettyClientConfig.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/config/NettyClientConfig.java index 18ee88f696ae..ff41e9b10156 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/config/NettyClientConfig.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/config/NettyClientConfig.java @@ -17,11 +17,12 @@ package org.apache.dolphinscheduler.remote.config; +import org.apache.dolphinscheduler.remote.utils.Constants; + import lombok.AllArgsConstructor; import lombok.Builder; import lombok.Data; import lombok.NoArgsConstructor; -import org.apache.dolphinscheduler.remote.utils.Constants; @Data @Builder diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/dto/TaskInstanceExecuteDto.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/dto/TaskInstanceExecuteDto.java index 1779da3270a3..0bffe9272b12 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/dto/TaskInstanceExecuteDto.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/dto/TaskInstanceExecuteDto.java @@ -19,15 +19,14 @@ import org.apache.dolphinscheduler.common.enums.Flag; import org.apache.dolphinscheduler.common.enums.Priority; +import org.apache.dolphinscheduler.common.enums.TaskExecuteType; +import org.apache.dolphinscheduler.plugin.task.api.enums.TaskExecutionStatus; import java.util.Date; import java.util.Map; import lombok.Data; -import org.apache.dolphinscheduler.common.enums.TaskExecuteType; -import org.apache.dolphinscheduler.plugin.task.api.enums.TaskExecutionStatus; - @Data public class TaskInstanceExecuteDto { diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/dto/WorkflowExecuteDto.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/dto/WorkflowExecuteDto.java index 3eac76da7a69..a2b5d54f61e4 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/dto/WorkflowExecuteDto.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/dto/WorkflowExecuteDto.java @@ -17,7 +17,13 @@ package org.apache.dolphinscheduler.remote.dto; -import org.apache.dolphinscheduler.common.enums.*; +import org.apache.dolphinscheduler.common.enums.CommandType; +import org.apache.dolphinscheduler.common.enums.FailureStrategy; +import org.apache.dolphinscheduler.common.enums.Flag; +import org.apache.dolphinscheduler.common.enums.Priority; +import org.apache.dolphinscheduler.common.enums.TaskDependType; +import org.apache.dolphinscheduler.common.enums.WarningType; +import org.apache.dolphinscheduler.common.enums.WorkflowExecutionStatus; import java.util.Collection; import java.util.Date; diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/exceptions/RemotingException.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/exceptions/RemotingException.java index d774dc8b9eb3..1d025cc2683c 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/exceptions/RemotingException.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/exceptions/RemotingException.java @@ -55,5 +55,4 @@ public RemotingException(Throwable cause) { super(cause); } - } diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/exceptions/RemotingTimeoutException.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/exceptions/RemotingTimeoutException.java index 3d91ba57f6bc..5788221d6c58 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/exceptions/RemotingTimeoutException.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/exceptions/RemotingTimeoutException.java @@ -17,17 +17,15 @@ package org.apache.dolphinscheduler.remote.exceptions; - /** * timeout exception */ -public class RemotingTimeoutException extends RemotingException{ +public class RemotingTimeoutException extends RemotingException { public RemotingTimeoutException(String message) { super(message); } - public RemotingTimeoutException(String address, long timeoutMillis) { this(address, timeoutMillis, null); } diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/exceptions/RemotingTooMuchRequestException.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/exceptions/RemotingTooMuchRequestException.java index 82cc3f4dbf21..4e3149731191 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/exceptions/RemotingTooMuchRequestException.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/exceptions/RemotingTooMuchRequestException.java @@ -19,7 +19,7 @@ /** * too much request exception */ -public class RemotingTooMuchRequestException extends RemotingException{ +public class RemotingTooMuchRequestException extends RemotingException { public RemotingTooMuchRequestException(String message) { super(message); diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/future/ReleaseSemaphore.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/future/ReleaseSemaphore.java index 95a04b1f1a56..c8284890f928 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/future/ReleaseSemaphore.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/future/ReleaseSemaphore.java @@ -28,13 +28,13 @@ public class ReleaseSemaphore { private final AtomicBoolean released; - public ReleaseSemaphore(Semaphore semaphore){ + public ReleaseSemaphore(Semaphore semaphore) { this.semaphore = semaphore; this.released = new AtomicBoolean(false); } - public void release(){ - if(this.released.compareAndSet(false, true)){ + public void release() { + if (this.released.compareAndSet(false, true)) { this.semaphore.release(); } } diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/future/ResponseFuture.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/future/ResponseFuture.java index 5e5462c7e851..4e4ae1456c4a 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/future/ResponseFuture.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/future/ResponseFuture.java @@ -72,7 +72,8 @@ public class ResponseFuture { private Throwable cause; - public ResponseFuture(long opaque, long timeoutMillis, InvokeCallback invokeCallback, ReleaseSemaphore releaseSemaphore) { + public ResponseFuture(long opaque, long timeoutMillis, InvokeCallback invokeCallback, + ReleaseSemaphore releaseSemaphore) { this.opaque = opaque; this.timeoutMillis = timeoutMillis; this.invokeCallback = invokeCallback; diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/handler/NettyClientHandler.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/handler/NettyClientHandler.java index ad9787d705e8..56c9fc70e4fd 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/handler/NettyClientHandler.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/handler/NettyClientHandler.java @@ -117,7 +117,8 @@ public void registerProcessor(final CommandType commandType, final NettyRequestP * @param processor processor * @param executor thread executor */ - public void registerProcessor(final CommandType commandType, final NettyRequestProcessor processor, final ExecutorService executor) { + public void registerProcessor(final CommandType commandType, final NettyRequestProcessor processor, + final ExecutorService executor) { ExecutorService executorRef = executor; if (executorRef == null) { executorRef = defaultExecutor; @@ -159,7 +160,8 @@ public void processByCommandType(final Channel channel, final Command command) { try { pair.getRight().submit(run); } catch (RejectedExecutionException e) { - logger.warn("thread pool is full, discard command {} from {}", command, ChannelUtils.getRemoteAddress(channel)); + logger.warn("thread pool is full, discard command {} from {}", command, + ChannelUtils.getRemoteAddress(channel)); } } else { logger.warn("receive response {}, but not matched any request ", command); diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/handler/NettyServerHandler.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/handler/NettyServerHandler.java index 2be00c914aa8..8ae04a70b670 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/handler/NettyServerHandler.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/handler/NettyServerHandler.java @@ -38,7 +38,6 @@ import io.netty.channel.ChannelInboundHandlerAdapter; import io.netty.handler.timeout.IdleStateEvent; - /** * netty server request handler */ @@ -55,7 +54,8 @@ public class NettyServerHandler extends ChannelInboundHandlerAdapter { /** * server processors queue */ - private final ConcurrentHashMap> processors = new ConcurrentHashMap<>(); + private final ConcurrentHashMap> processors = + new ConcurrentHashMap<>(); public NettyServerHandler(NettyRemotingServer nettyRemotingServer) { this.nettyRemotingServer = nettyRemotingServer; @@ -100,7 +100,8 @@ public void registerProcessor(final CommandType commandType, final NettyRequestP * @param processor processor * @param executor thread executor */ - public void registerProcessor(final CommandType commandType, final NettyRequestProcessor processor, final ExecutorService executor) { + public void registerProcessor(final CommandType commandType, final NettyRequestProcessor processor, + final ExecutorService executor) { ExecutorService executorRef = executor; if (executorRef == null) { executorRef = nettyRemotingServer.getDefaultExecutor(); diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/processor/NettyRemoteChannel.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/processor/NettyRemoteChannel.java index 0d623e398a52..4609c6d239fa 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/processor/NettyRemoteChannel.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/processor/NettyRemoteChannel.java @@ -44,7 +44,6 @@ public class NettyRemoteChannel { */ private final Host host; - public NettyRemoteChannel(Channel channel, long opaque) { this.channel = channel; this.host = ChannelUtils.toAddress(channel); @@ -69,7 +68,7 @@ public Host getHost() { return host; } - public boolean isActive(){ + public boolean isActive() { return this.channel.isActive(); } @@ -77,7 +76,7 @@ public ChannelFuture writeAndFlush(Command command) { return this.channel.writeAndFlush(command); } - public void close(){ + public void close() { this.channel.close(); } } diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/processor/NettyRequestProcessor.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/processor/NettyRequestProcessor.java index 6966b53d1755..d3e37f02ee05 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/processor/NettyRequestProcessor.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/processor/NettyRequestProcessor.java @@ -16,9 +16,10 @@ */ package org.apache.dolphinscheduler.remote.processor; -import io.netty.channel.Channel; import org.apache.dolphinscheduler.remote.command.Command; +import io.netty.channel.Channel; + /** * netty request processor */ diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/processor/StateEventCallbackService.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/processor/StateEventCallbackService.java index 32a0f4673da7..e35fb44d4162 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/processor/StateEventCallbackService.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/processor/StateEventCallbackService.java @@ -141,8 +141,7 @@ public Command sendSync(Host host, Command requestCommand) { Thread.currentThread().interrupt(); } catch (RemotingException e) { logger.error("send sync fail, host:{}, command:{}", host, requestCommand, e); - } - finally { + } finally { this.nettyRemotingClient.closeChannel(host); } return null; diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/utils/CallerThreadExecutePolicy.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/utils/CallerThreadExecutePolicy.java index 048ea86acb5a..318e20d9674f 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/utils/CallerThreadExecutePolicy.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/utils/CallerThreadExecutePolicy.java @@ -17,12 +17,12 @@ package org.apache.dolphinscheduler.remote.utils; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - import java.util.concurrent.RejectedExecutionHandler; import java.util.concurrent.ThreadPoolExecutor; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + /** * caller thread execute */ diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/utils/Constants.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/utils/Constants.java index 6748db03565a..8611dda54ce6 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/utils/Constants.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/utils/Constants.java @@ -49,7 +49,6 @@ private Constants() { */ public static final int CPUS = Runtime.getRuntime().availableProcessors(); - public static final String LOCAL_ADDRESS = NetUtils.getHost(); /** diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/utils/JsonSerializer.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/utils/JsonSerializer.java index e183289f739b..e860a72b3bcb 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/utils/JsonSerializer.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/utils/JsonSerializer.java @@ -30,6 +30,7 @@ * json serialize or deserialize */ public class JsonSerializer { + private static final ObjectMapper objectMapper = new ObjectMapper(); private static final Logger logger = LoggerFactory.getLogger(JsonSerializer.class); diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/utils/NamedThreadFactory.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/utils/NamedThreadFactory.java index be84f0f221bf..69f69c1ba088 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/utils/NamedThreadFactory.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/utils/NamedThreadFactory.java @@ -36,11 +36,11 @@ public class NamedThreadFactory implements ThreadFactory { */ private final int count; - public NamedThreadFactory(String name){ + public NamedThreadFactory(String name) { this(name, 0); } - public NamedThreadFactory(String name, int count){ + public NamedThreadFactory(String name, int count) { this.name = name; this.count = count; } diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/base/Rpc.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/base/Rpc.java index 655aa2730e7d..9857c9f4a340 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/base/Rpc.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/base/Rpc.java @@ -42,7 +42,7 @@ boolean callBack() default false; - //todo It is better to set the timeout period for synchronous calls + // todo It is better to set the timeout period for synchronous calls /** * When it is asynchronous transmission, callback must be set @@ -51,7 +51,4 @@ Class ackCallback() default AbstractRpcCallBack.class; - - - } diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/base/RpcService.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/base/RpcService.java index 5ebb2cfe7a63..81b16280e058 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/base/RpcService.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/base/RpcService.java @@ -25,5 +25,6 @@ @Target({ElementType.TYPE}) @Retention(RetentionPolicy.RUNTIME) public @interface RpcService { + String value() default ""; } diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/client/ConsumerInterceptor.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/client/ConsumerInterceptor.java index c001e5a753bd..d35edcdaea1b 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/client/ConsumerInterceptor.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/client/ConsumerInterceptor.java @@ -31,13 +31,13 @@ import java.lang.reflect.Method; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - import net.bytebuddy.implementation.bind.annotation.AllArguments; import net.bytebuddy.implementation.bind.annotation.Origin; import net.bytebuddy.implementation.bind.annotation.RuntimeType; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + /** * ConsumerInterceptor */ @@ -70,7 +70,7 @@ public Object intercept(@AllArguments Object[] args, @Origin Method method) thro while (retries-- > 0) { RpcResponse rsp; rsp = nettyClient.sendMsg(host, protocol, async); - //success + // success if (null != rsp && rsp.getStatus() == 0) { return rsp.getResult(); } diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/codec/NettyDecoder.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/codec/NettyDecoder.java index 813bf719f283..5263615994df 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/codec/NettyDecoder.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/codec/NettyDecoder.java @@ -42,7 +42,8 @@ public NettyDecoder(Class genericClass) { } @Override - protected void decode(ChannelHandlerContext channelHandlerContext, ByteBuf byteBuf, List list) throws Exception { + protected void decode(ChannelHandlerContext channelHandlerContext, ByteBuf byteBuf, + List list) throws Exception { if (byteBuf.readableBytes() < RpcProtocolConstants.HEADER_LENGTH) { return; } diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/codec/NettyEncoder.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/codec/NettyEncoder.java index 58328134b92b..d076e4760afb 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/codec/NettyEncoder.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/codec/NettyEncoder.java @@ -32,7 +32,8 @@ public class NettyEncoder extends MessageToByteEncoder> { @Override - protected void encode(ChannelHandlerContext channelHandlerContext, RpcProtocol msg, ByteBuf byteBuf) throws Exception { + protected void encode(ChannelHandlerContext channelHandlerContext, RpcProtocol msg, + ByteBuf byteBuf) throws Exception { MessageHeader msgHeader = msg.getMsgHeader(); byteBuf.writeShort(msgHeader.getMagic()); byteBuf.writeByte(msgHeader.getEventType()); diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/common/RequestEventType.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/common/RequestEventType.java index 558c88067bfe..5810993fc097 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/common/RequestEventType.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/common/RequestEventType.java @@ -19,9 +19,8 @@ public enum RequestEventType { - HEARTBEAT((byte)1,"heartbeat"), - BUSINESS((byte)2,"business request"); - + HEARTBEAT((byte) 1, "heartbeat"), + BUSINESS((byte) 2, "business request"); private Byte type; diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/common/ResponseEventType.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/common/ResponseEventType.java index 66ca157e2340..26f44814ede3 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/common/ResponseEventType.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/common/ResponseEventType.java @@ -1,19 +1,19 @@ package org.apache.dolphinscheduler.rpc.common;/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ public enum ResponseEventType { diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/common/ThreadPoolManager.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/common/ThreadPoolManager.java index 1ddd574e0d30..1a2ce2efeed4 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/common/ThreadPoolManager.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/common/ThreadPoolManager.java @@ -33,7 +33,8 @@ public enum ThreadPoolManager { private static final long KEEP_ALIVE_TIME = 60; ThreadPoolManager() { - executorService = new ThreadPoolExecutor(Runtime.getRuntime().availableProcessors() * 2, Runtime.getRuntime().availableProcessors() * 4, KEEP_ALIVE_TIME, TimeUnit.SECONDS, + executorService = new ThreadPoolExecutor(Runtime.getRuntime().availableProcessors() * 2, + Runtime.getRuntime().availableProcessors() * 4, KEEP_ALIVE_TIME, TimeUnit.SECONDS, new ArrayBlockingQueue<>(WORK_QUEUE_SIZE), new DiscardPolicy()); } diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/protocol/EventType.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/protocol/EventType.java index e6a85d3b8d88..224ab7b869b6 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/protocol/EventType.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/protocol/EventType.java @@ -19,9 +19,9 @@ public enum EventType { - HEARTBEAT((byte)1,"heartbeat"), - REQUEST((byte)2,"business request"), - RESPONSE((byte)3,"business response"); + HEARTBEAT((byte) 1, "heartbeat"), + REQUEST((byte) 2, "business request"), + RESPONSE((byte) 3, "business response"); private Byte type; diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/remote/NettyClient.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/remote/NettyClient.java index 1c8e81a79e75..633efa21d65b 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/remote/NettyClient.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/remote/NettyClient.java @@ -80,7 +80,6 @@ private static class NettyClientInner { */ private final NettyClientConfig clientConfig; - /** * client bootstrap */ @@ -143,6 +142,7 @@ private NettyClient(final NettyClientConfig clientConfig) { this.clientConfig = clientConfig; if (Epoll.isAvailable()) { this.workerGroup = new EpollEventLoopGroup(clientConfig.getWorkerThreads(), new ThreadFactory() { + private AtomicInteger threadIndex = new AtomicInteger(0); @Override @@ -152,6 +152,7 @@ public Thread newThread(Runnable r) { }); } else { this.workerGroup = new NioEventLoopGroup(clientConfig.getWorkerThreads(), new ThreadFactory() { + private AtomicInteger threadIndex = new AtomicInteger(0); @Override @@ -179,12 +180,15 @@ private void start() { .option(ChannelOption.CONNECT_TIMEOUT_MILLIS, clientConfig.getConnectTimeoutMillis()) .handler(new LoggingHandler(LogLevel.DEBUG)) .handler(new ChannelInitializer() { + @Override public void initChannel(SocketChannel ch) { ch.pipeline() .addLast(new NettyEncoder()) .addLast(new NettyDecoder(RpcResponse.class)) - .addLast("client-idle-handler", new IdleStateHandler(Constants.NETTY_CLIENT_HEART_BEAT_TIME, 0, 0, TimeUnit.MILLISECONDS)) + .addLast("client-idle-handler", + new IdleStateHandler(Constants.NETTY_CLIENT_HEART_BEAT_TIME, 0, 0, + TimeUnit.MILLISECONDS)) .addLast(new NettyClientHandler()); } }); diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/remote/NettyClientHandler.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/remote/NettyClientHandler.java index e810af2ce7dc..c8baec20744b 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/remote/NettyClientHandler.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/remote/NettyClientHandler.java @@ -44,7 +44,6 @@ @ChannelHandler.Sharable public class NettyClientHandler extends ChannelInboundHandlerAdapter { - private static final Logger logger = LoggerFactory.getLogger(NettyClientHandler.class); private static final ThreadPoolManager threadPoolManager = ThreadPoolManager.INSTANCE; @@ -87,7 +86,8 @@ private void readHandler(RpcResponse rsp, RpcRequestCache rpcRequest, long reqId try { consumerConfig.getServiceCallBackClass().getDeclaredConstructor().newInstance().run(rsp.getResult()); - } catch (InstantiationException | IllegalAccessException | NoSuchMethodException | InvocationTargetException e) { + } catch (InstantiationException | IllegalAccessException | NoSuchMethodException + | InvocationTargetException e) { logger.error("rpc service call back error,serviceName {},rsp {}", serviceName, rsp); } } else { diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/remote/NettyServer.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/remote/NettyServer.java index 4652d0a5bbd2..559eb90e46d7 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/remote/NettyServer.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/remote/NettyServer.java @@ -86,6 +86,7 @@ public NettyServer(final NettyServerConfig serverConfig) { this.serverConfig = serverConfig; if (Epoll.isAvailable()) { this.bossGroup = new EpollEventLoopGroup(1, new ThreadFactory() { + private AtomicInteger threadIndex = new AtomicInteger(0); @Override @@ -95,15 +96,18 @@ public Thread newThread(Runnable r) { }); this.workGroup = new EpollEventLoopGroup(serverConfig.getWorkerThread(), new ThreadFactory() { + private AtomicInteger threadIndex = new AtomicInteger(0); @Override public Thread newThread(Runnable r) { - return new Thread(r, String.format("NettyServerWorkerThread_%d", this.threadIndex.incrementAndGet())); + return new Thread(r, + String.format("NettyServerWorkerThread_%d", this.threadIndex.incrementAndGet())); } }); } else { this.bossGroup = new NioEventLoopGroup(1, new ThreadFactory() { + private AtomicInteger threadIndex = new AtomicInteger(0); @Override @@ -113,11 +117,13 @@ public Thread newThread(Runnable r) { }); this.workGroup = new NioEventLoopGroup(serverConfig.getWorkerThread(), new ThreadFactory() { + private AtomicInteger threadIndex = new AtomicInteger(0); @Override public Thread newThread(Runnable r) { - return new Thread(r, String.format("NettyServerWorkerThread_%d", this.threadIndex.incrementAndGet())); + return new Thread(r, + String.format("NettyServerWorkerThread_%d", this.threadIndex.incrementAndGet())); } }); } @@ -152,14 +158,18 @@ protected void initChannel(SocketChannel ch) { future = serverBootstrap.bind(serverConfig.getListenPort()).sync(); } catch (Exception e) { logger.error("NettyRemotingServer bind fail {}, exit", e.getMessage(), e); - throw new RuntimeException(String.format("NettyRemotingServer bind %s fail", serverConfig.getListenPort())); + throw new RuntimeException( + String.format("NettyRemotingServer bind %s fail", serverConfig.getListenPort())); } if (future.isSuccess()) { logger.info("NettyRemotingServer bind success at port : {}", serverConfig.getListenPort()); } else if (future.cause() != null) { - throw new RuntimeException(String.format("NettyRemotingServer bind %s fail", serverConfig.getListenPort()), future.cause()); + throw new RuntimeException( + String.format("NettyRemotingServer bind %s fail", serverConfig.getListenPort()), + future.cause()); } else { - throw new RuntimeException(String.format("NettyRemotingServer bind %s fail", serverConfig.getListenPort())); + throw new RuntimeException( + String.format("NettyRemotingServer bind %s fail", serverConfig.getListenPort())); } } } @@ -173,7 +183,8 @@ private void initNettyChannel(SocketChannel ch) { ch.pipeline() .addLast(new NettyDecoder(RpcRequest.class)) .addLast(new NettyEncoder()) - .addLast("server-idle-handle", new IdleStateHandler(0, 0, Constants.NETTY_SERVER_HEART_BEAT_TIME, TimeUnit.MILLISECONDS)) + .addLast("server-idle-handle", + new IdleStateHandler(0, 0, Constants.NETTY_SERVER_HEART_BEAT_TIME, TimeUnit.MILLISECONDS)) .addLast("handler", new NettyServerHandler()); } diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/serializer/ProtoStuffSerializer.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/serializer/ProtoStuffSerializer.java index a5b9250fc1f4..5ae26b2e0379 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/serializer/ProtoStuffSerializer.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/serializer/ProtoStuffSerializer.java @@ -1,19 +1,19 @@ package org.apache.dolphinscheduler.rpc.serializer;/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ import java.util.Map; import java.util.concurrent.ConcurrentHashMap; diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/serializer/RpcSerializer.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/serializer/RpcSerializer.java index 5c29d774a930..3ef5b4dddabe 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/serializer/RpcSerializer.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/serializer/RpcSerializer.java @@ -1,25 +1,24 @@ package org.apache.dolphinscheduler.rpc.serializer;/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ import java.util.HashMap; public enum RpcSerializer { - PROTOSTUFF((byte) 1, new ProtoStuffSerializer()); byte type; diff --git a/dolphinscheduler-remote/src/test/java/org/apache/dolphinscheduler/remote/NettyRemotingClientTest.java b/dolphinscheduler-remote/src/test/java/org/apache/dolphinscheduler/remote/NettyRemotingClientTest.java index 6652a60d7c6c..ac8f9ac9aa87 100644 --- a/dolphinscheduler-remote/src/test/java/org/apache/dolphinscheduler/remote/NettyRemotingClientTest.java +++ b/dolphinscheduler-remote/src/test/java/org/apache/dolphinscheduler/remote/NettyRemotingClientTest.java @@ -17,9 +17,6 @@ package org.apache.dolphinscheduler.remote; -import io.netty.buffer.ByteBuf; -import io.netty.buffer.Unpooled; -import io.netty.channel.Channel; import org.apache.dolphinscheduler.remote.command.Command; import org.apache.dolphinscheduler.remote.command.CommandType; import org.apache.dolphinscheduler.remote.config.NettyClientConfig; @@ -28,13 +25,18 @@ import org.apache.dolphinscheduler.remote.future.ResponseFuture; import org.apache.dolphinscheduler.remote.processor.NettyRequestProcessor; import org.apache.dolphinscheduler.remote.utils.Host; -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.Test; import java.io.Serializable; import java.util.concurrent.CountDownLatch; import java.util.concurrent.atomic.AtomicLong; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import io.netty.buffer.ByteBuf; +import io.netty.buffer.Unpooled; +import io.netty.channel.Channel; + /** * netty remote client test */ @@ -49,13 +51,13 @@ public void testSendSync() { NettyRemotingServer server = new NettyRemotingServer(serverConfig); server.registerProcessor(CommandType.PING, new NettyRequestProcessor() { + @Override public void process(Channel channel, Command command) { channel.writeAndFlush(Pong.create(command.getOpaque())); } }); - server.start(); // final NettyClientConfig clientConfig = new NettyClientConfig(); @@ -75,11 +77,12 @@ public void process(Channel channel, Command command) { * test sned async */ @Test - public void testSendAsync(){ + public void testSendAsync() { NettyServerConfig serverConfig = new NettyServerConfig(); NettyRemotingServer server = new NettyRemotingServer(serverConfig); server.registerProcessor(CommandType.PING, new NettyRequestProcessor() { + @Override public void process(Channel channel, Command command) { channel.writeAndFlush(Pong.create(command.getOpaque())); @@ -93,13 +96,15 @@ public void process(Channel channel, Command command) { Command commandPing = Ping.create(); try { final AtomicLong opaque = new AtomicLong(0); - client.sendAsync(new Host("127.0.0.1", serverConfig.getListenPort()), commandPing, 2000, new InvokeCallback() { - @Override - public void operationComplete(ResponseFuture responseFuture) { - opaque.set(responseFuture.getOpaque()); - latch.countDown(); - } - }); + client.sendAsync(new Host("127.0.0.1", serverConfig.getListenPort()), commandPing, 2000, + new InvokeCallback() { + + @Override + public void operationComplete(ResponseFuture responseFuture) { + opaque.set(responseFuture.getOpaque()); + latch.countDown(); + } + }); latch.await(); Assertions.assertEquals(commandPing.getOpaque(), opaque.get()); } catch (Exception e) { diff --git a/dolphinscheduler-remote/src/test/java/org/apache/dolphinscheduler/remote/command/alert/AlertSendRequestCommandTest.java b/dolphinscheduler-remote/src/test/java/org/apache/dolphinscheduler/remote/command/alert/AlertSendRequestCommandTest.java index cfd5b0673d5e..e0c3e20e90d5 100644 --- a/dolphinscheduler-remote/src/test/java/org/apache/dolphinscheduler/remote/command/alert/AlertSendRequestCommandTest.java +++ b/dolphinscheduler-remote/src/test/java/org/apache/dolphinscheduler/remote/command/alert/AlertSendRequestCommandTest.java @@ -20,6 +20,7 @@ import org.apache.dolphinscheduler.common.enums.WarningType; import org.apache.dolphinscheduler.remote.command.Command; import org.apache.dolphinscheduler.remote.command.CommandType; + import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; @@ -30,7 +31,8 @@ public void testConvert2Command() { int groupId = 1; String title = "test-title"; String content = "test-content"; - AlertSendRequestCommand requestCommand = new AlertSendRequestCommand(groupId,title,content,WarningType.FAILURE.getCode()); + AlertSendRequestCommand requestCommand = + new AlertSendRequestCommand(groupId, title, content, WarningType.FAILURE.getCode()); Command command = requestCommand.convert2Command(); Assertions.assertEquals(CommandType.ALERT_SEND_REQUEST, command.getType()); AlertSendRequestCommand verifyCommand = new AlertSendRequestCommand(); diff --git a/dolphinscheduler-remote/src/test/java/org/apache/dolphinscheduler/remote/command/alert/AlertSendResponseCommandTest.java b/dolphinscheduler-remote/src/test/java/org/apache/dolphinscheduler/remote/command/alert/AlertSendResponseCommandTest.java index 966386d12e0d..aceefb79803b 100644 --- a/dolphinscheduler-remote/src/test/java/org/apache/dolphinscheduler/remote/command/alert/AlertSendResponseCommandTest.java +++ b/dolphinscheduler-remote/src/test/java/org/apache/dolphinscheduler/remote/command/alert/AlertSendResponseCommandTest.java @@ -19,12 +19,13 @@ import org.apache.dolphinscheduler.remote.command.Command; import org.apache.dolphinscheduler.remote.command.CommandType; -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.Test; import java.util.ArrayList; import java.util.List; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + public class AlertSendResponseCommandTest { @Test diff --git a/dolphinscheduler-remote/src/test/java/org/apache/dolphinscheduler/remote/command/cache/CacheExpireCommandTest.java b/dolphinscheduler-remote/src/test/java/org/apache/dolphinscheduler/remote/command/cache/CacheExpireCommandTest.java index 8c47f267eec0..bcc6108422a0 100644 --- a/dolphinscheduler-remote/src/test/java/org/apache/dolphinscheduler/remote/command/cache/CacheExpireCommandTest.java +++ b/dolphinscheduler-remote/src/test/java/org/apache/dolphinscheduler/remote/command/cache/CacheExpireCommandTest.java @@ -21,6 +21,7 @@ import org.apache.dolphinscheduler.remote.command.CacheExpireCommand; import org.apache.dolphinscheduler.remote.command.Command; import org.apache.dolphinscheduler.remote.command.CommandType; + import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; diff --git a/dolphinscheduler-remote/src/test/java/org/apache/dolphinscheduler/remote/command/future/ResponseFutureTest.java b/dolphinscheduler-remote/src/test/java/org/apache/dolphinscheduler/remote/command/future/ResponseFutureTest.java index 1958786120ca..3cb6fb319c84 100644 --- a/dolphinscheduler-remote/src/test/java/org/apache/dolphinscheduler/remote/command/future/ResponseFutureTest.java +++ b/dolphinscheduler-remote/src/test/java/org/apache/dolphinscheduler/remote/command/future/ResponseFutureTest.java @@ -20,20 +20,23 @@ import org.apache.dolphinscheduler.remote.future.InvokeCallback; import org.apache.dolphinscheduler.remote.future.ResponseFuture; import org.apache.dolphinscheduler.remote.utils.NamedThreadFactory; -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.Test; import java.util.concurrent.CountDownLatch; import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + public class ResponseFutureTest { @Test public void testScanFutureTable() { - ScheduledExecutorService executorService = Executors.newSingleThreadScheduledExecutor(new NamedThreadFactory("executor-service")); + ScheduledExecutorService executorService = + Executors.newSingleThreadScheduledExecutor(new NamedThreadFactory("executor-service")); executorService.scheduleAtFixedRate(new Runnable() { + @Override public void run() { ResponseFuture.scanFutureTable(); @@ -42,6 +45,7 @@ public void run() { CountDownLatch latch = new CountDownLatch(1); InvokeCallback invokeCallback = new InvokeCallback() { + @Override public void operationComplete(ResponseFuture responseFuture) { latch.countDown(); diff --git a/dolphinscheduler-remote/src/test/java/org/apache/dolphinscheduler/remote/command/log/GetLogBytesRequestCommandTest.java b/dolphinscheduler-remote/src/test/java/org/apache/dolphinscheduler/remote/command/log/GetLogBytesRequestCommandTest.java index 898092c0b84f..dd9027f68d53 100644 --- a/dolphinscheduler-remote/src/test/java/org/apache/dolphinscheduler/remote/command/log/GetLogBytesRequestCommandTest.java +++ b/dolphinscheduler-remote/src/test/java/org/apache/dolphinscheduler/remote/command/log/GetLogBytesRequestCommandTest.java @@ -19,6 +19,7 @@ import org.apache.dolphinscheduler.remote.command.Command; import org.apache.dolphinscheduler.remote.command.CommandType; + import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; diff --git a/dolphinscheduler-remote/src/test/java/org/apache/dolphinscheduler/remote/command/log/GetLogBytesResponseCommandTest.java b/dolphinscheduler-remote/src/test/java/org/apache/dolphinscheduler/remote/command/log/GetLogBytesResponseCommandTest.java index 1a4a6b83a430..a6968174fba6 100644 --- a/dolphinscheduler-remote/src/test/java/org/apache/dolphinscheduler/remote/command/log/GetLogBytesResponseCommandTest.java +++ b/dolphinscheduler-remote/src/test/java/org/apache/dolphinscheduler/remote/command/log/GetLogBytesResponseCommandTest.java @@ -18,6 +18,7 @@ package org.apache.dolphinscheduler.remote.command.log; import org.apache.dolphinscheduler.remote.command.Command; + import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; diff --git a/dolphinscheduler-remote/src/test/java/org/apache/dolphinscheduler/remote/command/log/RemoveTaskLogRequestCommandTest.java b/dolphinscheduler-remote/src/test/java/org/apache/dolphinscheduler/remote/command/log/RemoveTaskLogRequestCommandTest.java index f5380c01c72f..802589ccdffd 100644 --- a/dolphinscheduler-remote/src/test/java/org/apache/dolphinscheduler/remote/command/log/RemoveTaskLogRequestCommandTest.java +++ b/dolphinscheduler-remote/src/test/java/org/apache/dolphinscheduler/remote/command/log/RemoveTaskLogRequestCommandTest.java @@ -18,6 +18,7 @@ package org.apache.dolphinscheduler.remote.command.log; import org.apache.dolphinscheduler.remote.command.Command; + import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; diff --git a/dolphinscheduler-remote/src/test/java/org/apache/dolphinscheduler/remote/command/log/RemoveTaskLogResponseCommandTest.java b/dolphinscheduler-remote/src/test/java/org/apache/dolphinscheduler/remote/command/log/RemoveTaskLogResponseCommandTest.java index 58a3706a0bc1..574b795d05c7 100644 --- a/dolphinscheduler-remote/src/test/java/org/apache/dolphinscheduler/remote/command/log/RemoveTaskLogResponseCommandTest.java +++ b/dolphinscheduler-remote/src/test/java/org/apache/dolphinscheduler/remote/command/log/RemoveTaskLogResponseCommandTest.java @@ -18,6 +18,7 @@ package org.apache.dolphinscheduler.remote.command.log; import org.apache.dolphinscheduler.remote.command.Command; + import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; diff --git a/dolphinscheduler-remote/src/test/java/org/apache/dolphinscheduler/remote/command/log/RollViewLogRequestCommandTest.java b/dolphinscheduler-remote/src/test/java/org/apache/dolphinscheduler/remote/command/log/RollViewLogRequestCommandTest.java index 3e37bfb9735e..cfda03610510 100644 --- a/dolphinscheduler-remote/src/test/java/org/apache/dolphinscheduler/remote/command/log/RollViewLogRequestCommandTest.java +++ b/dolphinscheduler-remote/src/test/java/org/apache/dolphinscheduler/remote/command/log/RollViewLogRequestCommandTest.java @@ -18,6 +18,7 @@ package org.apache.dolphinscheduler.remote.command.log; import org.apache.dolphinscheduler.remote.command.Command; + import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; diff --git a/dolphinscheduler-remote/src/test/java/org/apache/dolphinscheduler/rpc/RpcTest.java b/dolphinscheduler-remote/src/test/java/org/apache/dolphinscheduler/rpc/RpcTest.java index 09c60c4e77f1..3ff50d666a8c 100644 --- a/dolphinscheduler-remote/src/test/java/org/apache/dolphinscheduler/rpc/RpcTest.java +++ b/dolphinscheduler-remote/src/test/java/org/apache/dolphinscheduler/rpc/RpcTest.java @@ -28,6 +28,7 @@ import org.junit.jupiter.api.BeforeEach; public class RpcTest { + private NettyServer nettyServer; private IUserService userService; diff --git a/dolphinscheduler-scheduler-plugin/dolphinscheduler-scheduler-api/pom.xml b/dolphinscheduler-scheduler-plugin/dolphinscheduler-scheduler-api/pom.xml index 230c59282772..88a6aef9491b 100644 --- a/dolphinscheduler-scheduler-plugin/dolphinscheduler-scheduler-api/pom.xml +++ b/dolphinscheduler-scheduler-plugin/dolphinscheduler-scheduler-api/pom.xml @@ -15,15 +15,14 @@ ~ See the License for the specific language governing permissions and ~ limitations under the License. --> - + 4.0.0 org.apache.dolphinscheduler dolphinscheduler-scheduler-plugin dev-SNAPSHOT - 4.0.0 dolphinscheduler-scheduler-api @@ -34,4 +33,4 @@ - \ No newline at end of file + diff --git a/dolphinscheduler-scheduler-plugin/dolphinscheduler-scheduler-api/src/main/java/org/apache/dolphinscheduler/scheduler/api/SchedulerApi.java b/dolphinscheduler-scheduler-plugin/dolphinscheduler-scheduler-api/src/main/java/org/apache/dolphinscheduler/scheduler/api/SchedulerApi.java index 36f04c22bb59..c5d21cde4f56 100644 --- a/dolphinscheduler-scheduler-plugin/dolphinscheduler-scheduler-api/src/main/java/org/apache/dolphinscheduler/scheduler/api/SchedulerApi.java +++ b/dolphinscheduler-scheduler-plugin/dolphinscheduler-scheduler-api/src/main/java/org/apache/dolphinscheduler/scheduler/api/SchedulerApi.java @@ -22,7 +22,7 @@ /** * This is the interface for scheduler, contains methods to operate schedule task. */ -public interface SchedulerApi extends AutoCloseable{ +public interface SchedulerApi extends AutoCloseable { /** * Start the scheduler, if not start, the scheduler will not execute task. diff --git a/dolphinscheduler-scheduler-plugin/dolphinscheduler-scheduler-quartz/pom.xml b/dolphinscheduler-scheduler-plugin/dolphinscheduler-scheduler-quartz/pom.xml index d3afa0e70f0e..27e5795f5f64 100644 --- a/dolphinscheduler-scheduler-plugin/dolphinscheduler-scheduler-quartz/pom.xml +++ b/dolphinscheduler-scheduler-plugin/dolphinscheduler-scheduler-quartz/pom.xml @@ -15,15 +15,14 @@ ~ See the License for the specific language governing permissions and ~ limitations under the License. --> - + 4.0.0 org.apache.dolphinscheduler dolphinscheduler-scheduler-plugin dev-SNAPSHOT - 4.0.0 dolphinscheduler-scheduler-quartz @@ -75,4 +74,4 @@ - \ No newline at end of file + diff --git a/dolphinscheduler-scheduler-plugin/dolphinscheduler-scheduler-quartz/src/main/java/org/apache/dolphinscheduler/scheduler/quartz/ProcessScheduleTask.java b/dolphinscheduler-scheduler-plugin/dolphinscheduler-scheduler-quartz/src/main/java/org/apache/dolphinscheduler/scheduler/quartz/ProcessScheduleTask.java index 61eb86c982da..f51264d668d9 100644 --- a/dolphinscheduler-scheduler-plugin/dolphinscheduler-scheduler-quartz/src/main/java/org/apache/dolphinscheduler/scheduler/quartz/ProcessScheduleTask.java +++ b/dolphinscheduler-scheduler-plugin/dolphinscheduler-scheduler-quartz/src/main/java/org/apache/dolphinscheduler/scheduler/quartz/ProcessScheduleTask.java @@ -66,16 +66,21 @@ protected void executeInternal(JobExecutionContext context) { // query schedule Schedule schedule = processService.querySchedule(scheduleId); if (schedule == null || ReleaseState.OFFLINE == schedule.getReleaseState()) { - logger.warn("process schedule does not exist in db or process schedule offline，delete schedule job in quartz, projectId:{}, scheduleId:{}", projectId, scheduleId); + logger.warn( + "process schedule does not exist in db or process schedule offline，delete schedule job in quartz, projectId:{}, scheduleId:{}", + projectId, scheduleId); deleteJob(context, projectId, scheduleId); return; } - ProcessDefinition processDefinition = processService.findProcessDefinitionByCode(schedule.getProcessDefinitionCode()); + ProcessDefinition processDefinition = + processService.findProcessDefinitionByCode(schedule.getProcessDefinitionCode()); // release state : online/offline ReleaseState releaseState = processDefinition.getReleaseState(); if (releaseState == ReleaseState.OFFLINE) { - logger.warn("process definition does not exist in db or offline，need not to create command, projectId:{}, processDefinitionId:{}", projectId, processDefinition.getId()); + logger.warn( + "process definition does not exist in db or offline，need not to create command, projectId:{}, processDefinitionId:{}", + projectId, processDefinition.getId()); return; } @@ -87,7 +92,8 @@ protected void executeInternal(JobExecutionContext context) { command.setScheduleTime(scheduledFireTime); command.setStartTime(fireTime); command.setWarningGroupId(schedule.getWarningGroupId()); - String workerGroup = StringUtils.isEmpty(schedule.getWorkerGroup()) ? Constants.DEFAULT_WORKER_GROUP : schedule.getWorkerGroup(); + String workerGroup = StringUtils.isEmpty(schedule.getWorkerGroup()) ? Constants.DEFAULT_WORKER_GROUP + : schedule.getWorkerGroup(); command.setWorkerGroup(workerGroup); command.setWarningType(schedule.getWarningType()); command.setProcessInstancePriority(schedule.getProcessInstancePriority()); diff --git a/dolphinscheduler-scheduler-plugin/dolphinscheduler-scheduler-quartz/src/main/java/org/apache/dolphinscheduler/scheduler/quartz/QuartzScheduler.java b/dolphinscheduler-scheduler-plugin/dolphinscheduler-scheduler-quartz/src/main/java/org/apache/dolphinscheduler/scheduler/quartz/QuartzScheduler.java index 6ca8dc103780..1b62f6c82042 100644 --- a/dolphinscheduler-scheduler-plugin/dolphinscheduler-scheduler-quartz/src/main/java/org/apache/dolphinscheduler/scheduler/quartz/QuartzScheduler.java +++ b/dolphinscheduler-scheduler-plugin/dolphinscheduler-scheduler-quartz/src/main/java/org/apache/dolphinscheduler/scheduler/quartz/QuartzScheduler.java @@ -82,7 +82,7 @@ public void insertOrUpdateScheduleTask(int projectId, Schedule schedule) throws try { JobDetail jobDetail; - //add a task (if this task already exists, return this task directly) + // add a task (if this task already exists, return this task directly) if (scheduler.checkExists(jobKey)) { jobDetail = scheduler.getJobDetail(jobKey); @@ -99,11 +99,9 @@ public void insertOrUpdateScheduleTask(int projectId, Schedule schedule) throws TriggerKey triggerKey = new TriggerKey(jobKey.getName(), jobKey.getGroup()); /* - * Instructs the Scheduler that upon a mis-fire - * situation, the CronTrigger wants to have it's - * next-fire-time updated to the next time in the schedule after the - * current time (taking into account any associated Calendar), - * but it does not want to be fired now. + * Instructs the Scheduler that upon a mis-fire situation, the CronTrigger wants to have it's next-fire-time + * updated to the next time in the schedule after the current time (taking into account any associated + * Calendar), but it does not want to be fired now. */ CronTrigger cronTrigger = newTrigger() .withIdentity(triggerKey) @@ -112,8 +110,7 @@ public void insertOrUpdateScheduleTask(int projectId, Schedule schedule) throws .withSchedule( cronSchedule(cronExpression) .withMisfireHandlingInstructionFireAndProceed() - .inTimeZone(DateUtils.getTimezone(timezoneId)) - ) + .inTimeZone(DateUtils.getTimezone(timezoneId))) .forJob(jobDetail).build(); if (scheduler.checkExists(triggerKey)) { @@ -124,12 +121,14 @@ public void insertOrUpdateScheduleTask(int projectId, Schedule schedule) throws if (!Strings.nullToEmpty(cronExpression).equalsIgnoreCase(Strings.nullToEmpty(oldCronExpression))) { // reschedule job trigger scheduler.rescheduleJob(triggerKey, cronTrigger); - logger.info("reschedule job trigger, triggerName: {}, triggerGroupName: {}, cronExpression: {}, startDate: {}, endDate: {}", + logger.info( + "reschedule job trigger, triggerName: {}, triggerGroupName: {}, cronExpression: {}, startDate: {}, endDate: {}", triggerKey.getName(), triggerKey.getGroup(), cronExpression, startDate, endDate); } } else { scheduler.scheduleJob(cronTrigger); - logger.info("schedule job trigger, triggerName: {}, triggerGroupName: {}, cronExpression: {}, startDate: {}, endDate: {}", + logger.info( + "schedule job trigger, triggerName: {}, triggerGroupName: {}, cronExpression: {}, startDate: {}, endDate: {}", triggerKey.getName(), triggerKey.getGroup(), cronExpression, startDate, endDate); } diff --git a/dolphinscheduler-scheduler-plugin/pom.xml b/dolphinscheduler-scheduler-plugin/pom.xml index 65b887367404..5865678ecaa1 100644 --- a/dolphinscheduler-scheduler-plugin/pom.xml +++ b/dolphinscheduler-scheduler-plugin/pom.xml @@ -15,18 +15,17 @@ ~ See the License for the specific language governing permissions and ~ limitations under the License. --> - + 4.0.0 - dolphinscheduler org.apache.dolphinscheduler + dolphinscheduler dev-SNAPSHOT - 4.0.0 - pom dolphinscheduler-scheduler-plugin + pom dolphinscheduler-scheduler-api @@ -44,4 +43,4 @@ - \ No newline at end of file + diff --git a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/bean/SpringApplicationContext.java b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/bean/SpringApplicationContext.java index 5b37b1f72de2..50d0f835f769 100644 --- a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/bean/SpringApplicationContext.java +++ b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/bean/SpringApplicationContext.java @@ -38,7 +38,7 @@ public void setApplicationContext(ApplicationContext applicationContext) throws */ @Override public void close() { - ((AbstractApplicationContext)applicationContext).close(); + ((AbstractApplicationContext) applicationContext).close(); } public static T getBean(Class requiredType) { diff --git a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/cache/CacheNotifyService.java b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/cache/CacheNotifyService.java index 09c557149719..945efd574bf2 100644 --- a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/cache/CacheNotifyService.java +++ b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/cache/CacheNotifyService.java @@ -20,5 +20,6 @@ import org.apache.dolphinscheduler.remote.command.Command; public interface CacheNotifyService { + void notifyMaster(Command command); } diff --git a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/cron/AbstractCycle.java b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/cron/AbstractCycle.java index aa890af99d3a..f7ffe6ce1674 100644 --- a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/cron/AbstractCycle.java +++ b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/cron/AbstractCycle.java @@ -173,7 +173,7 @@ protected boolean dayofWeekFieldIsEvery() { FieldExpression dayOfWeekFieldExpression = dayOfWeekField.getExpression(); return (dayOfWeekFieldExpression instanceof Every || dayOfWeekFieldExpression instanceof Always); } - + /** * whether the year field has a value of every or always * diff --git a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/cron/CronUtils.java b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/cron/CronUtils.java index 81bf76a082d0..b15cee86487d 100644 --- a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/cron/CronUtils.java +++ b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/cron/CronUtils.java @@ -17,12 +17,16 @@ package org.apache.dolphinscheduler.service.cron; -import com.cronutils.model.Cron; -import com.cronutils.model.definition.CronDefinitionBuilder; -import com.cronutils.model.time.ExecutionTime; -import com.cronutils.parser.CronParser; -import lombok.NonNull; -import org.apache.commons.collections.CollectionUtils; +import static com.cronutils.model.CronType.QUARTZ; +import static org.apache.dolphinscheduler.common.Constants.CMDPARAM_COMPLEMENT_DATA_SCHEDULE_DATE_LIST; +import static org.apache.dolphinscheduler.common.Constants.COMMA; +import static org.apache.dolphinscheduler.service.cron.CycleFactory.day; +import static org.apache.dolphinscheduler.service.cron.CycleFactory.hour; +import static org.apache.dolphinscheduler.service.cron.CycleFactory.min; +import static org.apache.dolphinscheduler.service.cron.CycleFactory.month; +import static org.apache.dolphinscheduler.service.cron.CycleFactory.week; +import static org.apache.dolphinscheduler.service.cron.CycleFactory.year; + import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.enums.CycleEnum; import org.apache.dolphinscheduler.common.lifecycle.ServerLifeCycleManager; @@ -30,8 +34,8 @@ import org.apache.dolphinscheduler.dao.entity.Schedule; import org.apache.dolphinscheduler.service.exceptions.CronParseException; import org.apache.dolphinscheduler.spi.utils.StringUtils; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; + +import org.apache.commons.collections.CollectionUtils; import java.time.ZoneId; import java.time.ZonedDateTime; @@ -44,15 +48,15 @@ import java.util.Optional; import java.util.stream.Collectors; -import static com.cronutils.model.CronType.QUARTZ; -import static org.apache.dolphinscheduler.common.Constants.CMDPARAM_COMPLEMENT_DATA_SCHEDULE_DATE_LIST; -import static org.apache.dolphinscheduler.common.Constants.COMMA; -import static org.apache.dolphinscheduler.service.cron.CycleFactory.day; -import static org.apache.dolphinscheduler.service.cron.CycleFactory.hour; -import static org.apache.dolphinscheduler.service.cron.CycleFactory.min; -import static org.apache.dolphinscheduler.service.cron.CycleFactory.month; -import static org.apache.dolphinscheduler.service.cron.CycleFactory.week; -import static org.apache.dolphinscheduler.service.cron.CycleFactory.year; +import lombok.NonNull; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.cronutils.model.Cron; +import com.cronutils.model.definition.CronDefinitionBuilder; +import com.cronutils.model.time.ExecutionTime; +import com.cronutils.parser.CronParser; /** * // todo: this utils is heavy, it rely on quartz and corn-utils. diff --git a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/cron/CycleFactory.java b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/cron/CycleFactory.java index 00ca1029c6a2..b5c8bbba0fe1 100644 --- a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/cron/CycleFactory.java +++ b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/cron/CycleFactory.java @@ -16,15 +16,17 @@ */ package org.apache.dolphinscheduler.service.cron; +import org.apache.dolphinscheduler.common.enums.CycleEnum; + import com.cronutils.model.Cron; import com.cronutils.model.field.expression.Always; import com.cronutils.model.field.expression.QuestionMark; -import org.apache.dolphinscheduler.common.enums.CycleEnum; /** * Crontab Cycle Tool Factory */ public class CycleFactory { + private CycleFactory() { throw new IllegalStateException("CycleFactory class"); } @@ -34,7 +36,7 @@ private CycleFactory() { * @return AbstractCycle */ public static AbstractCycle min(Cron cron) { - return new MinCycle(cron); + return new MinCycle(cron); } /** @@ -43,7 +45,7 @@ public static AbstractCycle min(Cron cron) { * @return AbstractCycle */ public static AbstractCycle hour(Cron cron) { - return new HourCycle(cron); + return new HourCycle(cron); } /** @@ -52,7 +54,7 @@ public static AbstractCycle hour(Cron cron) { * @return AbstractCycle */ public static AbstractCycle day(Cron cron) { - return new DayCycle(cron); + return new DayCycle(cron); } /** @@ -61,7 +63,7 @@ public static AbstractCycle day(Cron cron) { * @return AbstractCycle */ public static AbstractCycle week(Cron cron) { - return new WeekCycle(cron); + return new WeekCycle(cron); } /** @@ -70,9 +72,9 @@ public static AbstractCycle week(Cron cron) { * @return AbstractCycle */ public static AbstractCycle month(Cron cron) { - return new MonthCycle(cron); + return new MonthCycle(cron); } - + /** * year * @param cron cron @@ -82,217 +84,219 @@ public static AbstractCycle year(Cron cron) { return new YearCycle(cron); } - /** - * day cycle - */ - public static class DayCycle extends AbstractCycle { + /** + * day cycle + */ + public static class DayCycle extends AbstractCycle { - public DayCycle(Cron cron) { - super(cron); - } + public DayCycle(Cron cron) { + super(cron); + } - /** - * get cycle - * @return CycleEnum - */ + /** + * get cycle + * @return CycleEnum + */ @Override protected CycleEnum getCycle() { - if (minFiledIsSetAll() - && hourFiledIsSetAll() - && dayOfMonthFieldIsEvery() - && dayOfWeekField.getExpression() instanceof QuestionMark - && monthField.getExpression() instanceof Always) { - return CycleEnum.DAY; - } + if (minFiledIsSetAll() + && hourFiledIsSetAll() + && dayOfMonthFieldIsEvery() + && dayOfWeekField.getExpression() instanceof QuestionMark + && monthField.getExpression() instanceof Always) { + return CycleEnum.DAY; + } - return null; + return null; } - /** - * get min cycle - * @return CycleEnum - */ + /** + * get min cycle + * @return CycleEnum + */ @Override protected CycleEnum getMiniCycle() { - if (dayOfMonthFieldIsEvery()) { - return CycleEnum.DAY; - } + if (dayOfMonthFieldIsEvery()) { + return CycleEnum.DAY; + } - return null; + return null; } - } + } + + /** + * hour cycle + */ + public static class HourCycle extends AbstractCycle { + + public HourCycle(Cron cron) { + super(cron); + } + + /** + * get cycle + * @return CycleEnum + */ + @Override + protected CycleEnum getCycle() { + if (minFiledIsSetAll() + && hourFiledIsEvery() + && dayOfMonthField.getExpression() instanceof Always + && dayOfWeekField.getExpression() instanceof QuestionMark + && monthField.getExpression() instanceof Always) { + return CycleEnum.HOUR; + } - /** - * hour cycle - */ - public static class HourCycle extends AbstractCycle { + return null; + } - public HourCycle(Cron cron) { - super(cron); + /** + * get mini cycle + * @return CycleEnum + */ + @Override + protected CycleEnum getMiniCycle() { + if (hourFiledIsEvery()) { + return CycleEnum.HOUR; + } + return null; + } } - /** - * get cycle - * @return CycleEnum - */ + /** + * minute cycle + */ + public static class MinCycle extends AbstractCycle { + + public MinCycle(Cron cron) { + super(cron); + } + + /** + * get cycle + * @return CycleEnum + */ @Override protected CycleEnum getCycle() { - if (minFiledIsSetAll() - && hourFiledIsEvery() - && dayOfMonthField.getExpression() instanceof Always - && dayOfWeekField.getExpression() instanceof QuestionMark - && monthField.getExpression() instanceof Always) { - return CycleEnum.HOUR; - } - - return null; + if (minFiledIsEvery() + && hourField.getExpression() instanceof Always + && dayOfMonthField.getExpression() instanceof Always + && monthField.getExpression() instanceof Always) { + return CycleEnum.MINUTE; + } + + return null; } - /** - * get mini cycle - * @return CycleEnum - */ + /** + * get min cycle + * @return CycleEnum + */ @Override protected CycleEnum getMiniCycle() { - if(hourFiledIsEvery()){ - return CycleEnum.HOUR; - } - return null; + if (minFiledIsEvery()) { + return CycleEnum.MINUTE; + } + return null; } - } - - /** - * minute cycle - */ - public static class MinCycle extends AbstractCycle { - - public MinCycle(Cron cron) { - super(cron); - } - - /** - * get cycle - * @return CycleEnum - */ - @Override - protected CycleEnum getCycle() { - if (minFiledIsEvery() - && hourField.getExpression() instanceof Always - && dayOfMonthField.getExpression() instanceof Always - && monthField.getExpression() instanceof Always) { - return CycleEnum.MINUTE; - } - - return null; - } - - /** - * get min cycle - * @return CycleEnum - */ - @Override - protected CycleEnum getMiniCycle() { - if(minFiledIsEvery()){ - return CycleEnum.MINUTE; - } - return null; - } - } - - /** - * month cycle - */ - public static class MonthCycle extends AbstractCycle { - - public MonthCycle(Cron cron) { - super(cron); } - /** - * get cycle - * @return CycleEnum - */ + /** + * month cycle + */ + public static class MonthCycle extends AbstractCycle { + + public MonthCycle(Cron cron) { + super(cron); + } + + /** + * get cycle + * @return CycleEnum + */ @Override protected CycleEnum getCycle() { - boolean flag = (minFiledIsSetAll() - && hourFiledIsSetAll() - && dayOfMonthFieldIsSetAll() - && dayOfWeekField.getExpression() instanceof QuestionMark - && monthFieldIsEvery()) || - (minFiledIsSetAll() - && hourFiledIsSetAll() - && dayOfMonthField.getExpression() instanceof QuestionMark - && dayofWeekFieldIsSetAll() - && monthFieldIsEvery()); - if (flag) { - return CycleEnum.MONTH; - } - - return null; + boolean flag = (minFiledIsSetAll() + && hourFiledIsSetAll() + && dayOfMonthFieldIsSetAll() + && dayOfWeekField.getExpression() instanceof QuestionMark + && monthFieldIsEvery()) || + (minFiledIsSetAll() + && hourFiledIsSetAll() + && dayOfMonthField.getExpression() instanceof QuestionMark + && dayofWeekFieldIsSetAll() + && monthFieldIsEvery()); + if (flag) { + return CycleEnum.MONTH; + } + + return null; } - /** - * get mini cycle - * @return CycleEnum - */ + /** + * get mini cycle + * @return CycleEnum + */ @Override protected CycleEnum getMiniCycle() { - if (monthFieldIsEvery()) { - return CycleEnum.MONTH; - } + if (monthFieldIsEvery()) { + return CycleEnum.MONTH; + } - return null; + return null; } - } - - /** - * week cycle - */ - public static class WeekCycle extends AbstractCycle { - public WeekCycle(Cron cron) { - super(cron); } - /** - * get cycle - * @return CycleEnum - */ + /** + * week cycle + */ + public static class WeekCycle extends AbstractCycle { + + public WeekCycle(Cron cron) { + super(cron); + } + + /** + * get cycle + * @return CycleEnum + */ @Override protected CycleEnum getCycle() { - if (minFiledIsSetAll() - && hourFiledIsSetAll() - && dayOfMonthField.getExpression() instanceof QuestionMark - && dayofWeekFieldIsEvery() - && monthField.getExpression() instanceof Always) { - return CycleEnum.WEEK; - } - - return null; + if (minFiledIsSetAll() + && hourFiledIsSetAll() + && dayOfMonthField.getExpression() instanceof QuestionMark + && dayofWeekFieldIsEvery() + && monthField.getExpression() instanceof Always) { + return CycleEnum.WEEK; + } + + return null; } - /** - * get mini cycle - * @return CycleEnum - */ + /** + * get mini cycle + * @return CycleEnum + */ @Override protected CycleEnum getMiniCycle() { - if (dayofWeekFieldIsEvery()) { - return CycleEnum.WEEK; - } + if (dayofWeekFieldIsEvery()) { + return CycleEnum.WEEK; + } - return null; + return null; } - } - + } + /** * year cycle */ public static class YearCycle extends AbstractCycle { + public YearCycle(Cron cron) { super(cron); } - + /** * get cycle * @return CycleEnum @@ -311,14 +315,14 @@ && hourFiledIsSetAll() && dayofWeekFieldIsSetAll() && monthFieldIsSetAll() && yearFieldIsEvery()); - + if (flag) { return CycleEnum.YEAR; } - + return null; } - + /** * get mini cycle * @return CycleEnum @@ -328,7 +332,7 @@ protected CycleEnum getMiniCycle() { if (yearFieldIsEvery()) { return CycleEnum.YEAR; } - + return null; } } diff --git a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/cron/CycleLinks.java b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/cron/CycleLinks.java index d4011bcd74ab..20d0dcb5f267 100644 --- a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/cron/CycleLinks.java +++ b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/cron/CycleLinks.java @@ -16,62 +16,64 @@ */ package org.apache.dolphinscheduler.service.cron; -import com.cronutils.model.Cron; import org.apache.dolphinscheduler.common.enums.CycleEnum; import java.util.ArrayList; import java.util.List; +import com.cronutils.model.Cron; + /** * DAG Cycle judge */ public class CycleLinks extends AbstractCycle { - private final List cycleList = new ArrayList<>(); - public CycleLinks(Cron cron) { - super(cron); - } + private final List cycleList = new ArrayList<>(); - /** - * add cycle - * @param cycle cycle - * @return CycleLinks - */ - @Override - public CycleLinks addCycle(AbstractCycle cycle) { - cycleList.add(cycle); - return this; - } + public CycleLinks(Cron cron) { + super(cron); + } - /** - * get cycle - * @return CycleEnum - */ - @Override - protected CycleEnum getCycle() { - for (AbstractCycle abstractCycle : cycleList) { - CycleEnum cycle = abstractCycle.getCycle(); - if (cycle != null) { - return cycle; - } + /** + * add cycle + * @param cycle cycle + * @return CycleLinks + */ + @Override + public CycleLinks addCycle(AbstractCycle cycle) { + cycleList.add(cycle); + return this; } - return null; - } + /** + * get cycle + * @return CycleEnum + */ + @Override + protected CycleEnum getCycle() { + for (AbstractCycle abstractCycle : cycleList) { + CycleEnum cycle = abstractCycle.getCycle(); + if (cycle != null) { + return cycle; + } + } - /** - * get mini cycle - * @return CycleEnum - */ - @Override - protected CycleEnum getMiniCycle() { - for (AbstractCycle cycleHelper : cycleList) { - CycleEnum cycle = cycleHelper.getMiniCycle(); - if (cycle != null) { - return cycle; - } + return null; } - return null; - } -} \ No newline at end of file + /** + * get mini cycle + * @return CycleEnum + */ + @Override + protected CycleEnum getMiniCycle() { + for (AbstractCycle cycleHelper : cycleList) { + CycleEnum cycle = cycleHelper.getMiniCycle(); + if (cycle != null) { + return cycle; + } + } + + return null; + } +} diff --git a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/expand/CuringParamsService.java b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/expand/CuringParamsService.java index 3342bd6bbaa4..dfb4f14c954e 100644 --- a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/expand/CuringParamsService.java +++ b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/expand/CuringParamsService.java @@ -17,11 +17,9 @@ package org.apache.dolphinscheduler.service.expand; -import lombok.NonNull; import org.apache.dolphinscheduler.common.enums.CommandType; import org.apache.dolphinscheduler.dao.entity.ProcessInstance; import org.apache.dolphinscheduler.dao.entity.TaskInstance; -import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext; import org.apache.dolphinscheduler.plugin.task.api.model.Property; import org.apache.dolphinscheduler.plugin.task.api.parameters.AbstractParameters; @@ -29,6 +27,8 @@ import java.util.List; import java.util.Map; +import lombok.NonNull; + public interface CuringParamsService { /** @@ -65,7 +65,9 @@ public interface CuringParamsService { * @param timezone * @return */ - String curingGlobalParams(Integer processInstanceId, Map globalParamMap, List globalParamList, CommandType commandType, Date scheduleTime, String timezone); + String curingGlobalParams(Integer processInstanceId, Map globalParamMap, + List globalParamList, CommandType commandType, Date scheduleTime, + String timezone); /** * param parsing preparation @@ -74,7 +76,9 @@ public interface CuringParamsService { * @param processInstance * @return */ - Map paramParsingPreparation(@NonNull TaskInstance taskInstance, @NonNull AbstractParameters parameters, @NonNull ProcessInstance processInstance); + Map paramParsingPreparation(@NonNull TaskInstance taskInstance, + @NonNull AbstractParameters parameters, + @NonNull ProcessInstance processInstance); /** * preBuildBusinessParams diff --git a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/json/DateJsonSerializer.java b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/json/DateJsonSerializer.java index c45665ab9996..9324440970e2 100644 --- a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/json/DateJsonSerializer.java +++ b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/json/DateJsonSerializer.java @@ -30,6 +30,7 @@ @JsonComponent public class DateJsonSerializer extends JsonSerializer { + @Override public void serialize(Date value, JsonGenerator gen, SerializerProvider serializers) throws IOException { gen.writeString(DateUtils.dateToString(value)); diff --git a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/queue/MasterPriorityQueue.java b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/queue/MasterPriorityQueue.java index 73d420ccf075..e5405489e37e 100644 --- a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/queue/MasterPriorityQueue.java +++ b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/queue/MasterPriorityQueue.java @@ -102,6 +102,7 @@ public int getIndex(String addr) { * server comparator, used to sort server by createTime in reverse order. */ private class ServerComparator implements Comparator { + @Override public int compare(Server o1, Server o2) { return o2.getCreateTime().compareTo(o1.getCreateTime()); diff --git a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/queue/PeerTaskInstancePriorityQueue.java b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/queue/PeerTaskInstancePriorityQueue.java index 0cf03abe8d43..30b671508235 100644 --- a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/queue/PeerTaskInstancePriorityQueue.java +++ b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/queue/PeerTaskInstancePriorityQueue.java @@ -36,6 +36,7 @@ * All the task instances are in the same process instance. */ public class PeerTaskInstancePriorityQueue implements TaskPriorityQueue { + /** * queue size */ @@ -88,7 +89,8 @@ public TaskInstance take() throws TaskPriorityQueueException { */ @Override public TaskInstance poll(long timeout, TimeUnit unit) throws TaskPriorityQueueException { - throw new TaskPriorityQueueException("This operation is not currently supported and suggest to use PriorityBlockingQueue if you want！"); + throw new TaskPriorityQueueException( + "This operation is not currently supported and suggest to use PriorityBlockingQueue if you want！"); } /** @@ -157,8 +159,7 @@ private String getTaskInstanceIdentify(TaskInstance taskInstance) { return String.join( String.valueOf(taskInstance.getProcessInstanceId()), String.valueOf(taskInstance.getTaskCode()), - String.valueOf(taskInstance.getTaskDefinitionVersion()) - , "-"); + String.valueOf(taskInstance.getTaskDefinitionVersion()), "-"); } /** @@ -175,9 +176,9 @@ private static class TaskInfoComparator implements Comparator { */ @Override public int compare(TaskInstance o1, TaskInstance o2) { - if(o1.getTaskInstancePriority().equals(o2.getTaskInstancePriority())){ + if (o1.getTaskInstancePriority().equals(o2.getTaskInstancePriority())) { // larger number, higher priority - return Constants.OPPOSITE_VALUE * Integer.compare(o1.getTaskGroupPriority(),o2.getTaskGroupPriority()); + return Constants.OPPOSITE_VALUE * Integer.compare(o1.getTaskGroupPriority(), o2.getTaskGroupPriority()); } return o1.getTaskInstancePriority().compareTo(o2.getTaskInstancePriority()); } diff --git a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/queue/TaskPriority.java b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/queue/TaskPriority.java index 823ec81d3c5f..f5af1edda90f 100644 --- a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/queue/TaskPriority.java +++ b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/queue/TaskPriority.java @@ -183,9 +183,10 @@ public int compareTo(TaskPriority other) { if (this.getTaskInstancePriority() < other.getTaskInstancePriority()) { return -1; } - if(this.getTaskGroupPriority() != other.getTaskGroupPriority()){ + if (this.getTaskGroupPriority() != other.getTaskGroupPriority()) { // larger number, higher priority - return Constants.OPPOSITE_VALUE * Integer.compare(this.getTaskGroupPriority(), other.getTaskGroupPriority()); + return Constants.OPPOSITE_VALUE + * Integer.compare(this.getTaskGroupPriority(), other.getTaskGroupPriority()); } if (this.getTaskId() > other.getTaskId()) { return 1; @@ -193,9 +194,11 @@ public int compareTo(TaskPriority other) { if (this.getTaskId() < other.getTaskId()) { return -1; } - String thisGroupName = StringUtils.isNotBlank(this.getGroupName()) ? this.getGroupName() : Constants.EMPTY_STRING; - String otherGroupName = StringUtils.isNotBlank(other.getGroupName()) ? other.getGroupName() : Constants.EMPTY_STRING; - if(!thisGroupName.equals(otherGroupName)){ + String thisGroupName = + StringUtils.isNotBlank(this.getGroupName()) ? this.getGroupName() : Constants.EMPTY_STRING; + String otherGroupName = + StringUtils.isNotBlank(other.getGroupName()) ? other.getGroupName() : Constants.EMPTY_STRING; + if (!thisGroupName.equals(otherGroupName)) { return thisGroupName.compareTo(otherGroupName); } return Long.compare(this.getCheckpoint(), other.getCheckpoint()); @@ -211,45 +214,45 @@ public boolean equals(Object o) { } TaskPriority that = (TaskPriority) o; return processInstancePriority == that.processInstancePriority - && processInstanceId == that.processInstanceId - && taskInstancePriority == that.taskInstancePriority - && taskId == that.taskId - && taskGroupPriority == that.taskGroupPriority - && Objects.equals(groupName, that.groupName); + && processInstanceId == that.processInstanceId + && taskInstancePriority == that.taskInstancePriority + && taskId == that.taskId + && taskGroupPriority == that.taskGroupPriority + && Objects.equals(groupName, that.groupName); } @Override public int hashCode() { return Objects.hash(processInstancePriority, - processInstanceId, - taskInstancePriority, - taskId, - taskGroupPriority, - groupName); + processInstanceId, + taskInstancePriority, + taskId, + taskGroupPriority, + groupName); } @Override public String toString() { return "TaskPriority{" - + "processInstancePriority=" - + processInstancePriority - + ", processInstanceId=" - + processInstanceId - + ", taskInstancePriority=" - + taskInstancePriority - + ", taskId=" - + taskId - + ", taskExecutionContext=" - + taskExecutionContext - + ", groupName='" - + groupName - + '\'' - + ", context=" - + context - + ", checkpoint=" - + checkpoint - + ", taskGroupPriority=" - + taskGroupPriority - + '}'; + + "processInstancePriority=" + + processInstancePriority + + ", processInstanceId=" + + processInstanceId + + ", taskInstancePriority=" + + taskInstancePriority + + ", taskId=" + + taskId + + ", taskExecutionContext=" + + taskExecutionContext + + ", groupName='" + + groupName + + '\'' + + ", context=" + + context + + ", checkpoint=" + + checkpoint + + ", taskGroupPriority=" + + taskGroupPriority + + '}'; } } diff --git a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/queue/TaskPriorityQueue.java b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/queue/TaskPriorityQueue.java index 0007581d3cda..736e117fe508 100644 --- a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/queue/TaskPriorityQueue.java +++ b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/queue/TaskPriorityQueue.java @@ -43,7 +43,6 @@ public interface TaskPriorityQueue { */ T take() throws TaskPriorityQueueException, InterruptedException; - /** * poll taskInfo with timeout * @param timeout @@ -61,4 +60,4 @@ public interface TaskPriorityQueue { * @throws TaskPriorityQueueException */ int size() throws TaskPriorityQueueException; -} \ No newline at end of file +} diff --git a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/queue/TaskPriorityQueueImpl.java b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/queue/TaskPriorityQueueImpl.java index 0d0eebe03db3..d9578b0757d7 100644 --- a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/queue/TaskPriorityQueueImpl.java +++ b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/queue/TaskPriorityQueueImpl.java @@ -68,7 +68,7 @@ public TaskPriority take() throws TaskPriorityQueueException, InterruptedExcepti */ @Override public TaskPriority poll(long timeout, TimeUnit unit) throws TaskPriorityQueueException, InterruptedException { - return queue.poll(timeout,unit); + return queue.poll(timeout, unit); } /** diff --git a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/task/TaskPluginManager.java b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/task/TaskPluginManager.java index e3776e09ba4d..6bcd079870dc 100644 --- a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/task/TaskPluginManager.java +++ b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/task/TaskPluginManager.java @@ -22,9 +22,6 @@ import org.apache.dolphinscheduler.plugin.task.api.parameters.AbstractParameters; import org.apache.dolphinscheduler.plugin.task.api.parameters.ParametersNode; import org.apache.dolphinscheduler.spi.plugin.PrioritySPIFactory; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.springframework.stereotype.Component; import java.util.Collections; import java.util.HashMap; @@ -32,8 +29,13 @@ import java.util.Objects; import java.util.concurrent.atomic.AtomicBoolean; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.stereotype.Component; + @Component public class TaskPluginManager { + private static final Logger logger = LoggerFactory.getLogger(TaskPluginManager.class); private final Map taskChannelFactoryMap = new HashMap<>(); diff --git a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/utils/ProcessData.java b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/utils/ProcessData.java index 4481448a1d0c..f0ba597f50d5 100644 --- a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/utils/ProcessData.java +++ b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/utils/ProcessData.java @@ -16,17 +16,19 @@ */ package org.apache.dolphinscheduler.service.utils; -import org.apache.dolphinscheduler.common.utils.CollectionUtils; import org.apache.dolphinscheduler.plugin.task.api.model.Property; import org.apache.dolphinscheduler.service.model.TaskNode; + +import java.util.List; + import lombok.Data; import lombok.EqualsAndHashCode; import lombok.NoArgsConstructor; -import java.util.List; @Data @NoArgsConstructor public class ProcessData { + @EqualsAndHashCode.Include private List tasks; diff --git a/dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/alert/AlertClientServiceTest.java b/dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/alert/AlertClientServiceTest.java index 3b7d7b2eae93..dd065cb0b603 100644 --- a/dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/alert/AlertClientServiceTest.java +++ b/dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/alert/AlertClientServiceTest.java @@ -24,6 +24,11 @@ import org.apache.dolphinscheduler.remote.command.alert.AlertSendResponseCommand; import org.apache.dolphinscheduler.remote.command.alert.AlertSendResponseResult; import org.apache.dolphinscheduler.remote.factory.NettyRemotingClientFactory; + +import java.util.ArrayList; +import java.util.List; +import java.util.Objects; + import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.BeforeEach; @@ -35,10 +40,6 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.util.ArrayList; -import java.util.List; -import java.util.Objects; - @ExtendWith(MockitoExtension.class) public class AlertClientServiceTest { diff --git a/dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/alert/ProcessAlertManagerTest.java b/dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/alert/ProcessAlertManagerTest.java index a1622da903df..2dc864eb1ec6 100644 --- a/dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/alert/ProcessAlertManagerTest.java +++ b/dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/alert/ProcessAlertManagerTest.java @@ -24,6 +24,11 @@ import org.apache.dolphinscheduler.dao.entity.ProcessInstance; import org.apache.dolphinscheduler.dao.entity.ProjectUser; import org.apache.dolphinscheduler.dao.entity.TaskInstance; + +import java.util.ArrayList; +import java.util.Date; +import java.util.List; + import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; import org.mockito.InjectMocks; @@ -32,10 +37,6 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.util.ArrayList; -import java.util.Date; -import java.util.List; - /** * ProcessAlertManager Test */ diff --git a/dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/cache/CacheNotifyServiceTest.java b/dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/cache/CacheNotifyServiceTest.java index 8655def27b6d..595e3cc071df 100644 --- a/dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/cache/CacheNotifyServiceTest.java +++ b/dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/cache/CacheNotifyServiceTest.java @@ -28,6 +28,10 @@ import org.apache.dolphinscheduler.remote.config.NettyServerConfig; import org.apache.dolphinscheduler.service.cache.impl.CacheNotifyServiceImpl; import org.apache.dolphinscheduler.service.registry.RegistryClient; + +import java.util.ArrayList; +import java.util.List; + import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; @@ -36,9 +40,6 @@ import org.mockito.Mockito; import org.mockito.junit.jupiter.MockitoExtension; -import java.util.ArrayList; -import java.util.List; - /** * tenant cache proxy test */ diff --git a/dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/cron/CronUtilsTest.java b/dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/cron/CronUtilsTest.java index 199a351feb93..6be6d954594e 100644 --- a/dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/cron/CronUtilsTest.java +++ b/dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/cron/CronUtilsTest.java @@ -17,26 +17,36 @@ package org.apache.dolphinscheduler.service.cron; -import com.cronutils.builder.CronBuilder; -import com.cronutils.model.Cron; -import com.cronutils.model.CronType; -import com.cronutils.model.definition.CronDefinitionBuilder; -import com.cronutils.model.field.CronField; -import com.cronutils.model.field.CronFieldName; -import com.cronutils.model.field.expression.*; +import static com.cronutils.model.field.expression.FieldExpressionFactory.always; +import static com.cronutils.model.field.expression.FieldExpressionFactory.every; +import static com.cronutils.model.field.expression.FieldExpressionFactory.on; +import static com.cronutils.model.field.expression.FieldExpressionFactory.questionMark; + import org.apache.dolphinscheduler.common.enums.CycleEnum; import org.apache.dolphinscheduler.common.utils.DateUtils; import org.apache.dolphinscheduler.service.exceptions.CronParseException; -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.Test; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import java.time.ZoneId; import java.time.ZonedDateTime; import java.util.Date; -import static com.cronutils.model.field.expression.FieldExpressionFactory.*; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.cronutils.builder.CronBuilder; +import com.cronutils.model.Cron; +import com.cronutils.model.CronType; +import com.cronutils.model.definition.CronDefinitionBuilder; +import com.cronutils.model.field.CronField; +import com.cronutils.model.field.CronFieldName; +import com.cronutils.model.field.expression.Always; +import com.cronutils.model.field.expression.And; +import com.cronutils.model.field.expression.Between; +import com.cronutils.model.field.expression.Every; +import com.cronutils.model.field.expression.On; +import com.cronutils.model.field.expression.QuestionMark; /** * CronUtilsTest @@ -51,16 +61,15 @@ public class CronUtilsTest { @Test public void testCronAsString() { Cron cron = CronBuilder.cron(CronDefinitionBuilder.instanceDefinitionFor(CronType.QUARTZ)).withYear(always()) - .withDoW(questionMark()).withMonth(always()).withDoM(always()).withHour(always()).withMinute(every(5)) - .withSecond(on(0)).instance(); + .withDoW(questionMark()).withMonth(always()).withDoM(always()).withHour(always()).withMinute(every(5)) + .withSecond(on(0)).instance(); // Obtain the string expression String cronAsString = cron.asString(); - // 0 */5 * * * ? * Every five minutes(once every 5 minutes) + // 0 */5 * * * ? * Every five minutes(once every 5 minutes) Assertions.assertEquals("0 */5 * * * ? *", cronAsString); } - /** * cron parse test */ @@ -112,12 +121,12 @@ public void testScheduleType() throws CronParseException { @Test public void test2() throws CronParseException { Cron cron1 = CronBuilder.cron(CronDefinitionBuilder.instanceDefinitionFor(CronType.QUARTZ)).withYear(always()) - .withDoW(questionMark()).withMonth(always()).withDoM(always()).withHour(always()).withMinute(every(5)) - .withSecond(on(0)).instance(); + .withDoW(questionMark()).withMonth(always()).withDoM(always()).withHour(always()).withMinute(every(5)) + .withSecond(on(0)).instance(); // minute cycle String[] cronArayy = - new String[] {"* * * * * ? *", "* 0 * * * ? *", "* 5 * * 3/5 ? *", "0 0 * * * ? *", "0 0 7 * 1 ? *", - "0 0 7 * 1/1 ? *", "0 0 7 * 1-2 ? *", "0 0 7 * 1,2 ? *"}; + new String[]{"* * * * * ? *", "* 0 * * * ? *", "* 5 * * 3/5 ? *", "0 0 * * * ? *", "0 0 7 * 1 ? *", + "0 0 7 * 1/1 ? *", "0 0 7 * 1-2 ? *", "0 0 7 * 1,2 ? *"}; for (String minCrontab : cronArayy) { Cron cron = CronUtils.parse2Cron(minCrontab); CronField minField = cron.retrieve(CronFieldName.MINUTE); @@ -140,7 +149,8 @@ public void test2() throws CronParseException { logger.info("dayOfMonthField instanceof On:" + (dayOfMonthField.getExpression() instanceof On)); logger.info("dayOfMonthField instanceof And:" + (dayOfMonthField.getExpression() instanceof And)); logger.info( - "dayOfMonthField instanceof QuestionMark:" + (dayOfMonthField.getExpression() instanceof QuestionMark)); + "dayOfMonthField instanceof QuestionMark:" + + (dayOfMonthField.getExpression() instanceof QuestionMark)); CronField monthField = cron.retrieve(CronFieldName.MONTH); logger.info("monthField instanceof Between:" + (monthField.getExpression() instanceof Between)); @@ -157,7 +167,8 @@ public void test2() throws CronParseException { logger.info("dayOfWeekField instanceof On:" + (dayOfWeekField.getExpression() instanceof On)); logger.info("dayOfWeekField instanceof And:" + (dayOfWeekField.getExpression() instanceof And)); logger.info( - "dayOfWeekField instanceof QuestionMark:" + (dayOfWeekField.getExpression() instanceof QuestionMark)); + "dayOfWeekField instanceof QuestionMark:" + + (dayOfWeekField.getExpression() instanceof QuestionMark)); CronField yearField = cron.retrieve(CronFieldName.YEAR); logger.info("yearField instanceof Between:" + (yearField.getExpression() instanceof Between)); @@ -180,9 +191,11 @@ public void test2() throws CronParseException { @Test public void getSelfFireDateList() throws CronParseException { ZonedDateTime from = - ZonedDateTime.ofInstant(DateUtils.stringToDate("2020-01-01 00:00:00").toInstant(), ZoneId.systemDefault()); + ZonedDateTime.ofInstant(DateUtils.stringToDate("2020-01-01 00:00:00").toInstant(), + ZoneId.systemDefault()); ZonedDateTime to = - ZonedDateTime.ofInstant(DateUtils.stringToDate("2020-01-31 00:00:00").toInstant(), ZoneId.systemDefault()); + ZonedDateTime.ofInstant(DateUtils.stringToDate("2020-01-31 00:00:00").toInstant(), + ZoneId.systemDefault()); // test date Assertions.assertEquals(0, CronUtils.getFireDateList(to, from, "0 0 0 * * ? ").size()); try { @@ -197,20 +210,25 @@ public void getSelfFireDateList() throws CronParseException { Assertions.assertEquals(30, CronUtils.getFireDateList(from, to, "0 0 0 * * ? ").size()); // test other Assertions.assertEquals(30, CronUtils.getFireDateList(from, to, CronUtils.parse2Cron("0 0 0 * * ? ")).size()); - Assertions.assertEquals(5, CronUtils.getSelfFireDateList(from, to, CronUtils.parse2Cron("0 0 0 * * ? "), 5).size()); + Assertions.assertEquals(5, + CronUtils.getSelfFireDateList(from, to, CronUtils.parse2Cron("0 0 0 * * ? "), 5).size()); from = - ZonedDateTime.ofInstant(DateUtils.stringToDate("2020-01-01 00:02:00").toInstant(), ZoneId.systemDefault()); + ZonedDateTime.ofInstant(DateUtils.stringToDate("2020-01-01 00:02:00").toInstant(), + ZoneId.systemDefault()); to = ZonedDateTime.ofInstant(DateUtils.stringToDate("2020-01-01 00:02:00").toInstant(), ZoneId.systemDefault()); Assertions.assertEquals(1, - CronUtils.getFireDateList(from.minusSeconds(1L), to, CronUtils.parse2Cron("0 * * * * ? ")).size()); + CronUtils.getFireDateList(from.minusSeconds(1L), to, CronUtils.parse2Cron("0 * * * * ? ")).size()); from = - ZonedDateTime.ofInstant(DateUtils.stringToDate("2020-01-01 00:02:00").toInstant(), ZoneId.systemDefault()); + ZonedDateTime.ofInstant(DateUtils.stringToDate("2020-01-01 00:02:00").toInstant(), + ZoneId.systemDefault()); to = ZonedDateTime.ofInstant(DateUtils.stringToDate("2020-01-01 00:04:00").toInstant(), - ZoneId.systemDefault()); + ZoneId.systemDefault()); Assertions.assertEquals(2, - CronUtils.getFireDateList(from.minusSeconds(1L), to.minusSeconds(1L), CronUtils.parse2Cron("0 * * * * ? ")) - .size()); + CronUtils + .getFireDateList(from.minusSeconds(1L), to.minusSeconds(1L), + CronUtils.parse2Cron("0 * * * * ? ")) + .size()); } @Test diff --git a/dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/expand/CuringGlobalParamsServiceTest.java b/dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/expand/CuringGlobalParamsServiceTest.java index b8fe2245617a..464e1ea8d9e0 100644 --- a/dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/expand/CuringGlobalParamsServiceTest.java +++ b/dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/expand/CuringGlobalParamsServiceTest.java @@ -15,7 +15,6 @@ * limitations under the License. */ - package org.apache.dolphinscheduler.service.expand; import org.apache.dolphinscheduler.common.enums.CommandType; @@ -24,6 +23,13 @@ import org.apache.dolphinscheduler.plugin.task.api.enums.DataType; import org.apache.dolphinscheduler.plugin.task.api.enums.Direct; import org.apache.dolphinscheduler.plugin.task.api.model.Property; + +import java.util.ArrayList; +import java.util.Date; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -33,8 +39,6 @@ import org.mockito.Mockito; import org.mockito.junit.jupiter.MockitoExtension; -import java.util.*; - @ExtendWith(MockitoExtension.class) public class CuringGlobalParamsServiceTest { @@ -61,7 +65,8 @@ public void init() { @Test public void testConvertParameterPlaceholders() { - Mockito.when(curingGlobalParamsService.convertParameterPlaceholders(placeHolderName, globalParamMap)).thenReturn("2022-06-26"); + Mockito.when(curingGlobalParamsService.convertParameterPlaceholders(placeHolderName, globalParamMap)) + .thenReturn("2022-06-26"); String result = curingGlobalParamsService.convertParameterPlaceholders(placeHolderName, globalParamMap); Assertions.assertNotNull(result); } @@ -80,36 +85,42 @@ public void testTimeFunctionExtension() { @Test public void testCuringGlobalParams() { - //define globalMap + // define globalMap Map globalParamMap = new HashMap<>(); globalParamMap.put("globalParams1", "Params1"); - //define globalParamList + // define globalParamList List globalParamList = new ArrayList<>(); - //define scheduleTime + // define scheduleTime Date scheduleTime = DateUtils.stringToDate("2019-12-20 00:00:00"); - //test globalParamList is null - String result = dolphinSchedulerCuringGlobalParams.curingGlobalParams(1, globalParamMap, globalParamList, CommandType.START_CURRENT_TASK_PROCESS, scheduleTime, null); + // test globalParamList is null + String result = dolphinSchedulerCuringGlobalParams.curingGlobalParams(1, globalParamMap, globalParamList, + CommandType.START_CURRENT_TASK_PROCESS, scheduleTime, null); Assertions.assertNull(result); - Assertions.assertNull(dolphinSchedulerCuringGlobalParams.curingGlobalParams(1, null, null, CommandType.START_CURRENT_TASK_PROCESS, null, null)); - Assertions.assertNull(dolphinSchedulerCuringGlobalParams.curingGlobalParams(1, globalParamMap, null, CommandType.START_CURRENT_TASK_PROCESS, scheduleTime, null)); + Assertions.assertNull(dolphinSchedulerCuringGlobalParams.curingGlobalParams(1, null, null, + CommandType.START_CURRENT_TASK_PROCESS, null, null)); + Assertions.assertNull(dolphinSchedulerCuringGlobalParams.curingGlobalParams(1, globalParamMap, null, + CommandType.START_CURRENT_TASK_PROCESS, scheduleTime, null)); - //test globalParamList is not null + // test globalParamList is not null Property property = new Property("testGlobalParam", Direct.IN, DataType.VARCHAR, "testGlobalParam"); globalParamList.add(property); - String result2 = dolphinSchedulerCuringGlobalParams.curingGlobalParams(1, null, globalParamList, CommandType.START_CURRENT_TASK_PROCESS, scheduleTime, null); + String result2 = dolphinSchedulerCuringGlobalParams.curingGlobalParams(1, null, globalParamList, + CommandType.START_CURRENT_TASK_PROCESS, scheduleTime, null); Assertions.assertEquals(result2, JSONUtils.toJsonString(globalParamList)); - String result3 = dolphinSchedulerCuringGlobalParams.curingGlobalParams(1, globalParamMap, globalParamList, CommandType.START_CURRENT_TASK_PROCESS, null, null); + String result3 = dolphinSchedulerCuringGlobalParams.curingGlobalParams(1, globalParamMap, globalParamList, + CommandType.START_CURRENT_TASK_PROCESS, null, null); Assertions.assertEquals(result3, JSONUtils.toJsonString(globalParamList)); - String result4 = dolphinSchedulerCuringGlobalParams.curingGlobalParams(1, globalParamMap, globalParamList, CommandType.START_CURRENT_TASK_PROCESS, scheduleTime, null); + String result4 = dolphinSchedulerCuringGlobalParams.curingGlobalParams(1, globalParamMap, globalParamList, + CommandType.START_CURRENT_TASK_PROCESS, scheduleTime, null); Assertions.assertEquals(result4, JSONUtils.toJsonString(globalParamList)); - //test var $ startsWith + // test var $ startsWith globalParamMap.put("bizDate", "${system.biz.date}"); globalParamMap.put("b1zCurdate", "${system.biz.curdate}"); @@ -121,12 +132,14 @@ public void testCuringGlobalParams() { globalParamList.add(property3); globalParamList.add(property4); - String result5 = dolphinSchedulerCuringGlobalParams.curingGlobalParams(1, globalParamMap, globalParamList, CommandType.START_CURRENT_TASK_PROCESS, scheduleTime, null); + String result5 = dolphinSchedulerCuringGlobalParams.curingGlobalParams(1, globalParamMap, globalParamList, + CommandType.START_CURRENT_TASK_PROCESS, scheduleTime, null); Assertions.assertEquals(result5, JSONUtils.toJsonString(globalParamList)); Property testStartParamProperty = new Property("testStartParam", Direct.IN, DataType.VARCHAR, ""); globalParamList.add(testStartParamProperty); - Property testStartParam2Property = new Property("testStartParam2", Direct.IN, DataType.VARCHAR, "$[yyyy-MM-dd+1]"); + Property testStartParam2Property = + new Property("testStartParam2", Direct.IN, DataType.VARCHAR, "$[yyyy-MM-dd+1]"); globalParamList.add(testStartParam2Property); globalParamMap.put("testStartParam", ""); globalParamMap.put("testStartParam2", "$[yyyy-MM-dd+1]"); @@ -141,7 +154,8 @@ public void testCuringGlobalParams() { } } - String result6 = dolphinSchedulerCuringGlobalParams.curingGlobalParams(1, globalParamMap, globalParamList, CommandType.START_CURRENT_TASK_PROCESS, scheduleTime, null); + String result6 = dolphinSchedulerCuringGlobalParams.curingGlobalParams(1, globalParamMap, globalParamList, + CommandType.START_CURRENT_TASK_PROCESS, scheduleTime, null); Assertions.assertTrue(result6.contains("20191220")); } } diff --git a/dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/expand/TimePlaceholderResolverExpandServiceTest.java b/dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/expand/TimePlaceholderResolverExpandServiceTest.java index 025ed9f7cf22..07e44205ee34 100644 --- a/dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/expand/TimePlaceholderResolverExpandServiceTest.java +++ b/dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/expand/TimePlaceholderResolverExpandServiceTest.java @@ -18,6 +18,7 @@ package org.apache.dolphinscheduler.service.expand; import org.apache.commons.lang3.StringUtils; + import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; @@ -45,7 +46,8 @@ public void testTimePlaceholderResolverExpandService() { boolean implCheckResult = timePlaceholderResolverExpandServiceImpl.timeFunctionNeedExpand(placeHolderName); Assertions.assertFalse(implCheckResult); - String implResultString = timePlaceholderResolverExpandServiceImpl.timeFunctionExtension(1, "", placeHolderName); + String implResultString = + timePlaceholderResolverExpandServiceImpl.timeFunctionExtension(1, "", placeHolderName); Assertions.assertTrue(StringUtils.isEmpty(implResultString)); } } diff --git a/dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/log/LogClientTest.java b/dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/log/LogClientTest.java index 1c483682111b..9ccfadd49d21 100644 --- a/dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/log/LogClientTest.java +++ b/dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/log/LogClientTest.java @@ -28,6 +28,9 @@ import org.apache.dolphinscheduler.remote.factory.NettyRemotingClientFactory; import org.apache.dolphinscheduler.remote.utils.Host; import org.apache.dolphinscheduler.service.utils.LoggerUtils; + +import java.nio.charset.StandardCharsets; + import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; @@ -35,8 +38,6 @@ import org.mockito.Mockito; import org.mockito.junit.jupiter.MockitoExtension; -import java.nio.charset.StandardCharsets; - @ExtendWith(MockitoExtension.class) public class LogClientTest { diff --git a/dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/log/LoggerRequestProcessorTest.java b/dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/log/LoggerRequestProcessorTest.java index 44b783a7beee..8bfd72170935 100644 --- a/dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/log/LoggerRequestProcessorTest.java +++ b/dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/log/LoggerRequestProcessorTest.java @@ -17,12 +17,12 @@ package org.apache.dolphinscheduler.service.log; -import io.netty.channel.Channel; import org.apache.dolphinscheduler.common.utils.JSONUtils; import org.apache.dolphinscheduler.remote.command.Command; import org.apache.dolphinscheduler.remote.command.CommandType; import org.apache.dolphinscheduler.remote.command.log.ViewLogRequestCommand; import org.apache.dolphinscheduler.service.utils.LoggerUtils; + import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.BeforeEach; @@ -32,6 +32,8 @@ import org.mockito.Mockito; import org.mockito.junit.jupiter.MockitoExtension; +import io.netty.channel.Channel; + @ExtendWith(MockitoExtension.class) public class LoggerRequestProcessorTest { diff --git a/dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/log/MasterLogFilterTest.java b/dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/log/MasterLogFilterTest.java index ecc28cd8aaea..19af2561030c 100644 --- a/dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/log/MasterLogFilterTest.java +++ b/dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/log/MasterLogFilterTest.java @@ -16,13 +16,15 @@ */ package org.apache.dolphinscheduler.service.log; -import ch.qos.logback.classic.Level; -import ch.qos.logback.classic.spi.LoggingEvent; -import ch.qos.logback.core.spi.FilterReply; import org.apache.dolphinscheduler.common.Constants; + import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; +import ch.qos.logback.classic.Level; +import ch.qos.logback.classic.spi.LoggingEvent; +import ch.qos.logback.core.spi.FilterReply; + public class MasterLogFilterTest { @Test diff --git a/dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/log/SensitiveDataConverterTest.java b/dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/log/SensitiveDataConverterTest.java index d4514ab95ce6..ec99b610c9f6 100644 --- a/dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/log/SensitiveDataConverterTest.java +++ b/dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/log/SensitiveDataConverterTest.java @@ -17,16 +17,17 @@ package org.apache.dolphinscheduler.service.log; +import static org.apache.dolphinscheduler.service.log.SensitiveDataConverter.passwordHandler; + import org.apache.dolphinscheduler.common.Constants; + +import java.util.regex.Pattern; + import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.util.regex.Pattern; - -import static org.apache.dolphinscheduler.service.log.SensitiveDataConverter.passwordHandler; - public class SensitiveDataConverterTest { private final Logger logger = LoggerFactory.getLogger(SensitiveDataConverterTest.class); diff --git a/dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/log/TaskLogDiscriminatorTest.java b/dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/log/TaskLogDiscriminatorTest.java index c8bec75d08d7..e06041d6058d 100644 --- a/dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/log/TaskLogDiscriminatorTest.java +++ b/dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/log/TaskLogDiscriminatorTest.java @@ -16,13 +16,15 @@ */ package org.apache.dolphinscheduler.service.log; -import ch.qos.logback.classic.Level; -import ch.qos.logback.classic.spi.LoggingEvent; import org.apache.dolphinscheduler.plugin.task.api.TaskConstants; + import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; +import ch.qos.logback.classic.Level; +import ch.qos.logback.classic.spi.LoggingEvent; + public class TaskLogDiscriminatorTest { /** diff --git a/dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/log/TaskLogFilterTest.java b/dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/log/TaskLogFilterTest.java index 56a1023fa457..9c1c9f80c97e 100644 --- a/dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/log/TaskLogFilterTest.java +++ b/dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/log/TaskLogFilterTest.java @@ -16,13 +16,15 @@ */ package org.apache.dolphinscheduler.service.log; -import ch.qos.logback.classic.Level; -import ch.qos.logback.classic.spi.LoggingEvent; -import ch.qos.logback.core.spi.FilterReply; import org.apache.dolphinscheduler.plugin.task.api.TaskConstants; + import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; +import ch.qos.logback.classic.Level; +import ch.qos.logback.classic.spi.LoggingEvent; +import ch.qos.logback.core.spi.FilterReply; + public class TaskLogFilterTest { @Test diff --git a/dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/log/WorkerLogFilterTest.java b/dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/log/WorkerLogFilterTest.java index d0d408f797e4..c4f2c2f9a1eb 100644 --- a/dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/log/WorkerLogFilterTest.java +++ b/dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/log/WorkerLogFilterTest.java @@ -16,13 +16,15 @@ */ package org.apache.dolphinscheduler.service.log; -import ch.qos.logback.classic.Level; -import ch.qos.logback.classic.spi.LoggingEvent; -import ch.qos.logback.core.spi.FilterReply; import org.apache.dolphinscheduler.common.Constants; + import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; +import ch.qos.logback.classic.Level; +import ch.qos.logback.classic.spi.LoggingEvent; +import ch.qos.logback.core.spi.FilterReply; + public class WorkerLogFilterTest { @Test diff --git a/dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/process/ProcessServiceTest.java b/dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/process/ProcessServiceTest.java index bb47c74529a5..d0ef8f8e1808 100644 --- a/dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/process/ProcessServiceTest.java +++ b/dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/process/ProcessServiceTest.java @@ -17,18 +17,65 @@ package org.apache.dolphinscheduler.service.process; -import com.fasterxml.jackson.databind.JsonNode; +import static org.apache.dolphinscheduler.common.Constants.CMD_PARAM_RECOVER_PROCESS_ID_STRING; +import static org.apache.dolphinscheduler.common.Constants.CMD_PARAM_START_PARAMS; +import static org.apache.dolphinscheduler.common.Constants.CMD_PARAM_SUB_PROCESS_DEFINE_CODE; +import static org.mockito.ArgumentMatchers.any; + import org.apache.dolphinscheduler.common.Constants; -import org.apache.dolphinscheduler.common.enums.*; +import org.apache.dolphinscheduler.common.enums.CommandType; +import org.apache.dolphinscheduler.common.enums.Flag; +import org.apache.dolphinscheduler.common.enums.ProcessExecutionTypeEnum; +import org.apache.dolphinscheduler.common.enums.TaskGroupQueueStatus; +import org.apache.dolphinscheduler.common.enums.UserType; +import org.apache.dolphinscheduler.common.enums.WarningType; import org.apache.dolphinscheduler.common.graph.DAG; import org.apache.dolphinscheduler.common.model.TaskNodeRelation; import org.apache.dolphinscheduler.common.utils.CodeGenerateUtils; import org.apache.dolphinscheduler.common.utils.DateUtils; import org.apache.dolphinscheduler.common.utils.JSONUtils; -import org.apache.dolphinscheduler.dao.entity.*; -import org.apache.dolphinscheduler.dao.mapper.*; +import org.apache.dolphinscheduler.dao.entity.Command; +import org.apache.dolphinscheduler.dao.entity.DqExecuteResult; +import org.apache.dolphinscheduler.dao.entity.DqRule; +import org.apache.dolphinscheduler.dao.entity.DqRuleExecuteSql; +import org.apache.dolphinscheduler.dao.entity.DqRuleInputEntry; +import org.apache.dolphinscheduler.dao.entity.ProcessDefinition; +import org.apache.dolphinscheduler.dao.entity.ProcessDefinitionLog; +import org.apache.dolphinscheduler.dao.entity.ProcessInstance; +import org.apache.dolphinscheduler.dao.entity.ProcessInstanceMap; +import org.apache.dolphinscheduler.dao.entity.ProcessTaskRelationLog; +import org.apache.dolphinscheduler.dao.entity.Resource; +import org.apache.dolphinscheduler.dao.entity.TaskDefinitionLog; +import org.apache.dolphinscheduler.dao.entity.TaskGroupQueue; +import org.apache.dolphinscheduler.dao.entity.TaskInstance; +import org.apache.dolphinscheduler.dao.entity.User; +import org.apache.dolphinscheduler.dao.mapper.CommandMapper; +import org.apache.dolphinscheduler.dao.mapper.DataSourceMapper; +import org.apache.dolphinscheduler.dao.mapper.DqComparisonTypeMapper; +import org.apache.dolphinscheduler.dao.mapper.DqExecuteResultMapper; +import org.apache.dolphinscheduler.dao.mapper.DqRuleExecuteSqlMapper; +import org.apache.dolphinscheduler.dao.mapper.DqRuleInputEntryMapper; +import org.apache.dolphinscheduler.dao.mapper.DqRuleMapper; +import org.apache.dolphinscheduler.dao.mapper.ErrorCommandMapper; +import org.apache.dolphinscheduler.dao.mapper.ProcessDefinitionLogMapper; +import org.apache.dolphinscheduler.dao.mapper.ProcessDefinitionMapper; +import org.apache.dolphinscheduler.dao.mapper.ProcessInstanceMapper; +import org.apache.dolphinscheduler.dao.mapper.ProcessTaskRelationLogMapper; +import org.apache.dolphinscheduler.dao.mapper.ProcessTaskRelationMapper; +import org.apache.dolphinscheduler.dao.mapper.ResourceMapper; +import org.apache.dolphinscheduler.dao.mapper.ScheduleMapper; +import org.apache.dolphinscheduler.dao.mapper.TaskDefinitionLogMapper; +import org.apache.dolphinscheduler.dao.mapper.TaskDefinitionMapper; +import org.apache.dolphinscheduler.dao.mapper.TaskGroupMapper; +import org.apache.dolphinscheduler.dao.mapper.TaskGroupQueueMapper; +import org.apache.dolphinscheduler.dao.mapper.TaskInstanceMapper; +import org.apache.dolphinscheduler.dao.mapper.UserMapper; import org.apache.dolphinscheduler.dao.repository.ProcessInstanceDao; -import org.apache.dolphinscheduler.plugin.task.api.enums.dp.*; +import org.apache.dolphinscheduler.plugin.task.api.enums.dp.DqTaskState; +import org.apache.dolphinscheduler.plugin.task.api.enums.dp.ExecuteSqlType; +import org.apache.dolphinscheduler.plugin.task.api.enums.dp.InputType; +import org.apache.dolphinscheduler.plugin.task.api.enums.dp.OptionSourceType; +import org.apache.dolphinscheduler.plugin.task.api.enums.dp.ValueType; import org.apache.dolphinscheduler.plugin.task.api.model.DateInterval; import org.apache.dolphinscheduler.plugin.task.api.model.ResourceInfo; import org.apache.dolphinscheduler.service.cron.CronUtilsTest; @@ -38,6 +85,14 @@ import org.apache.dolphinscheduler.service.model.TaskNode; import org.apache.dolphinscheduler.service.task.TaskPluginManager; import org.apache.dolphinscheduler.spi.params.base.FormType; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.Date; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; @@ -50,10 +105,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.util.*; - -import static org.apache.dolphinscheduler.common.Constants.*; -import static org.mockito.ArgumentMatchers.any; +import com.fasterxml.jackson.databind.JsonNode; /** * process service test diff --git a/dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/queue/PeerTaskInstancePriorityQueueTest.java b/dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/queue/PeerTaskInstancePriorityQueueTest.java index 3abe0db486a6..a430ab3c950f 100644 --- a/dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/queue/PeerTaskInstancePriorityQueueTest.java +++ b/dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/queue/PeerTaskInstancePriorityQueueTest.java @@ -20,11 +20,12 @@ import org.apache.dolphinscheduler.common.enums.Priority; import org.apache.dolphinscheduler.dao.entity.TaskInstance; import org.apache.dolphinscheduler.service.exceptions.TaskPriorityQueueException; -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.Test; import java.util.concurrent.TimeUnit; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + public class PeerTaskInstancePriorityQueueTest { @Test @@ -47,7 +48,6 @@ public void take() throws Exception { Assertions.assertTrue(queue.size() < peekBeforeLength); } - @Test public void poll() throws Exception { PeerTaskInstancePriorityQueue queue = getPeerTaskInstancePriorityQueue(); @@ -64,7 +64,7 @@ public void peek() throws Exception { } @Test - public void peekTaskGroupPriority() throws Exception{ + public void peekTaskGroupPriority() throws Exception { PeerTaskInstancePriorityQueue queue = new PeerTaskInstancePriorityQueue(); TaskInstance taskInstanceHigPriority = createTaskInstance("high", Priority.HIGH, 2); TaskInstance taskInstanceMediumPriority = createTaskInstance("medium", Priority.HIGH, 1); diff --git a/dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/queue/TaskPriorityQueueImplTest.java b/dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/queue/TaskPriorityQueueImplTest.java index a78cc959a06e..aeaf079c9e3c 100644 --- a/dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/queue/TaskPriorityQueueImplTest.java +++ b/dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/queue/TaskPriorityQueueImplTest.java @@ -18,14 +18,15 @@ package org.apache.dolphinscheduler.service.queue; import org.apache.dolphinscheduler.common.enums.Priority; -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.Test; import java.util.Arrays; import java.util.Collections; import java.util.List; import java.util.concurrent.TimeUnit; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + public class TaskPriorityQueueImplTest { @Test @@ -36,9 +37,8 @@ public void testSort() { List taskPriorities = Arrays.asList(priorityOne, priorityThree, priorityTwo); Collections.sort(taskPriorities); Assertions.assertEquals( - Arrays.asList(priorityOne, priorityTwo, priorityThree), - taskPriorities - ); + Arrays.asList(priorityOne, priorityTwo, priorityThree), + taskPriorities); priorityOne = new TaskPriority(0, 1, 0, 0, 1, "default"); priorityTwo = new TaskPriority(0, 2, 0, 0, 1, "default"); @@ -46,9 +46,8 @@ public void testSort() { taskPriorities = Arrays.asList(priorityOne, priorityThree, priorityTwo); Collections.sort(taskPriorities); Assertions.assertEquals( - Arrays.asList(priorityOne, priorityTwo, priorityThree), - taskPriorities - ); + Arrays.asList(priorityOne, priorityTwo, priorityThree), + taskPriorities); priorityOne = new TaskPriority(0, 0, 1, 0, 1, "default"); priorityTwo = new TaskPriority(0, 0, 2, 0, 1, "default"); @@ -56,9 +55,8 @@ public void testSort() { taskPriorities = Arrays.asList(priorityOne, priorityThree, priorityTwo); Collections.sort(taskPriorities); Assertions.assertEquals( - Arrays.asList(priorityOne, priorityTwo, priorityThree), - taskPriorities - ); + Arrays.asList(priorityOne, priorityTwo, priorityThree), + taskPriorities); priorityOne = new TaskPriority(0, 0, 0, 1, 1, "default"); priorityTwo = new TaskPriority(0, 0, 0, 2, 1, "default"); @@ -66,49 +64,44 @@ public void testSort() { taskPriorities = Arrays.asList(priorityOne, priorityThree, priorityTwo); Collections.sort(taskPriorities); Assertions.assertEquals( - Arrays.asList(priorityOne, priorityTwo, priorityThree), - taskPriorities - ); + Arrays.asList(priorityOne, priorityTwo, priorityThree), + taskPriorities); priorityOne = new TaskPriority(0, 0, 0, 0, 1, "default_1"); - priorityTwo = new TaskPriority(0, 0, 0, 0, 1,"default_2"); + priorityTwo = new TaskPriority(0, 0, 0, 0, 1, "default_2"); priorityThree = new TaskPriority(0, 0, 0, 0, 1, "default_3"); taskPriorities = Arrays.asList(priorityOne, priorityThree, priorityTwo); Collections.sort(taskPriorities); Assertions.assertEquals( - Arrays.asList(priorityOne, priorityTwo, priorityThree), - taskPriorities - ); + Arrays.asList(priorityOne, priorityTwo, priorityThree), + taskPriorities); priorityOne = new TaskPriority(0, 0, 0, 0, 2, "default_1"); - priorityTwo = new TaskPriority(0, 0, 0, 0, 1,"default_2"); + priorityTwo = new TaskPriority(0, 0, 0, 0, 1, "default_2"); priorityThree = new TaskPriority(0, 0, 0, 0, 3, "default_3"); taskPriorities = Arrays.asList(priorityOne, priorityThree, priorityTwo); Collections.sort(taskPriorities); Assertions.assertEquals( Arrays.asList(priorityThree, priorityOne, priorityTwo), - taskPriorities - ); + taskPriorities); priorityOne = new TaskPriority(0, 0, 0, 0, 1, "default_1"); - priorityTwo = new TaskPriority(0, 0, 0, 0, 1,"default_2"); + priorityTwo = new TaskPriority(0, 0, 0, 0, 1, "default_2"); priorityThree = new TaskPriority(0, 0, 0, 0, 3, "default_3"); taskPriorities = Arrays.asList(priorityOne, priorityThree, priorityTwo); Collections.sort(taskPriorities); Assertions.assertEquals( Arrays.asList(priorityThree, priorityOne, priorityTwo), - taskPriorities - ); + taskPriorities); - priorityTwo = new TaskPriority(0, 0, 0, 0, 1,"default_1"); + priorityTwo = new TaskPriority(0, 0, 0, 0, 1, "default_1"); priorityOne = new TaskPriority(0, 0, 0, 0, 1, "default_1"); priorityThree = new TaskPriority(0, 0, 0, 0, 3, "default_1"); taskPriorities = Arrays.asList(priorityTwo, priorityOne, priorityThree); Collections.sort(taskPriorities); Assertions.assertEquals( Arrays.asList(priorityThree, priorityTwo, priorityOne), - taskPriorities - ); + taskPriorities); } @Test @@ -166,4 +159,4 @@ private TaskPriority createTaskPriority(Integer priority, Integer processInstanc TaskPriority priorityOne = new TaskPriority(priority, processInstanceId, 0, 0, 1, "default"); return priorityOne; } -} \ No newline at end of file +} diff --git a/dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/storage/impl/HadoopUtilsTest.java b/dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/storage/impl/HadoopUtilsTest.java index d9d70172c524..a7398d7953d4 100644 --- a/dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/storage/impl/HadoopUtilsTest.java +++ b/dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/storage/impl/HadoopUtilsTest.java @@ -19,6 +19,7 @@ import org.apache.dolphinscheduler.common.utils.HttpUtils; import org.apache.dolphinscheduler.spi.enums.ResourceType; + import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; diff --git a/dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/storage/impl/OssOperatorTest.java b/dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/storage/impl/OssOperatorTest.java index 384a0139d13e..68df14d9900d 100644 --- a/dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/storage/impl/OssOperatorTest.java +++ b/dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/storage/impl/OssOperatorTest.java @@ -17,8 +17,20 @@ package org.apache.dolphinscheduler.service.storage.impl; -import com.aliyun.oss.OSS; +import static org.apache.dolphinscheduler.common.Constants.FOLDER_SEPARATOR; +import static org.apache.dolphinscheduler.common.Constants.FORMAT_S_S; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.Mockito.doNothing; +import static org.mockito.Mockito.doReturn; +import static org.mockito.Mockito.spy; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; + import org.apache.dolphinscheduler.spi.enums.ResourceType; + +import java.io.IOException; + import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -26,13 +38,7 @@ import org.mockito.Mock; import org.mockito.junit.jupiter.MockitoExtension; -import java.io.IOException; - -import static org.apache.dolphinscheduler.common.Constants.FOLDER_SEPARATOR; -import static org.apache.dolphinscheduler.common.Constants.FORMAT_S_S; -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.ArgumentMatchers.anyString; -import static org.mockito.Mockito.*; +import com.aliyun.oss.OSS; @ExtendWith(MockitoExtension.class) public class OssOperatorTest { diff --git a/dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/utils/CommonUtilsTest.java b/dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/utils/CommonUtilsTest.java index 98ca251fd610..a53b3e06997e 100644 --- a/dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/utils/CommonUtilsTest.java +++ b/dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/utils/CommonUtilsTest.java @@ -19,6 +19,10 @@ import org.apache.dolphinscheduler.common.utils.FileUtils; import org.apache.dolphinscheduler.service.storage.impl.HadoopUtils; + +import java.net.InetAddress; +import java.net.UnknownHostException; + import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; @@ -26,9 +30,6 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.net.InetAddress; -import java.net.UnknownHostException; - /** * configuration test */ diff --git a/dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/utils/LogUtilsTest.java b/dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/utils/LogUtilsTest.java index 4635e9b4ef41..dcd2a734f040 100644 --- a/dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/utils/LogUtilsTest.java +++ b/dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/utils/LogUtilsTest.java @@ -17,12 +17,15 @@ package org.apache.dolphinscheduler.service.utils; -import ch.qos.logback.classic.Logger; -import ch.qos.logback.classic.sift.SiftingAppender; import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.utils.DateUtils; import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext; import org.apache.dolphinscheduler.service.log.TaskLogDiscriminator; + +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.Date; + import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; @@ -30,9 +33,8 @@ import org.mockito.junit.jupiter.MockitoExtension; import org.slf4j.LoggerFactory; -import java.nio.file.Path; -import java.nio.file.Paths; -import java.util.Date; +import ch.qos.logback.classic.Logger; +import ch.qos.logback.classic.sift.SiftingAppender; @ExtendWith(MockitoExtension.class) public class LogUtilsTest { diff --git a/dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/utils/ProcessUtilsTest.java b/dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/utils/ProcessUtilsTest.java index bec60332daea..9c74bc384413 100644 --- a/dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/utils/ProcessUtilsTest.java +++ b/dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/utils/ProcessUtilsTest.java @@ -17,11 +17,17 @@ package org.apache.dolphinscheduler.service.utils; +import static org.mockito.ArgumentMatchers.anyString; + import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.utils.OSUtils; import org.apache.dolphinscheduler.common.utils.PropertyUtils; import org.apache.dolphinscheduler.plugin.task.api.enums.TaskExecutionStatus; import org.apache.dolphinscheduler.service.storage.impl.HadoopUtils; + +import java.util.ArrayList; +import java.util.List; + import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -33,11 +39,6 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.util.ArrayList; -import java.util.List; - -import static org.mockito.ArgumentMatchers.anyString; - @ExtendWith(MockitoExtension.class) public class ProcessUtilsTest { diff --git a/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/datasource/DataSourceChannelFactory.java b/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/datasource/DataSourceChannelFactory.java index eceb39fda9b1..d423ae3a5172 100644 --- a/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/datasource/DataSourceChannelFactory.java +++ b/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/datasource/DataSourceChannelFactory.java @@ -21,6 +21,7 @@ import org.apache.dolphinscheduler.spi.plugin.SPIIdentify; public interface DataSourceChannelFactory extends PrioritySPI { + /** * get datasource client */ diff --git a/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/enums/CommandType.java b/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/enums/CommandType.java index 4bc2c1a2f623..9eef16d77e62 100644 --- a/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/enums/CommandType.java +++ b/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/enums/CommandType.java @@ -72,7 +72,7 @@ public String getDescp() { static { for (CommandType commandType : CommandType.values()) { - COMMAND_TYPE_MAP.put(commandType.code,commandType); + COMMAND_TYPE_MAP.put(commandType.code, commandType); } } diff --git a/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/enums/DbType.java b/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/enums/DbType.java index 99c5ec81876a..cd00bf1234c5 100644 --- a/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/enums/DbType.java +++ b/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/enums/DbType.java @@ -27,6 +27,7 @@ import com.google.common.base.Functions; public enum DbType { + MYSQL(0, "mysql"), POSTGRESQL(1, "postgresql"), HIVE(2, "hive"), @@ -37,8 +38,8 @@ public enum DbType { DB2(7, "db2"), PRESTO(8, "presto"), H2(9, "h2"), - REDSHIFT(10,"redshift"), - ATHENA(11,"athena"), + REDSHIFT(10, "redshift"), + ATHENA(11, "athena"), ; @EnumValue @@ -69,7 +70,8 @@ public static DbType of(int type) { } public static DbType ofName(String name) { - return Arrays.stream(DbType.values()).filter(e -> e.name().equals(name)).findFirst().orElseThrow(() -> new NoSuchElementException("no such db type")); + return Arrays.stream(DbType.values()).filter(e -> e.name().equals(name)).findFirst() + .orElseThrow(() -> new NoSuchElementException("no such db type")); } public boolean isHive() { diff --git a/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/enums/Flag.java b/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/enums/Flag.java index 0640259e0db4..242b3cdc42f5 100644 --- a/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/enums/Flag.java +++ b/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/enums/Flag.java @@ -26,6 +26,7 @@ * have_alert */ public enum Flag { + /** * 0 no * 1 yes diff --git a/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/enums/ResourceType.java b/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/enums/ResourceType.java index 8e80802b857b..a072f36ec42c 100644 --- a/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/enums/ResourceType.java +++ b/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/enums/ResourceType.java @@ -23,6 +23,7 @@ * resource type */ public enum ResourceType { + /** * 0 file, 1 udf */ diff --git a/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/params/PluginParamsTransfer.java b/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/params/PluginParamsTransfer.java index c5706d2b8969..76fb5487e14d 100644 --- a/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/params/PluginParamsTransfer.java +++ b/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/params/PluginParamsTransfer.java @@ -73,12 +73,16 @@ public static List> generatePluginParams(String paramsJsonSt * @param pluginParamsTemplate pluginParamsTemplate * @return return plugin params value */ - public static List> generatePluginParams(Map paramsMap, String pluginParamsTemplate) { + public static List> generatePluginParams(Map paramsMap, + String pluginParamsTemplate) { if (paramsMap == null || paramsMap.isEmpty()) { return null; } - List> pluginParamsList = JSONUtils.parseObject(pluginParamsTemplate, new TypeReference>>() {}); - pluginParamsList.forEach(pluginParams -> pluginParams.put(STRING_PLUGIN_PARAM_VALUE, paramsMap.get(pluginParams.get(STRING_PLUGIN_PARAM_FIELD)))); + List> pluginParamsList = + JSONUtils.parseObject(pluginParamsTemplate, new TypeReference>>() { + }); + pluginParamsList.forEach(pluginParams -> pluginParams.put(STRING_PLUGIN_PARAM_VALUE, + paramsMap.get(pluginParams.get(STRING_PLUGIN_PARAM_FIELD)))); return pluginParamsList; } } diff --git a/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/params/base/ParamsOptions.java b/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/params/base/ParamsOptions.java index 91d493637c43..cb020ab3112b 100644 --- a/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/params/base/ParamsOptions.java +++ b/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/params/base/ParamsOptions.java @@ -34,7 +34,8 @@ public class ParamsOptions { */ private boolean disabled; - public ParamsOptions() {} + public ParamsOptions() { + } public ParamsOptions(String label, Object value, boolean disabled) { this.label = label; diff --git a/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/params/base/PluginParams.java b/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/params/base/PluginParams.java index 1b65133c9412..a8b1d4f95eaa 100644 --- a/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/params/base/PluginParams.java +++ b/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/params/base/PluginParams.java @@ -17,6 +17,7 @@ package org.apache.dolphinscheduler.spi.params.base; +import static java.util.Objects.requireNonNull; import static org.apache.dolphinscheduler.spi.utils.Constants.STRING_PLUGIN_PARAM_EMIT; import static org.apache.dolphinscheduler.spi.utils.Constants.STRING_PLUGIN_PARAM_FIELD; import static org.apache.dolphinscheduler.spi.utils.Constants.STRING_PLUGIN_PARAM_NAME; @@ -26,8 +27,6 @@ import static org.apache.dolphinscheduler.spi.utils.Constants.STRING_PLUGIN_PARAM_VALIDATE; import static org.apache.dolphinscheduler.spi.utils.Constants.STRING_PLUGIN_PARAM_VALUE; -import static java.util.Objects.requireNonNull; - import java.util.List; import com.fasterxml.jackson.annotation.JsonCreator; @@ -118,7 +117,8 @@ protected PluginParams(Builder builder) { @JsonPOJOBuilder(buildMethodName = "build", withPrefix = "set") public static class Builder { - //Must have + + // Must have protected String name; protected FormType formType; @@ -127,7 +127,7 @@ public static class Builder { protected String fieldName; - //option params + // option params protected ParamsProps props; protected Object value; @@ -154,7 +154,7 @@ public Builder(String name, this.fieldName = title; } - //for json deserialize to POJO + // for json deserialize to POJO @JsonCreator public Builder(@JsonProperty("field") String name, @JsonProperty("type") FormType formType, @@ -166,8 +166,7 @@ public Builder(@JsonProperty("field") String name, @JsonProperty("emit") List emit, @JsonProperty("info") String info, @JsonProperty("hidden") Boolean hidden, - @JsonProperty("display") Boolean display - ) { + @JsonProperty("display") Boolean display) { requireNonNull(name, "name is null"); requireNonNull(formType, "formType is null"); requireNonNull(title, "title is null"); @@ -222,5 +221,3 @@ public List getEmit() { } } - - diff --git a/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/params/base/Validate.java b/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/params/base/Validate.java index d26ffac6668b..cc02e9581fdc 100644 --- a/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/params/base/Validate.java +++ b/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/params/base/Validate.java @@ -64,6 +64,7 @@ public static Builder newBuilder() { @JsonPOJOBuilder(buildMethodName = "build", withPrefix = "set") public static class Builder { + private boolean required = false; private String message; diff --git a/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/params/fswitch/SwitchParam.java b/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/params/fswitch/SwitchParam.java index b6bfe17cf983..a06bdc26f033 100644 --- a/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/params/fswitch/SwitchParam.java +++ b/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/params/fswitch/SwitchParam.java @@ -33,7 +33,7 @@ public class SwitchParam extends PluginParams { private SwitchParam(Builder builder) { super(builder); } - + private SwitchParamProps props; public static Builder newBuilder(String name, String title) { diff --git a/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/params/group/GroupParam.java b/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/params/group/GroupParam.java index 3268b6a4eefb..c2a6de5cdd3d 100644 --- a/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/params/group/GroupParam.java +++ b/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/params/group/GroupParam.java @@ -75,7 +75,7 @@ public Builder setEmit(List emit) { this.emit = emit; return this; } - + public Builder setProps(GroupParamsProps props) { this.props = props; return this; @@ -86,10 +86,10 @@ public Builder setRules(List rules) { this.setProps(new GroupParamsProps()); } - ((GroupParamsProps)this.props).setRules(rules); + ((GroupParamsProps) this.props).setRules(rules); return this; } - + @Override public GroupParam build() { return new GroupParam(this); diff --git a/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/params/input/InputParam.java b/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/params/input/InputParam.java index 2a16bd933e31..0c838edf631e 100644 --- a/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/params/input/InputParam.java +++ b/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/params/input/InputParam.java @@ -53,7 +53,7 @@ public Builder setProps(InputParamProps props) { this.props = props; return this; } - + public Builder setPlaceholder(String placeholder) { if (this.props == null) { this.setProps(new InputParamProps()); diff --git a/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/params/radio/RadioParam.java b/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/params/radio/RadioParam.java index 7ba34d94d2b7..cd4c527c5d75 100644 --- a/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/params/radio/RadioParam.java +++ b/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/params/radio/RadioParam.java @@ -53,11 +53,11 @@ public static class Builder extends PluginParams.Builder { public Builder(String name, String title) { super(name, RADIO, title); } - + private List options; private RadioParamProps props; - + public Builder setOptions(List options) { this.options = options; return this; diff --git a/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/plugin/PrioritySPIFactory.java b/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/plugin/PrioritySPIFactory.java index 8b1921b7005b..970e7cd66d56 100644 --- a/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/plugin/PrioritySPIFactory.java +++ b/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/plugin/PrioritySPIFactory.java @@ -17,13 +17,13 @@ package org.apache.dolphinscheduler.spi.plugin; -import lombok.extern.slf4j.Slf4j; - import java.util.Collections; import java.util.HashMap; import java.util.Map; import java.util.ServiceLoader; +import lombok.extern.slf4j.Slf4j; + @Slf4j public class PrioritySPIFactory { @@ -48,8 +48,9 @@ private void resolveConflict(T newSPI) { T oldSPI = map.get(identify.getName()); if (newSPI.compareTo(oldSPI.getIdentify().getPriority()) == 0) { - throw new IllegalArgumentException(String.format("These two spi plugins has conflict identify name with the same priority: %s, %s", - oldSPI.getIdentify(), newSPI.getIdentify())); + throw new IllegalArgumentException( + String.format("These two spi plugins has conflict identify name with the same priority: %s, %s", + oldSPI.getIdentify(), newSPI.getIdentify())); } else if (newSPI.compareTo(oldSPI.getIdentify().getPriority()) > 0) { log.info("The {} plugin has high priority, will override {}", newSPI.getIdentify(), oldSPI); map.put(identify.getName(), newSPI); diff --git a/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/utils/Constants.java b/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/utils/Constants.java index bb47fd64bd5d..08d7da511813 100644 --- a/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/utils/Constants.java +++ b/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/utils/Constants.java @@ -21,6 +21,7 @@ * constants */ public class Constants { + private Constants() { throw new IllegalStateException("Constants class"); } @@ -44,7 +45,6 @@ private Constants() { /**plugin param emit string **/ public static final String STRING_PLUGIN_PARAM_EMIT = "emit"; - /** string true */ public static final String STRING_TRUE = "true"; /** string false */ @@ -122,7 +122,8 @@ private Constants() { /** * hadoop.security.authentication */ - public static final String HADOOP_SECURITY_AUTHENTICATION_STARTUP_STATE = "hadoop.security.authentication.startup.state"; + public static final String HADOOP_SECURITY_AUTHENTICATION_STARTUP_STATE = + "hadoop.security.authentication.startup.state"; /** * loginUserFromKeytab user @@ -164,7 +165,6 @@ private Constants() { public static final String COM_REDSHIFT_JDBC_DRIVER = "com.amazon.redshift.jdbc42.Driver"; public static final String COM_ATHENA_JDBC_DRIVER = "com.simba.athena.jdbc.Driver"; - /** * validation Query */ @@ -225,7 +225,6 @@ private Constants() { */ public static final String COLON = ":"; - /** * AT SIGN @ */ @@ -236,7 +235,6 @@ private Constants() { */ public static final String SEMICOLON = ";"; - /** * EQUAL_SIGN = */ diff --git a/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/utils/DateUtils.java b/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/utils/DateUtils.java index 4bc950469365..58877f3e971d 100644 --- a/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/utils/DateUtils.java +++ b/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/utils/DateUtils.java @@ -27,6 +27,7 @@ import java.util.TimeZone; import javax.annotation.Nullable; + import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -441,8 +442,7 @@ public static String getTimestampString() { * @param timeStamp time stamp (milliseconds) * @return local date */ - public static @Nullable - Date timeStampToDate(long timeStamp) { + public static @Nullable Date timeStampToDate(long timeStamp) { return timeStamp <= 0L ? null : new Date(timeStamp); } } diff --git a/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/utils/JSONUtils.java b/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/utils/JSONUtils.java index 5fa432a0ff7f..cce60c30b3d7 100644 --- a/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/utils/JSONUtils.java +++ b/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/utils/JSONUtils.java @@ -17,12 +17,11 @@ package org.apache.dolphinscheduler.spi.utils; -import static java.nio.charset.StandardCharsets.UTF_8; - import static com.fasterxml.jackson.databind.DeserializationFeature.ACCEPT_EMPTY_ARRAY_AS_NULL_OBJECT; import static com.fasterxml.jackson.databind.DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES; import static com.fasterxml.jackson.databind.DeserializationFeature.READ_UNKNOWN_ENUM_VALUES_AS_NULL; import static com.fasterxml.jackson.databind.MapperFeature.REQUIRE_SETTERS_FOR_GETTERS; +import static java.nio.charset.StandardCharsets.UTF_8; import java.io.IOException; import java.text.SimpleDateFormat; @@ -34,21 +33,21 @@ import java.util.Map; import java.util.TimeZone; -import com.fasterxml.jackson.core.JsonGenerator; -import com.fasterxml.jackson.databind.JsonSerializer; -import com.fasterxml.jackson.databind.SerializerProvider; -import com.fasterxml.jackson.databind.module.SimpleModule; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import com.fasterxml.jackson.core.JsonGenerator; import com.fasterxml.jackson.core.JsonParser; import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.DeserializationContext; import com.fasterxml.jackson.databind.JsonDeserializer; import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.JsonSerializer; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.ObjectWriter; import com.fasterxml.jackson.databind.SerializationFeature; +import com.fasterxml.jackson.databind.SerializerProvider; +import com.fasterxml.jackson.databind.module.SimpleModule; import com.fasterxml.jackson.databind.node.ArrayNode; import com.fasterxml.jackson.databind.node.ObjectNode; import com.fasterxml.jackson.databind.node.TextNode; @@ -62,7 +61,7 @@ public class JSONUtils { private static final Logger logger = LoggerFactory.getLogger(JSONUtils.class); static { - logger.info("init timezone: {}",TimeZone.getDefault()); + logger.info("init timezone: {}", TimeZone.getDefault()); } private static final SimpleModule LOCAL_DATE_TIME_MODULE = new SimpleModule() @@ -190,7 +189,8 @@ public static List toList(String json, Class clazz) { * @return json to map */ public static Map toMap(String json) { - return parseObject(json, new TypeReference>() {}); + return parseObject(json, new TypeReference>() { + }); } /** @@ -204,7 +204,8 @@ public static Map toMap(String json) { * @return to map */ public static Map toMap(String json, Class classK, Class classV) { - return parseObject(json, new TypeReference>() {}); + return parseObject(json, new TypeReference>() { + }); } /** @@ -250,7 +251,7 @@ public static String toJsonString(Object object) { * @param object type * @return byte array */ - public static byte[] toJsonByteArray(T obj) { + public static byte[] toJsonByteArray(T obj) { if (obj == null) { return null; } diff --git a/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/utils/StringUtils.java b/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/utils/StringUtils.java index a98e38b59a12..44891af6475e 100644 --- a/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/utils/StringUtils.java +++ b/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/utils/StringUtils.java @@ -216,27 +216,27 @@ private static void escapeJavaStyleString(Writer out, String str, boolean escape out.write("\\u00" + hex(ch)); } else if (ch < 32) { switch (ch) { - case '\b' : + case '\b': out.write('\\'); out.write('b'); break; - case '\n' : + case '\n': out.write('\\'); out.write('n'); break; - case '\t' : + case '\t': out.write('\\'); out.write('t'); break; - case '\f' : + case '\f': out.write('\\'); out.write('f'); break; - case '\r' : + case '\r': out.write('\\'); out.write('r'); break; - default : + default: if (ch > 0xf) { out.write("\\u00" + hex(ch)); } else { @@ -246,27 +246,27 @@ private static void escapeJavaStyleString(Writer out, String str, boolean escape } } else { switch (ch) { - case '\'' : + case '\'': if (escapeSingleQuote) { out.write('\\'); } out.write('\''); break; - case '"' : + case '"': out.write('\\'); out.write('"'); break; - case '\\' : + case '\\': out.write('\\'); out.write('\\'); break; - case '/' : + case '/': if (escapeForwardSlash) { out.write('\\'); } out.write('/'); break; - default : + default: out.write(ch); break; } @@ -282,10 +282,11 @@ public static String replaceDoubleBrackets(String mainParameter) { mainParameter = mainParameter .replace(Constants.DOUBLE_BRACKETS_LEFT, Constants.DOUBLE_BRACKETS_LEFT_SPACE) .replace(Constants.DOUBLE_BRACKETS_RIGHT, Constants.DOUBLE_BRACKETS_RIGHT_SPACE); - if (mainParameter.contains(Constants.DOUBLE_BRACKETS_LEFT) || mainParameter.contains(Constants.DOUBLE_BRACKETS_RIGHT)) { + if (mainParameter.contains(Constants.DOUBLE_BRACKETS_LEFT) + || mainParameter.contains(Constants.DOUBLE_BRACKETS_RIGHT)) { return replaceDoubleBrackets(mainParameter); } else { - return mainParameter; + return mainParameter; } } } diff --git a/dolphinscheduler-spi/src/test/java/org/apache/dolphinscheduler/spi/params/PluginParamsTransferTest.java b/dolphinscheduler-spi/src/test/java/org/apache/dolphinscheduler/spi/params/PluginParamsTransferTest.java index 1bc3c3157a62..1f88584a836a 100644 --- a/dolphinscheduler-spi/src/test/java/org/apache/dolphinscheduler/spi/params/PluginParamsTransferTest.java +++ b/dolphinscheduler-spi/src/test/java/org/apache/dolphinscheduler/spi/params/PluginParamsTransferTest.java @@ -23,14 +23,15 @@ import org.apache.dolphinscheduler.spi.params.base.Validate; import org.apache.dolphinscheduler.spi.params.input.InputParam; import org.apache.dolphinscheduler.spi.params.radio.RadioParam; + +import java.util.ArrayList; +import java.util.List; + import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; -import java.util.ArrayList; -import java.util.List; - /** * PluginParamsTransfer Tester. */ @@ -182,36 +183,38 @@ public void testGetParamsJson() { @Test public void testGetPluginParams() { - String paramsJsonAssert = "[{\"props\":null,\"field\":\"field1\",\"props\":null,\"type\":\"input\",\"title\":\"field1\",\"value\":\"v1\",\"validate\":[" - + "{\"required\":true,\"message\":null,\"type\":\"string\",\"trigger\":\"blur\",\"min\":0.0,\"max\":0.0}]}," - + "{\"field\":\"field2\",\"props\":null,\"type\":\"input\",\"title\":\"field2\",\"value\":\"v2\",\"validate\":null}," - + "{\"field\":\"field3\",\"props\":null,\"type\":\"input\",\"title\":\"field3\",\"value\":\"v3\",\"validate\":[" - + "{\"required\":true,\"message\":null,\"type\":\"string\",\"trigger\":\"blur\",\"min\":0.0,\"max\":0.0}]}," - + "{\"field\":\"field4\",\"props\":null,\"type\":\"input\",\"title\":\"field4\",\"value\":\"v4\",\"validate\":[" - + "{\"required\":true,\"message\":null,\"type\":\"number\",\"trigger\":\"blur\",\"min\":0.0,\"max\":0.0}]}," - + "{\"field\":\"field5\",\"props\":null,\"type\":\"input\",\"title\":\"field5\",\"value\":\"v5\",\"validate\":[" - + "{\"required\":true,\"message\":null,\"type\":\"string\",\"trigger\":\"blur\",\"min\":0.0,\"max\":0.0}]}," - + "{\"field\":\"field6\",\"props\":null,\"type\":\"radio\",\"title\":\"field6\",\"value\":true,\"validate\":[" - + "{\"required\":true,\"message\":null,\"type\":\"string\",\"trigger\":\"blur\",\"min\":0.0,\"max\":0.0}],\"options\":[" - + "{\"label\":\"YES\",\"value\":true,\"disabled\":false},{\"label\":\"NO\",\"value\":false,\"disabled\":false}]}," - + "{\"field\":\"field7\",\"props\":{\"type\":null,\"placeholder\":\"if enable use authentication, you need input user\",\"rows\":0}," - + "\"type\":\"input\",\"title\":\"field7\",\"value\":\"v6\",\"validate\":null},{\"field\":\"field8\",\"props\":{" - + "\"type\":\"PASSWORD\",\"placeholder\":\"if enable use authentication, you need input password\",\"rows\":0}," - + "\"type\":\"input\",\"title\":\"field8\",\"value\":\"v7\",\"validate\":null},{\"field\":\"field9\"," - + "\"props\":null,\"type\":\"radio\",\"title\":\"field9\",\"value\":false,\"validate\":[" - + "{\"required\":true,\"message\":null,\"type\":\"string\",\"trigger\":\"blur\",\"min\":0.0,\"max\":0.0}],\"options\":[" - + "{\"label\":\"YES\",\"value\":true,\"disabled\":false},{\"label\":\"NO\",\"value\":false,\"disabled\":false}]}," - + "{\"field\":\"field10\",\"props\":null,\"type\":\"radio\",\"title\":\"field10\",\"value\":false,\"validate\":[" - + "{\"required\":true,\"message\":null,\"type\":\"string\",\"trigger\":\"blur\",\"min\":0.0,\"max\":0.0}]," - + "\"options\":[{\"label\":\"YES\",\"value\":true,\"disabled\":false},{\"label\":\"NO\",\"value\":false,\"disabled\":false}]}," - + "{\"field\":\"field11\",\"props\":null,\"type\":\"input\",\"title\":\"field11\",\"value\":\"*\",\"validate\":[" - + "{\"required\":true,\"message\":null,\"type\":\"string\",\"trigger\":\"blur\",\"min\":0.0,\"max\":0.0}]}," - + "{\"field\":\"showType\",\"props\":null,\"type\":\"radio\",\"title\":\"showType\",\"value\":\"table\",\"validate\":[" - + "{\"required\":true,\"message\":null,\"type\":\"string\",\"trigger\":\"blur\",\"min\":0.0,\"max\":0.0}],\"options\":[" - + "{\"label\":\"table\",\"value\":\"table\",\"disabled\":false},{\"label\":\"text\",\"value\":\"text\",\"disabled\":false}," - + "{\"label\":\"attachment\",\"value\":\"attachment\",\"disabled\":false},{\"label\":\"tableattachment\",\"value\":\"tableattachment\",\"disabled\":false}]}]"; + String paramsJsonAssert = + "[{\"props\":null,\"field\":\"field1\",\"props\":null,\"type\":\"input\",\"title\":\"field1\",\"value\":\"v1\",\"validate\":[" + + "{\"required\":true,\"message\":null,\"type\":\"string\",\"trigger\":\"blur\",\"min\":0.0,\"max\":0.0}]}," + + "{\"field\":\"field2\",\"props\":null,\"type\":\"input\",\"title\":\"field2\",\"value\":\"v2\",\"validate\":null}," + + "{\"field\":\"field3\",\"props\":null,\"type\":\"input\",\"title\":\"field3\",\"value\":\"v3\",\"validate\":[" + + "{\"required\":true,\"message\":null,\"type\":\"string\",\"trigger\":\"blur\",\"min\":0.0,\"max\":0.0}]}," + + "{\"field\":\"field4\",\"props\":null,\"type\":\"input\",\"title\":\"field4\",\"value\":\"v4\",\"validate\":[" + + "{\"required\":true,\"message\":null,\"type\":\"number\",\"trigger\":\"blur\",\"min\":0.0,\"max\":0.0}]}," + + "{\"field\":\"field5\",\"props\":null,\"type\":\"input\",\"title\":\"field5\",\"value\":\"v5\",\"validate\":[" + + "{\"required\":true,\"message\":null,\"type\":\"string\",\"trigger\":\"blur\",\"min\":0.0,\"max\":0.0}]}," + + "{\"field\":\"field6\",\"props\":null,\"type\":\"radio\",\"title\":\"field6\",\"value\":true,\"validate\":[" + + "{\"required\":true,\"message\":null,\"type\":\"string\",\"trigger\":\"blur\",\"min\":0.0,\"max\":0.0}],\"options\":[" + + "{\"label\":\"YES\",\"value\":true,\"disabled\":false},{\"label\":\"NO\",\"value\":false,\"disabled\":false}]}," + + "{\"field\":\"field7\",\"props\":{\"type\":null,\"placeholder\":\"if enable use authentication, you need input user\",\"rows\":0}," + + "\"type\":\"input\",\"title\":\"field7\",\"value\":\"v6\",\"validate\":null},{\"field\":\"field8\",\"props\":{" + + "\"type\":\"PASSWORD\",\"placeholder\":\"if enable use authentication, you need input password\",\"rows\":0}," + + "\"type\":\"input\",\"title\":\"field8\",\"value\":\"v7\",\"validate\":null},{\"field\":\"field9\"," + + "\"props\":null,\"type\":\"radio\",\"title\":\"field9\",\"value\":false,\"validate\":[" + + "{\"required\":true,\"message\":null,\"type\":\"string\",\"trigger\":\"blur\",\"min\":0.0,\"max\":0.0}],\"options\":[" + + "{\"label\":\"YES\",\"value\":true,\"disabled\":false},{\"label\":\"NO\",\"value\":false,\"disabled\":false}]}," + + "{\"field\":\"field10\",\"props\":null,\"type\":\"radio\",\"title\":\"field10\",\"value\":false,\"validate\":[" + + "{\"required\":true,\"message\":null,\"type\":\"string\",\"trigger\":\"blur\",\"min\":0.0,\"max\":0.0}]," + + "\"options\":[{\"label\":\"YES\",\"value\":true,\"disabled\":false},{\"label\":\"NO\",\"value\":false,\"disabled\":false}]}," + + "{\"field\":\"field11\",\"props\":null,\"type\":\"input\",\"title\":\"field11\",\"value\":\"*\",\"validate\":[" + + "{\"required\":true,\"message\":null,\"type\":\"string\",\"trigger\":\"blur\",\"min\":0.0,\"max\":0.0}]}," + + "{\"field\":\"showType\",\"props\":null,\"type\":\"radio\",\"title\":\"showType\",\"value\":\"table\",\"validate\":[" + + "{\"required\":true,\"message\":null,\"type\":\"string\",\"trigger\":\"blur\",\"min\":0.0,\"max\":0.0}],\"options\":[" + + "{\"label\":\"table\",\"value\":\"table\",\"disabled\":false},{\"label\":\"text\",\"value\":\"text\",\"disabled\":false}," + + "{\"label\":\"attachment\",\"value\":\"attachment\",\"disabled\":false},{\"label\":\"tableattachment\",\"value\":\"tableattachment\",\"disabled\":false}]}]"; List pluginParams = PluginParamsTransfer.transferJsonToParamsList(paramsJsonAssert); - String[] results = new String[]{"v1", "v2", "v3", "v4", "v5", "true", "v6", "v7", "false", "false", "*", "table", "v1"}; + String[] results = + new String[]{"v1", "v2", "v3", "v4", "v5", "true", "v6", "v7", "false", "false", "*", "table", "v1"}; Assertions.assertEquals(12, pluginParams.size()); for (int i = 0; i < pluginParams.size(); i++) { PluginParams param = pluginParams.get(i); diff --git a/dolphinscheduler-spi/src/test/java/org/apache/dolphinscheduler/spi/plugin/PrioritySPIFactoryTest.java b/dolphinscheduler-spi/src/test/java/org/apache/dolphinscheduler/spi/plugin/PrioritySPIFactoryTest.java index 71d3b8c8689b..64dedaa4b2d4 100644 --- a/dolphinscheduler-spi/src/test/java/org/apache/dolphinscheduler/spi/plugin/PrioritySPIFactoryTest.java +++ b/dolphinscheduler-spi/src/test/java/org/apache/dolphinscheduler/spi/plugin/PrioritySPIFactoryTest.java @@ -17,17 +17,19 @@ package org.apache.dolphinscheduler.spi.plugin; -import com.google.auto.service.AutoService; +import java.util.Map; + import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; -import java.util.Map; +import com.google.auto.service.AutoService; public class PrioritySPIFactoryTest { @Test public void loadHighPriority() { - PrioritySPIFactory factory = new PrioritySPIFactory<>(LoadHighPriorityConflictTestSPI.class); + PrioritySPIFactory factory = + new PrioritySPIFactory<>(LoadHighPriorityConflictTestSPI.class); Map spiMap = factory.getSPIMap(); Assertions.assertEquals(1, spiMap.get("A").getIdentify().getPriority()); } @@ -39,7 +41,6 @@ public void throwExceptionWhenPriorityIsSame() { }); } - public interface LoadHighPriorityConflictTestSPI extends PrioritySPI { } @@ -84,5 +85,4 @@ public SPIIdentify getIdentify() { } } - -} \ No newline at end of file +} diff --git a/dolphinscheduler-spi/src/test/java/org/apache/dolphinscheduler/spi/utils/JSONUtilsTest.java b/dolphinscheduler-spi/src/test/java/org/apache/dolphinscheduler/spi/utils/JSONUtilsTest.java index b06e3a32378e..09781d5ccfd2 100644 --- a/dolphinscheduler-spi/src/test/java/org/apache/dolphinscheduler/spi/utils/JSONUtilsTest.java +++ b/dolphinscheduler-spi/src/test/java/org/apache/dolphinscheduler/spi/utils/JSONUtilsTest.java @@ -17,20 +17,26 @@ package org.apache.dolphinscheduler.spi.utils; +import java.time.LocalDateTime; +import java.time.ZoneId; +import java.util.ArrayList; +import java.util.Date; +import java.util.HashMap; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.TimeZone; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.SerializationFeature; import com.fasterxml.jackson.databind.node.ArrayNode; import com.fasterxml.jackson.databind.node.ObjectNode; -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.Test; - -import java.time.LocalDateTime; -import java.time.ZoneId; -import java.util.*; public class JSONUtilsTest { - @Test public void createObjectNodeTest() { String jsonStr = "{\"a\":\"b\",\"b\":\"d\"}"; @@ -53,7 +59,6 @@ public void toMap() { } - @Test public void string2MapTest() { String str = list2String(); @@ -90,7 +95,6 @@ public void testParseObject() { Assertions.assertNull(JSONUtils.parseObject("foo", String.class)); } - @Test public void testJsonByteArray() { String str = "foo"; @@ -182,7 +186,9 @@ public void stringToLocalDateTime() { String time = "2022-02-22 13:38:24"; Date date = DateUtils.stringToDate(time); LocalDateTime localDateTime = LocalDateTime.ofInstant(date.toInstant(), ZoneId.systemDefault()); - List timeList = JSONUtils.parseObject("[\"2022-02-22 13:38:24\"]", new TypeReference>(){}); + List timeList = + JSONUtils.parseObject("[\"2022-02-22 13:38:24\"]", new TypeReference>() { + }); Assertions.assertNotNull(timeList); Assertions.assertEquals(1, timeList.size()); Assertions.assertEquals(localDateTime, timeList.get(0)); diff --git a/dolphinscheduler-standalone-server/src/main/java/org/apache/dolphinscheduler/StandaloneServer.java b/dolphinscheduler-standalone-server/src/main/java/org/apache/dolphinscheduler/StandaloneServer.java index 728fed8eaab4..1a95ff53ce67 100644 --- a/dolphinscheduler-standalone-server/src/main/java/org/apache/dolphinscheduler/StandaloneServer.java +++ b/dolphinscheduler-standalone-server/src/main/java/org/apache/dolphinscheduler/StandaloneServer.java @@ -21,6 +21,8 @@ import java.io.IOException; +import lombok.NonNull; + import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.boot.SpringApplication; @@ -30,8 +32,6 @@ import org.springframework.context.ApplicationListener; import org.springframework.context.event.ContextClosedEvent; -import lombok.NonNull; - @SpringBootApplication public class StandaloneServer implements ApplicationListener { diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-all/pom.xml b/dolphinscheduler-task-plugin/dolphinscheduler-task-all/pom.xml index eb18a84201ee..256aa9914419 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-all/pom.xml +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-all/pom.xml @@ -225,7 +225,7 @@ ${project.version} - + org.apache.dolphinscheduler dolphinscheduler-task-datasync ${project.version} diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/AbstractShell.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/AbstractShell.java index 72d2dcc8f769..98e33761b696 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/AbstractShell.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/AbstractShell.java @@ -148,7 +148,7 @@ private void runCommand() throws IOException { timeOutTimer = new Timer(); timeoutTimerTask = new ShellTimeoutTimerTask( this); - //One time scheduling. + // One time scheduling. timeOutTimer.schedule(timeoutTimerTask, timeOutInterval); } final BufferedReader errReader = @@ -162,6 +162,7 @@ private void runCommand() throws IOException { // read error and input streams as this would free up the buffers // free the error stream buffer Thread errThread = new Thread() { + @Override public void run() { try { @@ -177,6 +178,7 @@ public void run() { } }; Thread inThread = new Thread() { + @Override public void run() { try { @@ -204,8 +206,8 @@ public void run() { logger.warn("Interrupted while reading the error and in stream", ie); } completed.compareAndSet(false, true); - //the timeout thread handling - //taken care in finally block + // the timeout thread handling + // taken care in finally block if (exitCode != 0 || errMsg.length() > 0) { throw new ExitCodeException(exitCode, errMsg.toString()); } @@ -246,8 +248,7 @@ public void run() { * @param lines lines * @throws IOException errors */ - protected abstract void parseExecResult(BufferedReader lines) - throws IOException; + protected abstract void parseExecResult(BufferedReader lines) throws IOException; /** * get the current sub-process executing the given command @@ -291,9 +292,9 @@ public void run() { try { p.exitValue(); } catch (Exception e) { - //Process has not terminated. - //So check if it has completed - //if not just destroy it. + // Process has not terminated. + // So check if it has completed + // if not just destroy it. if (p != null && !shell.completed.get()) { shell.setTimedOut(); p.destroy(); @@ -306,6 +307,7 @@ public void run() { * This is an IOException with exit code added. */ public static class ExitCodeException extends IOException { + private final int exitCode; public ExitCodeException(int exitCode, String message) { @@ -322,6 +324,7 @@ public int getExitCode() { * process manage container */ public static class ProcessContainer extends ConcurrentHashMap { + private static final ProcessContainer container = new ProcessContainer(); private ProcessContainer() { diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/K8sTaskExecutionContext.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/K8sTaskExecutionContext.java index 4916d8e57fcd..7e39f184fab0 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/K8sTaskExecutionContext.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/K8sTaskExecutionContext.java @@ -24,6 +24,7 @@ */ public class K8sTaskExecutionContext implements Serializable { + private String configYaml; public String getConfigYaml() { @@ -37,7 +38,7 @@ public void setConfigYaml(String configYaml) { @Override public String toString() { return "K8sTaskExecutionContext{" - + "configYaml='" + configYaml + '\'' - + '}'; + + "configYaml='" + configYaml + '\'' + + '}'; } } diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/ProcessUtils.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/ProcessUtils.java index 91be73620b76..22518f7553a4 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/ProcessUtils.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/ProcessUtils.java @@ -24,12 +24,13 @@ import java.util.regex.Matcher; import java.util.regex.Pattern; +import lombok.NonNull; + import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import lombok.NonNull; - public final class ProcessUtils { + private static final Logger logger = LoggerFactory.getLogger(ProcessUtils.class); private ProcessUtils() { diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/SQLTaskExecutionContext.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/SQLTaskExecutionContext.java index 9c025e800c7b..e1d643299996 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/SQLTaskExecutionContext.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/SQLTaskExecutionContext.java @@ -27,7 +27,6 @@ */ public class SQLTaskExecutionContext implements Serializable { - /** * warningGroupId */ diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/ShellCommandExecutor.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/ShellCommandExecutor.java index 9ccc438277a7..e1a4fc4a9f67 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/ShellCommandExecutor.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/ShellCommandExecutor.java @@ -67,10 +67,8 @@ public ShellCommandExecutor(LinkedBlockingQueue logBuffer) { @Override protected String buildCommandFilePath() { // command file - return String.format("%s/%s.%s" - , taskRequest.getExecutePath() - , taskRequest.getTaskAppId() - , SystemUtils.IS_OS_WINDOWS ? "bat" : "command"); + return String.format("%s/%s.%s", taskRequest.getExecutePath(), taskRequest.getTaskAppId(), + SystemUtils.IS_OS_WINDOWS ? "bat" : "command"); } /** diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/ShellExecutor.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/ShellExecutor.java index 19efe20620b2..322035055841 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/ShellExecutor.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/ShellExecutor.java @@ -116,8 +116,7 @@ public static String execCommand(Map env, String[] cmd, * @return the output of the executed command. * @throws IOException errors */ - public static String execCommand(Map env, String... cmd) - throws IOException { + public static String execCommand(Map env, String... cmd) throws IOException { return execCommand(env, cmd, 0L); } @@ -154,7 +153,6 @@ public String getOutput() { return (output == null) ? "" : output.toString(); } - /** * Returns the commands of this instance. * Arguments with spaces in are presented with quotes round; other diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/TaskChannelFactory.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/TaskChannelFactory.java index 51bf1957e4e0..f7719b4728e9 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/TaskChannelFactory.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/TaskChannelFactory.java @@ -18,8 +18,8 @@ package org.apache.dolphinscheduler.plugin.task.api; import org.apache.dolphinscheduler.spi.common.UiChannelFactory; -import org.apache.dolphinscheduler.spi.plugin.SPIIdentify; import org.apache.dolphinscheduler.spi.plugin.PrioritySPI; +import org.apache.dolphinscheduler.spi.plugin.SPIIdentify; public interface TaskChannelFactory extends UiChannelFactory, PrioritySPI { diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/TaskConstants.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/TaskConstants.java index 11e9b97e82df..648f17cddaff 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/TaskConstants.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/TaskConstants.java @@ -18,8 +18,6 @@ package org.apache.dolphinscheduler.plugin.task.api; import java.time.Duration; -import java.util.Arrays; -import java.util.List; public class TaskConstants { diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/enums/DataType.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/enums/DataType.java index 22365f9f818f..16e0a0aa6529 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/enums/DataType.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/enums/DataType.java @@ -33,5 +33,5 @@ public enum DataType { * 8 Boolean * 9 list */ - VARCHAR,INTEGER,LONG,FLOAT,DOUBLE,DATE,TIME,TIMESTAMP,BOOLEAN,LIST + VARCHAR, INTEGER, LONG, FLOAT, DOUBLE, DATE, TIME, TIMESTAMP, BOOLEAN, LIST } diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/enums/DependResult.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/enums/DependResult.java index 0b24dad861b4..539f2e5712bf 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/enums/DependResult.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/enums/DependResult.java @@ -22,7 +22,6 @@ */ public enum DependResult { - /** * 0 success * 1 waiting diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/enums/DependentRelation.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/enums/DependentRelation.java index 75a02e776653..2a60a5e8f31b 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/enums/DependentRelation.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/enums/DependentRelation.java @@ -22,5 +22,5 @@ */ public enum DependentRelation { - AND,OR; + AND, OR; } diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/enums/Direct.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/enums/Direct.java index df76ed1fd8ac..7bb4112ecdb6 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/enums/Direct.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/enums/Direct.java @@ -24,5 +24,5 @@ public enum Direct { /** * 0 in; 1 out; */ - IN,OUT + IN, OUT } diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/enums/TaskTimeoutStrategy.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/enums/TaskTimeoutStrategy.java index 9a2818436e26..3d6c27a1aad8 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/enums/TaskTimeoutStrategy.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/enums/TaskTimeoutStrategy.java @@ -23,6 +23,7 @@ * task timeout strategy */ public enum TaskTimeoutStrategy { + /** * 0 warn * 1 failed diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/enums/UdfType.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/enums/UdfType.java index b9298cec92b2..b65dc07152f6 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/enums/UdfType.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/enums/UdfType.java @@ -21,6 +21,7 @@ * UDF type */ public enum UdfType { + /** * 0 hive; 1 spark */ diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/enums/dp/CheckType.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/enums/dp/CheckType.java index 964ef7f8e5e2..c1fec2e857d2 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/enums/dp/CheckType.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/enums/dp/CheckType.java @@ -26,16 +26,17 @@ * quality task result check type */ public enum CheckType { + /** * 0-comparison_minus_statistics * 1-statistics_minus_comparison * 2-statistics_comparison_percentage * 3-statistics_comparison_difference_comparison_percentage */ - COMPARISON_MINUS_STATISTICS(0,"comparison_minus_statistics"), - STATISTICS_MINUS_COMPARISON(1,"statistics_minus_comparison"), - STATISTICS_COMPARISON_PERCENTAGE(2,"statistics_comparison_percentage"), - STATISTICS_COMPARISON_DIFFERENCE_COMPARISON_PERCENTAGE(3,"statistics_comparison_difference_comparison_percentage"); + COMPARISON_MINUS_STATISTICS(0, "comparison_minus_statistics"), + STATISTICS_MINUS_COMPARISON(1, "statistics_minus_comparison"), + STATISTICS_COMPARISON_PERCENTAGE(2, "statistics_comparison_percentage"), + STATISTICS_COMPARISON_DIFFERENCE_COMPARISON_PERCENTAGE(3, "statistics_comparison_difference_comparison_percentage"); CheckType(int code, String description) { this.code = code; @@ -58,7 +59,7 @@ public String getDescription() { static { for (CheckType type : CheckType.values()) { - VALUES_MAP.put(type.code,type); + VALUES_MAP.put(type.code, type); } } @@ -68,4 +69,4 @@ public static CheckType of(Integer status) { } throw new IllegalArgumentException("invalid code : " + status); } -} \ No newline at end of file +} diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/enums/dp/ConnectorType.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/enums/dp/ConnectorType.java index f3d901a1b4c8..0e81d64f8431 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/enums/dp/ConnectorType.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/enums/dp/ConnectorType.java @@ -26,12 +26,13 @@ * connector type */ public enum ConnectorType { + /** * 0-jdbc * 1-hive */ - JDBC(0,"JDBC"), - HIVE(1,"HIVE"); + JDBC(0, "JDBC"), + HIVE(1, "HIVE"); ConnectorType(int code, String description) { this.code = code; @@ -54,7 +55,7 @@ public String getDescription() { static { for (ConnectorType type : ConnectorType.values()) { - VALUES_MAP.put(type.code,type); + VALUES_MAP.put(type.code, type); } } @@ -64,4 +65,4 @@ public static ConnectorType of(Integer status) { } throw new IllegalArgumentException("invalid code : " + status); } -} \ No newline at end of file +} diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/enums/dp/DqFailureStrategy.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/enums/dp/DqFailureStrategy.java index 34bc6a65a018..1c139a31a9b1 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/enums/dp/DqFailureStrategy.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/enums/dp/DqFailureStrategy.java @@ -24,6 +24,7 @@ * failure policy when dqs task node failed. */ public enum DqFailureStrategy { + /** * 0-alert and continue when dqc tasks failed. * 1-alert and block when dqc tasks failed. @@ -51,7 +52,7 @@ public String getDescription() { static { for (DqFailureStrategy type : DqFailureStrategy.values()) { - VALUES_MAP.put(type.code,type); + VALUES_MAP.put(type.code, type); } } diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/enums/dp/DqTaskState.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/enums/dp/DqTaskState.java index d927d5727084..710db2e4ae91 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/enums/dp/DqTaskState.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/enums/dp/DqTaskState.java @@ -26,14 +26,15 @@ * data quality task state */ public enum DqTaskState { + /** * 0-default * 1-success * 2-failure */ - DEFAULT(0,"default"), - SUCCESS(1,"success"), - FAILURE(2,"failure"); + DEFAULT(0, "default"), + SUCCESS(1, "success"), + FAILURE(2, "failure"); DqTaskState(int code, String description) { this.code = code; @@ -56,7 +57,7 @@ public String getDescription() { static { for (DqTaskState type : DqTaskState.values()) { - VALUES_MAP.put(type.code,type); + VALUES_MAP.put(type.code, type); } } @@ -66,4 +67,4 @@ public static DqTaskState of(Integer status) { } throw new IllegalArgumentException("invalid code : " + status); } -} \ No newline at end of file +} diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/enums/dp/ExecuteSqlType.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/enums/dp/ExecuteSqlType.java index ae6415226f38..f47cd1247898 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/enums/dp/ExecuteSqlType.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/enums/dp/ExecuteSqlType.java @@ -26,14 +26,15 @@ * execute sql type */ public enum ExecuteSqlType { + /** * 0-middle * 1-statistics * 2-comparison */ - MIDDLE(0,"middle"), - STATISTICS(1,"statistics"), - COMPARISON(2,"comparison"); + MIDDLE(0, "middle"), + STATISTICS(1, "statistics"), + COMPARISON(2, "comparison"); ExecuteSqlType(int code, String description) { this.code = code; @@ -56,7 +57,7 @@ public String getDescription() { static { for (ExecuteSqlType type : ExecuteSqlType.values()) { - VALUES_MAP.put(type.code,type); + VALUES_MAP.put(type.code, type); } } @@ -66,4 +67,4 @@ public static ExecuteSqlType of(Integer status) { } throw new IllegalArgumentException("invalid code : " + status); } -} \ No newline at end of file +} diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/enums/dp/InputType.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/enums/dp/InputType.java index 5489fb1cf811..938b8849513e 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/enums/dp/InputType.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/enums/dp/InputType.java @@ -26,16 +26,17 @@ * frontend form input entry type */ public enum InputType { + /** * 0-default * 1-statistics * 2-comparison * 3-check */ - DEFAULT(0,"default"), - STATISTICS(1,"statistics"), - COMPARISON(2,"comparison"), - CHECK(3,"check"); + DEFAULT(0, "default"), + STATISTICS(1, "statistics"), + COMPARISON(2, "comparison"), + CHECK(3, "check"); InputType(int code, String description) { this.code = code; @@ -58,7 +59,7 @@ public String getDescription() { static { for (InputType type : InputType.values()) { - VALUES_MAP.put(type.code,type); + VALUES_MAP.put(type.code, type); } } @@ -68,4 +69,4 @@ public static InputType of(Integer status) { } throw new IllegalArgumentException("invalid code : " + status); } -} \ No newline at end of file +} diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/enums/dp/OperatorType.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/enums/dp/OperatorType.java index 898dc12d7696..08e55f6e9d4a 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/enums/dp/OperatorType.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/enums/dp/OperatorType.java @@ -26,6 +26,7 @@ * operator type */ public enum OperatorType { + /** * 0-equal * 1-little than @@ -34,12 +35,12 @@ public enum OperatorType { * 4-great and equal * 5-not equal */ - EQ(0,"equal"), - LT(1,"little than"), - LE(2,"little and equal"), - GT(3,"great than"), - GE(4,"great and equal"), - NE(5,"not equal"); + EQ(0, "equal"), + LT(1, "little than"), + LE(2, "little and equal"), + GT(3, "great than"), + GE(4, "great and equal"), + NE(5, "not equal"); OperatorType(int code, String description) { this.code = code; @@ -62,7 +63,7 @@ public String getDescription() { static { for (OperatorType type : OperatorType.values()) { - VALUES_MAP.put(type.code,type); + VALUES_MAP.put(type.code, type); } } @@ -72,4 +73,4 @@ public static OperatorType of(Integer status) { } throw new IllegalArgumentException("invalid code : " + status); } -} \ No newline at end of file +} diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/enums/dp/OptionSourceType.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/enums/dp/OptionSourceType.java index 9e99c96aa79d..20da3894e34e 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/enums/dp/OptionSourceType.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/enums/dp/OptionSourceType.java @@ -26,16 +26,17 @@ * form options source type */ public enum OptionSourceType { + /** * 0-default * 1-datasource_id * 2-datasource_type * 3-comparison_type */ - DEFAULT(0,"default"), - DATASOURCE_ID(1,"datasource_id"), - DATASOURCE_TYPE(2,"datasource_type"), - COMPARISON_TYPE(3,"comparison_type"); + DEFAULT(0, "default"), + DATASOURCE_ID(1, "datasource_id"), + DATASOURCE_TYPE(2, "datasource_type"), + COMPARISON_TYPE(3, "comparison_type"); OptionSourceType(int code, String description) { this.code = code; @@ -58,7 +59,7 @@ public String getDescription() { static { for (OptionSourceType type : OptionSourceType.values()) { - VALUES_MAP.put(type.code,type); + VALUES_MAP.put(type.code, type); } } @@ -68,4 +69,4 @@ public static OptionSourceType of(Integer status) { } throw new IllegalArgumentException("invalid code : " + status); } -} \ No newline at end of file +} diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/enums/dp/RuleType.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/enums/dp/RuleType.java index bcede62e4b4b..46b42621f058 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/enums/dp/RuleType.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/enums/dp/RuleType.java @@ -26,16 +26,17 @@ * rule type */ public enum RuleType { + /** * 0-single_table * 1-single_table_custom_sql * 2-multi_table_accuracy * 3-multi_table_comparison */ - SINGLE_TABLE(0,"single_table"), - SINGLE_TABLE_CUSTOM_SQL(1,"single_table_custom_sql"), - MULTI_TABLE_ACCURACY(2,"multi_table_accuracy"), - MULTI_TABLE_COMPARISON(3,"multi_table_comparison"); + SINGLE_TABLE(0, "single_table"), + SINGLE_TABLE_CUSTOM_SQL(1, "single_table_custom_sql"), + MULTI_TABLE_ACCURACY(2, "multi_table_accuracy"), + MULTI_TABLE_COMPARISON(3, "multi_table_comparison"); RuleType(int code, String description) { this.code = code; @@ -58,7 +59,7 @@ public String getDescription() { static { for (RuleType type : RuleType.values()) { - VALUES_MAP.put(type.code,type); + VALUES_MAP.put(type.code, type); } } @@ -68,4 +69,4 @@ public static RuleType of(Integer status) { } throw new IllegalArgumentException("invalid code : " + status); } -} \ No newline at end of file +} diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/enums/dp/ValueType.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/enums/dp/ValueType.java index d63bf14cb7f4..0e1150af0357 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/enums/dp/ValueType.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/enums/dp/ValueType.java @@ -26,16 +26,17 @@ * rule input entry value type */ public enum ValueType { + /** * 0-string * 1-list * 2-number * 3-sql */ - STRING(0,"string"), - LIST(1,"list"), - NUMBER(2,"number"), - LIKE_SQL(3,"sql"); + STRING(0, "string"), + LIST(1, "list"), + NUMBER(2, "number"), + LIKE_SQL(3, "sql"); ValueType(int code, String description) { this.code = code; @@ -58,7 +59,7 @@ public String getDescription() { static { for (ValueType type : ValueType.values()) { - VALUES_MAP.put(type.code,type); + VALUES_MAP.put(type.code, type); } } @@ -68,4 +69,4 @@ public static ValueType of(Integer status) { } throw new IllegalArgumentException("invalid code : " + status); } -} \ No newline at end of file +} diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/k8s/AbstractK8sTask.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/k8s/AbstractK8sTask.java index 45670764052b..9cab4ed41946 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/k8s/AbstractK8sTask.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/k8s/AbstractK8sTask.java @@ -18,7 +18,6 @@ package org.apache.dolphinscheduler.plugin.task.api.k8s; import org.apache.dolphinscheduler.plugin.task.api.AbstractRemoteTask; -import org.apache.dolphinscheduler.plugin.task.api.AbstractTask; import org.apache.dolphinscheduler.plugin.task.api.TaskCallBack; import org.apache.dolphinscheduler.plugin.task.api.TaskException; import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext; @@ -26,6 +25,7 @@ import org.apache.dolphinscheduler.plugin.task.api.model.TaskResponse; public abstract class AbstractK8sTask extends AbstractRemoteTask { + /** * process task */ @@ -37,7 +37,7 @@ public abstract class AbstractK8sTask extends AbstractRemoteTask { */ protected AbstractK8sTask(TaskExecutionContext taskRequest) { super(taskRequest); - this.abstractK8sTaskExecutor = new K8sTaskExecutor(logger,taskRequest); + this.abstractK8sTaskExecutor = new K8sTaskExecutor(logger, taskRequest); } // todo split handle to submit and track @@ -49,7 +49,7 @@ public void handle(TaskCallBack taskCallBack) throws TaskException { setAppIds(response.getAppIds()); } catch (Exception e) { exitStatusCode = -1; - throw new TaskException("k8s process failure",e); + throw new TaskException("k8s process failure", e); } } diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/k8s/AbstractK8sTaskExecutor.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/k8s/AbstractK8sTaskExecutor.java index 1d619b26b2ec..d38064a07dc5 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/k8s/AbstractK8sTaskExecutor.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/k8s/AbstractK8sTaskExecutor.java @@ -27,6 +27,7 @@ import org.slf4j.Logger; public abstract class AbstractK8sTaskExecutor { + protected Logger logger; protected TaskExecutionContext taskRequest; protected K8sUtils k8sUtils; diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/k8s/K8sTaskMainParameters.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/k8s/K8sTaskMainParameters.java index 86caf43934f1..b0940ae83fc3 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/k8s/K8sTaskMainParameters.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/k8s/K8sTaskMainParameters.java @@ -82,12 +82,12 @@ public void setParamsMap(Map paramsMap) { @Override public String toString() { return "K8sTaskMainParameters{" - + "image='" + image + '\'' - + ", namespaceName='" + namespaceName + '\'' - + ", clusterName='" + clusterName + '\'' - + ", minCpuCores=" + minCpuCores - + ", minMemorySpace=" + minMemorySpace - + ", paramsMap=" + paramsMap - + '}'; + + "image='" + image + '\'' + + ", namespaceName='" + namespaceName + '\'' + + ", clusterName='" + clusterName + '\'' + + ", minCpuCores=" + minCpuCores + + ", minMemorySpace=" + minMemorySpace + + ", paramsMap=" + paramsMap + + '}'; } } diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/loop/LoopTaskDefinition.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/loop/LoopTaskDefinition.java index 4cf027c00cf1..d6d133f63abe 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/loop/LoopTaskDefinition.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/loop/LoopTaskDefinition.java @@ -19,8 +19,8 @@ import lombok.NonNull; -public interface LoopTaskDefinition { +public interface LoopTaskDefinition { + /** * The task name of this loop task, e.g: K8sPodTask */ diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/loop/LoopTaskInstanceStatus.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/loop/LoopTaskInstanceStatus.java index 6d66dfd1cbdb..78c610a87b2b 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/loop/LoopTaskInstanceStatus.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/loop/LoopTaskInstanceStatus.java @@ -21,6 +21,7 @@ * The return result of {@link LoopTaskQueryStatusMethodDefinition#queryTaskInstanceStatus(LoopTaskInstanceInfo)}. */ public interface LoopTaskInstanceStatus { + /** * Judge if the task instance is finished. * diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/loop/LoopTaskMethodDefinition.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/loop/LoopTaskMethodDefinition.java index 0c0b5067a62c..344050eab3a2 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/loop/LoopTaskMethodDefinition.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/loop/LoopTaskMethodDefinition.java @@ -20,7 +20,8 @@ import lombok.NonNull; public interface LoopTaskMethodDefinition { + @NonNull LoopTaskMethodType getLoopTaskMethodType(); - + } diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/loop/template/LoopTaskYamlDefinition.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/loop/template/LoopTaskYamlDefinition.java index 2e4582fb1bce..47756ce43ab0 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/loop/template/LoopTaskYamlDefinition.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/loop/template/LoopTaskYamlDefinition.java @@ -24,11 +24,13 @@ @Data public class LoopTaskYamlDefinition implements Serializable { + // todo: support multiple services private LoopTaskServiceYamlDefinition service; @Data public static class LoopTaskServiceYamlDefinition implements Serializable { + private String name; private String type; private LoopTaskAPIYamlDefinition api; @@ -36,6 +38,7 @@ public static class LoopTaskServiceYamlDefinition implements Serializable { @Data public static class LoopTaskAPIYamlDefinition implements Serializable { + private LoopTaskSubmitMethodYamlDefinition submit; private LoopTaskQueryStateYamlDefinition queryState; private LoopTaskCancelYamlDefinition cancel; @@ -44,6 +47,7 @@ public static class LoopTaskAPIYamlDefinition implements Serializable { @Data @SuppressWarnings("checkstyle:ModifierOrder") public static abstract class LoopTaskMethodYamlDefinition { + private String url; private String method; private String dataType; @@ -68,6 +72,7 @@ public static class LoopTaskSubmitMethodYamlDefinition extends LoopTaskMethodYam @Data public static class LoopTaskQueryStateYamlDefinition extends LoopTaskMethodYamlDefinition { + /** * Used to extract taskInstance finished state from response * todo: we need to support the function to calculate the finished state diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/loop/template/TemplateMethodTransformer.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/loop/template/TemplateMethodTransformer.java index 9f09a6c476d3..524362cc3ce7 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/loop/template/TemplateMethodTransformer.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/loop/template/TemplateMethodTransformer.java @@ -21,8 +21,7 @@ import lombok.NonNull; -public interface TemplateMethodTransformer { +public interface TemplateMethodTransformer { /** * Transform the {@link LoopTaskYamlDefinition.LoopTaskMethodYamlDefinition} to {@link HttpLoopTaskMethodDefinition}. diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/loop/template/http/HttpLoopTaskDefinition.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/loop/template/http/HttpLoopTaskDefinition.java index 2d80d5caa10a..0f9b45f24564 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/loop/template/http/HttpLoopTaskDefinition.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/loop/template/http/HttpLoopTaskDefinition.java @@ -24,8 +24,9 @@ import lombok.NonNull; -public class HttpLoopTaskDefinition implements - LoopTaskDefinition { +public class HttpLoopTaskDefinition + implements + LoopTaskDefinition { private final String taskName; private final HttpLoopTaskSubmitTaskMethodDefinition submitTaskMethod; diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/loop/template/http/HttpLoopTaskMethodDefinition.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/loop/template/http/HttpLoopTaskMethodDefinition.java index d5270863cb42..88c7de8f744f 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/loop/template/http/HttpLoopTaskMethodDefinition.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/loop/template/http/HttpLoopTaskMethodDefinition.java @@ -29,6 +29,7 @@ @Data @AllArgsConstructor public abstract class HttpLoopTaskMethodDefinition implements LoopTaskMethodDefinition { + protected final String url; protected final String httpMethodType; protected final String dataType; diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/loop/template/http/method/HttpLoopTaskCancelTaskMethodDefinition.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/loop/template/http/method/HttpLoopTaskCancelTaskMethodDefinition.java index 240cc74bfb7d..11620164c943 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/loop/template/http/method/HttpLoopTaskCancelTaskMethodDefinition.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/loop/template/http/method/HttpLoopTaskCancelTaskMethodDefinition.java @@ -29,7 +29,8 @@ import javax.annotation.Nullable; public class HttpLoopTaskCancelTaskMethodDefinition extends HttpLoopTaskMethodDefinition - implements LoopTaskCancelMethodDefinition { + implements + LoopTaskCancelMethodDefinition { private final String taskInstanceIdHolder = "${taskInstanceId}"; @@ -69,7 +70,7 @@ public void cancelTaskInstance(@Nullable LoopTaskInstanceInfo loopTaskInstanceIn OkHttpUtils.post(url, httpHeaders, requestParams, requestBody); } else { throw new IllegalArgumentException(String.format("http method type: %s is not supported", - httpMethodType)); + httpMethodType)); } } catch (IllegalArgumentException ex) { throw ex; diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/loop/template/http/method/HttpLoopTaskQueryStatusMethodDefinition.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/loop/template/http/method/HttpLoopTaskQueryStatusMethodDefinition.java index dbc44a932d6e..3c7740fc1116 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/loop/template/http/method/HttpLoopTaskQueryStatusMethodDefinition.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/loop/template/http/method/HttpLoopTaskQueryStatusMethodDefinition.java @@ -32,7 +32,8 @@ import lombok.NonNull; public class HttpLoopTaskQueryStatusMethodDefinition extends HttpLoopTaskMethodDefinition - implements LoopTaskQueryStatusMethodDefinition { + implements + LoopTaskQueryStatusMethodDefinition { private final String taskInstanceFinishedJPath; // inject the taskInstanceId into template @@ -75,7 +76,7 @@ public HttpLoopTaskQueryStatusMethodDefinition(String url, responseBody = OkHttpUtils.post(url, httpHeaders, requestParams, requestBody); } else { throw new IllegalArgumentException(String.format("http method type: %s is not supported", - httpMethodType)); + httpMethodType)); } } catch (IllegalArgumentException ex) { throw ex; diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/loop/template/http/method/HttpLoopTaskSubmitTaskMethodDefinition.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/loop/template/http/method/HttpLoopTaskSubmitTaskMethodDefinition.java index 20852a9cb44a..ec6e3af1a680 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/loop/template/http/method/HttpLoopTaskSubmitTaskMethodDefinition.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/loop/template/http/method/HttpLoopTaskSubmitTaskMethodDefinition.java @@ -32,7 +32,8 @@ import lombok.NonNull; public class HttpLoopTaskSubmitTaskMethodDefinition extends HttpLoopTaskMethodDefinition - implements LoopTaskSubmitTaskMethodDefinition { + implements + LoopTaskSubmitTaskMethodDefinition { private final String taskInstanceIdJPath; @@ -58,7 +59,7 @@ public HttpLoopTaskSubmitTaskMethodDefinition(String url, responseBody = OkHttpUtils.post(url, httpHeaders, requestParams, requestBody); } else { throw new IllegalArgumentException(String.format("The request method type: %s is not supported.", - httpMethodType)); + httpMethodType)); } } catch (IllegalArgumentException ex) { throw ex; @@ -67,9 +68,9 @@ public HttpLoopTaskSubmitTaskMethodDefinition(String url, } Optional taskInstanceIdOptional = JsonPathUtils.read(responseBody, taskInstanceIdJPath); String taskInstanceId = taskInstanceIdOptional.orElseThrow(() -> new RuntimeException(String.format( - "Resolve the taskInstanceId error, responseBody: %s, taskInstanceIdJPath: %s", - responseBody, - taskInstanceIdJPath))); + "Resolve the taskInstanceId error, responseBody: %s, taskInstanceIdJPath: %s", + responseBody, + taskInstanceIdJPath))); return new HttpLoopTaskInstanceInfo(taskInstanceId); } } diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/loop/template/http/parser/CancelTemplateMethodTransformer.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/loop/template/http/parser/CancelTemplateMethodTransformer.java index bcc95ba5c7ad..b367b4c4413b 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/loop/template/http/parser/CancelTemplateMethodTransformer.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/loop/template/http/parser/CancelTemplateMethodTransformer.java @@ -25,8 +25,9 @@ import lombok.NonNull; -public class CancelTemplateMethodTransformer implements - TemplateMethodTransformer { +public class CancelTemplateMethodTransformer + implements + TemplateMethodTransformer { @Override public @NonNull HttpLoopTaskCancelTaskMethodDefinition transform(@NonNull LoopTaskYamlDefinition.LoopTaskCancelYamlDefinition loopTaskAPIYamlDefinition) { @@ -37,10 +38,10 @@ public class CancelTemplateMethodTransformer implements Map requestParams = loopTaskAPIYamlDefinition.getRequestParams(); Map requestBody = loopTaskAPIYamlDefinition.getRequestBody(); return new HttpLoopTaskCancelTaskMethodDefinition(url, - method, - dataType, - httpHeaders, - requestParams, - requestBody); + method, + dataType, + httpHeaders, + requestParams, + requestBody); } } diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/loop/template/http/parser/HttpTaskDefinitionParser.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/loop/template/http/parser/HttpTaskDefinitionParser.java index 1bf723736d8f..b00942fbd528 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/loop/template/http/parser/HttpTaskDefinitionParser.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/loop/template/http/parser/HttpTaskDefinitionParser.java @@ -29,13 +29,13 @@ import java.io.FileReader; import java.io.IOException; +import lombok.NonNull; + import org.yaml.snakeyaml.Yaml; import org.yaml.snakeyaml.constructor.Constructor; import com.google.common.base.Preconditions; -import lombok.NonNull; - public class HttpTaskDefinitionParser implements TaskDefinitionParser { @Override @@ -50,12 +50,12 @@ public class HttpTaskDefinitionParser implements TaskDefinitionParser { +public class QueryStateTemplateMethodTransformer + implements + TemplateMethodTransformer { @Override public @NonNull HttpLoopTaskQueryStatusMethodDefinition transform(@NonNull LoopTaskYamlDefinition.LoopTaskQueryStateYamlDefinition loopTaskAPIYamlDefinition) { @@ -38,11 +39,11 @@ public class QueryStateTemplateMethodTransformer implements Map requestBody = loopTaskAPIYamlDefinition.getRequestBody(); String taskInstanceFinishedJPath = loopTaskAPIYamlDefinition.getTaskInstanceFinishedJPath(); return new HttpLoopTaskQueryStatusMethodDefinition(url, - method, - dataType, - httpHeaders, - requestParams, - requestBody, - taskInstanceFinishedJPath); + method, + dataType, + httpHeaders, + requestParams, + requestBody, + taskInstanceFinishedJPath); } } diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/loop/template/http/parser/SubmitTemplateMethodTransformer.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/loop/template/http/parser/SubmitTemplateMethodTransformer.java index 05b476a2889f..2cc7956f85a3 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/loop/template/http/parser/SubmitTemplateMethodTransformer.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/loop/template/http/parser/SubmitTemplateMethodTransformer.java @@ -25,8 +25,9 @@ import lombok.NonNull; -public class SubmitTemplateMethodTransformer implements - TemplateMethodTransformer { +public class SubmitTemplateMethodTransformer + implements + TemplateMethodTransformer { @Override public @NonNull HttpLoopTaskSubmitTaskMethodDefinition transform(@NonNull LoopTaskYamlDefinition.LoopTaskSubmitMethodYamlDefinition loopTaskAPIYamlDefinition) { @@ -38,11 +39,11 @@ public class SubmitTemplateMethodTransformer implements Map requestBody = loopTaskAPIYamlDefinition.getRequestBody(); String taskInstanceIdJPath = loopTaskAPIYamlDefinition.getTaskInstanceIdJPath(); return new HttpLoopTaskSubmitTaskMethodDefinition(url, - method, - dataType, - httpHeaders, - requestParams, - requestBody, - taskInstanceIdJPath); + method, + dataType, + httpHeaders, + requestParams, + requestBody, + taskInstanceIdJPath); } } diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/model/DateInterval.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/model/DateInterval.java index 893cee1d3fa5..ad5f34de1200 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/model/DateInterval.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/model/DateInterval.java @@ -24,6 +24,7 @@ * date interval class */ public class DateInterval { + private Date startTime; private Date endTime; diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/model/DependentItem.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/model/DependentItem.java index 0a9cf3f4999a..e19d1c1a5198 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/model/DependentItem.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/model/DependentItem.java @@ -17,10 +17,11 @@ package org.apache.dolphinscheduler.plugin.task.api.model; -import lombok.Data; import org.apache.dolphinscheduler.plugin.task.api.enums.DependResult; import org.apache.dolphinscheduler.plugin.task.api.enums.TaskExecutionStatus; +import lombok.Data; + /** * dependent item */ diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/model/DependentTaskModel.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/model/DependentTaskModel.java index 3501db67c8bb..08ebeb979e90 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/model/DependentTaskModel.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/model/DependentTaskModel.java @@ -17,11 +17,12 @@ package org.apache.dolphinscheduler.plugin.task.api.model; -import lombok.Data; import org.apache.dolphinscheduler.plugin.task.api.enums.DependentRelation; import java.util.List; +import lombok.Data; + @Data public class DependentTaskModel { diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/model/Property.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/model/Property.java index 7f51c7d9b2b4..bac4e651df99 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/model/Property.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/model/Property.java @@ -17,8 +17,8 @@ package org.apache.dolphinscheduler.plugin.task.api.model; -import org.apache.dolphinscheduler.plugin.task.api.enums.Direct; import org.apache.dolphinscheduler.plugin.task.api.enums.DataType; +import org.apache.dolphinscheduler.plugin.task.api.enums.Direct; import java.io.Serializable; import java.util.Objects; diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/model/SwitchResultVo.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/model/SwitchResultVo.java index dd28fc771138..65577890ccb2 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/model/SwitchResultVo.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/model/SwitchResultVo.java @@ -50,4 +50,4 @@ public void setNextNode(Object nextNode) { this.nextNode = (ArrayList) nextNode; } } -} \ No newline at end of file +} diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/parameters/AbstractParameters.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/parameters/AbstractParameters.java index 76c8b9c3fca3..d8cfb86316ac 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/parameters/AbstractParameters.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/parameters/AbstractParameters.java @@ -40,6 +40,7 @@ * job params related class */ public abstract class AbstractParameters implements IParameters { + @Override public abstract boolean checkParameters(); @@ -79,7 +80,7 @@ public Map getLocalParametersMap() { Map localParametersMaps = new LinkedHashMap<>(); if (localParams != null) { for (Property property : localParams) { - localParametersMaps.put(property.getProp(),property); + localParametersMaps.put(property.getProp(), property); } } return localParametersMaps; diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/parameters/ConditionsParameters.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/parameters/ConditionsParameters.java index a258ebdac449..922c344aae96 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/parameters/ConditionsParameters.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/parameters/ConditionsParameters.java @@ -26,7 +26,7 @@ public class ConditionsParameters extends AbstractParameters { - //depend node list and state, only need task name + // depend node list and state, only need task name private List dependTaskList; private DependentRelation dependRelation; @@ -80,9 +80,9 @@ public void setFailedNode(List failedNode) { public String getConditionResult() { return "{" - + "\"successNode\": [\"" + successNode.get(0) - + "\"],\"failedNode\": [\"" + failedNode.get(0) - + "\"]}"; + + "\"successNode\": [\"" + successNode.get(0) + + "\"],\"failedNode\": [\"" + failedNode.get(0) + + "\"]}"; } } diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/parameters/DependentParameters.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/parameters/DependentParameters.java index e0e6b40068ed..2b90f4a1cc66 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/parameters/DependentParameters.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/parameters/DependentParameters.java @@ -17,13 +17,14 @@ package org.apache.dolphinscheduler.plugin.task.api.parameters; -import lombok.Data; -import lombok.EqualsAndHashCode; import org.apache.dolphinscheduler.plugin.task.api.enums.DependentRelation; import org.apache.dolphinscheduler.plugin.task.api.model.DependentTaskModel; import java.util.List; +import lombok.Data; +import lombok.EqualsAndHashCode; + @Data @EqualsAndHashCode(callSuper = true) public class DependentParameters extends AbstractParameters { diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/parameters/IParameters.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/parameters/IParameters.java index 02603efa46cd..bcd914c8bedd 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/parameters/IParameters.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/parameters/IParameters.java @@ -1,19 +1,19 @@ package org.apache.dolphinscheduler.plugin.task.api.parameters;/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ import org.apache.dolphinscheduler.plugin.task.api.model.ResourceInfo; @@ -23,6 +23,7 @@ * job params interface */ public interface IParameters { + /** * check parameters is valid * diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/parameters/K8sTaskParameters.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/parameters/K8sTaskParameters.java index d9ab201c36c6..8121c1f7784c 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/parameters/K8sTaskParameters.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/parameters/K8sTaskParameters.java @@ -18,7 +18,6 @@ package org.apache.dolphinscheduler.plugin.task.api.parameters; import org.apache.dolphinscheduler.plugin.task.api.model.ResourceInfo; -import org.apache.dolphinscheduler.plugin.task.api.parameters.AbstractParameters; import org.apache.dolphinscheduler.spi.utils.StringUtils; import java.util.ArrayList; @@ -28,6 +27,7 @@ * k8s task parameters */ public class K8sTaskParameters extends AbstractParameters { + private String image; private String namespace; private double minCpuCores; @@ -67,8 +67,7 @@ public void setMinMemorySpace(double minMemorySpace) { @Override public boolean checkParameters() { - return StringUtils.isNotEmpty(image) && StringUtils.isNotEmpty(namespace) - ; + return StringUtils.isNotEmpty(image) && StringUtils.isNotEmpty(namespace); } @Override @@ -79,10 +78,10 @@ public List getResourceFilesList() { @Override public String toString() { return "K8sTaskParameters{" - + "image='" + image + '\'' - + ", namespace='" + namespace + '\'' - + ", minCpuCores=" + minCpuCores - + ", minMemorySpace=" + minMemorySpace - + '}'; + + "image='" + image + '\'' + + ", namespace='" + namespace + '\'' + + ", minCpuCores=" + minCpuCores + + ", minMemorySpace=" + minMemorySpace + + '}'; } } diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/parameters/ParametersNode.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/parameters/ParametersNode.java index 0f6b4107e379..b6eb88a68a41 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/parameters/ParametersNode.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/parameters/ParametersNode.java @@ -36,6 +36,7 @@ public static ParametersNode.ParametersNodeBuilder builder() { } public static class ParametersNodeBuilder { + private String taskType; private String taskParams; diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/parameters/SubProcessParameters.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/parameters/SubProcessParameters.java index 3616bc57f58d..336831393581 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/parameters/SubProcessParameters.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/parameters/SubProcessParameters.java @@ -37,4 +37,4 @@ public boolean checkParameters() { return this.processDefinitionCode != 0; } -} \ No newline at end of file +} diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/parameters/SwitchParameters.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/parameters/SwitchParameters.java index 502ea1470191..efdcba62ee51 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/parameters/SwitchParameters.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/parameters/SwitchParameters.java @@ -86,4 +86,4 @@ public void setNextNode(Object nextNode) { this.nextNode = (ArrayList) nextNode; } } -} \ No newline at end of file +} diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/parameters/dataquality/DataQualityParameters.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/parameters/dataquality/DataQualityParameters.java index f257c427ce2d..c6475c9631a2 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/parameters/dataquality/DataQualityParameters.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/parameters/dataquality/DataQualityParameters.java @@ -17,9 +17,9 @@ package org.apache.dolphinscheduler.plugin.task.api.parameters.dataquality; +import org.apache.dolphinscheduler.plugin.task.api.model.ResourceInfo; import org.apache.dolphinscheduler.plugin.task.api.parameters.AbstractParameters; import org.apache.dolphinscheduler.plugin.task.api.parameters.dataquality.spark.SparkParameters; -import org.apache.dolphinscheduler.plugin.task.api.model.ResourceInfo; import org.apache.dolphinscheduler.plugin.task.api.utils.MapUtils; import java.util.ArrayList; @@ -43,7 +43,7 @@ public class DataQualityParameters extends AbstractParameters { /** * rule input entry value map */ - private Map ruleInputParameter; + private Map ruleInputParameter; /** * spark parameters */ diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/parameters/resource/AbstractResourceParameters.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/parameters/resource/AbstractResourceParameters.java index b719f95b4edb..4d3ec179548a 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/parameters/resource/AbstractResourceParameters.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/parameters/resource/AbstractResourceParameters.java @@ -28,5 +28,4 @@ }) public abstract class AbstractResourceParameters { - } diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/parameters/resource/DataSourceParameters.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/parameters/resource/DataSourceParameters.java index ea9cb7ae5a4f..963eb9a3b46b 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/parameters/resource/DataSourceParameters.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/parameters/resource/DataSourceParameters.java @@ -17,9 +17,10 @@ package org.apache.dolphinscheduler.plugin.task.api.parameters.resource; -import com.fasterxml.jackson.annotation.JsonProperty; import org.apache.dolphinscheduler.spi.enums.DbType; +import com.fasterxml.jackson.annotation.JsonProperty; + public class DataSourceParameters extends AbstractResourceParameters { private DbType type; diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/parser/BusinessTimeUtils.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/parser/BusinessTimeUtils.java index b09d9f897268..1dd4b4252904 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/parser/BusinessTimeUtils.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/parser/BusinessTimeUtils.java @@ -35,6 +35,7 @@ * business time utils */ public class BusinessTimeUtils { + private BusinessTimeUtils() { throw new IllegalStateException("BusinessTimeUtils class"); } diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/parser/ParamUtils.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/parser/ParamUtils.java index 15a7c14b76d4..c106f7932d42 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/parser/ParamUtils.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/parser/ParamUtils.java @@ -17,9 +17,9 @@ package org.apache.dolphinscheduler.plugin.task.api.parser; +import org.apache.dolphinscheduler.plugin.task.api.enums.DataType; import org.apache.dolphinscheduler.plugin.task.api.enums.Direct; import org.apache.dolphinscheduler.plugin.task.api.model.Property; -import org.apache.dolphinscheduler.plugin.task.api.enums.DataType; import java.util.HashMap; import java.util.Iterator; @@ -36,7 +36,7 @@ public class ParamUtils { * @param paramsMap params map * @return Map of converted */ - public static Map convert(Map paramsMap) { + public static Map convert(Map paramsMap) { if (paramsMap == null) { return null; } @@ -63,9 +63,9 @@ public static Map getUserDefParamsMap(Map defi while (iter.hasNext()) { Map.Entry en = iter.next(); Property property = new Property(en.getKey(), Direct.IN, DataType.VARCHAR, en.getValue()); - userDefParamsMaps.put(property.getProp(),property); + userDefParamsMaps.put(property.getProp(), property); } } return userDefParamsMaps; } -} \ No newline at end of file +} diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/parser/ParameterUtils.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/parser/ParameterUtils.java index 8d3e265d6edf..393a51bf0df2 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/parser/ParameterUtils.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/parser/ParameterUtils.java @@ -73,7 +73,7 @@ public static String convertParameterPlaceholders(String parameterString, Map params, String s ret.append(","); } } - for (Object v : valueList ) { + for (Object v : valueList) { Property newProperty = new Property(); - if (v instanceof Integer) { + if (v instanceof Integer) { newProperty.setType(DataType.INTEGER); } else if (v instanceof Long) { newProperty.setType(DataType.LONG); @@ -215,7 +216,7 @@ public static String expandListParameter(Map params, String s */ public static String replaceScheduleTime(String text, Date scheduleTime) { Map paramsMap = new HashMap<>(); - //if getScheduleTime null ,is current date + // if getScheduleTime null ,is current date if (null == scheduleTime) { scheduleTime = new Date(); } diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/parser/PlaceholderUtils.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/parser/PlaceholderUtils.java index f266ced99b73..3820646364d4 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/parser/PlaceholderUtils.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/parser/PlaceholderUtils.java @@ -52,15 +52,16 @@ public class PlaceholderUtils { public static String replacePlaceholders(String value, Map paramsMap, boolean ignoreUnresolvablePlaceholders) { - //replacement tool， parameter key will be replaced by value,if can't match , will throw an exception + // replacement tool， parameter key will be replaced by value,if can't match , will throw an exception PropertyPlaceholderHelper strictHelper = getPropertyPlaceholderHelper(false); - //Non-strict replacement tool implementation, when the position to be replaced does not get the corresponding value, the current position is ignored, and the next position is replaced. + // Non-strict replacement tool implementation, when the position to be replaced does not get the corresponding + // value, the current position is ignored, and the next position is replaced. PropertyPlaceholderHelper nonStrictHelper = getPropertyPlaceholderHelper(true); PropertyPlaceholderHelper helper = (ignoreUnresolvablePlaceholders ? nonStrictHelper : strictHelper); - //the PlaceholderResolver to use for replacement + // the PlaceholderResolver to use for replacement return helper.replacePlaceholders(value, new PropertyPlaceholderResolver(value, paramsMap)); } @@ -72,7 +73,8 @@ public static String replacePlaceholders(String value, */ public static PropertyPlaceholderHelper getPropertyPlaceholderHelper(boolean ignoreUnresolvablePlaceholders) { - return new PropertyPlaceholderHelper(PLACEHOLDER_PREFIX, PLACEHOLDER_SUFFIX, null, ignoreUnresolvablePlaceholders); + return new PropertyPlaceholderHelper(PLACEHOLDER_PREFIX, PLACEHOLDER_SUFFIX, null, + ignoreUnresolvablePlaceholders); } /** diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/parser/PropertyPlaceholderHelper.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/parser/PropertyPlaceholderHelper.java index a4b08a6f98bc..98b6e01dc8f9 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/parser/PropertyPlaceholderHelper.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/parser/PropertyPlaceholderHelper.java @@ -48,7 +48,6 @@ public class PropertyPlaceholderHelper { wellKnownSimplePrefixes.put(")", "("); } - private final String placeholderPrefix; private final String placeholderSuffix; @@ -88,8 +87,7 @@ public PropertyPlaceholderHelper(String placeholderPrefix, String placeholderSuf String simplePrefixForSuffix = wellKnownSimplePrefixes.get(this.placeholderSuffix); if (simplePrefixForSuffix != null && this.placeholderPrefix.endsWith(simplePrefixForSuffix)) { this.simplePrefix = simplePrefixForSuffix; - } - else { + } else { this.simplePrefix = this.placeholderPrefix; } this.valueSeparator = valueSeparator; @@ -106,6 +104,7 @@ public PropertyPlaceholderHelper(String placeholderPrefix, String placeholderSuf public String replacePlaceholders(String value, final Properties properties) { notNull(properties, "'properties' must not be null"); return replacePlaceholders(value, new PlaceholderResolver() { + @Override public String resolvePlaceholder(String placeholderName) { return properties.getProperty(placeholderName); @@ -126,7 +125,8 @@ public String replacePlaceholders(String value, PlaceholderResolver placeholderR } protected String parseStringValue( - String value, PlaceholderResolver placeholderResolver, Set visitedPlaceholders) { + String value, PlaceholderResolver placeholderResolver, + Set visitedPlaceholders) { StringBuilder result = new StringBuilder(value); @@ -164,18 +164,15 @@ protected String parseStringValue( logger.trace("Resolved placeholder '" + placeholder + "'"); } startIndex = result.indexOf(this.placeholderPrefix, startIndex + propVal.length()); - } - else if (this.ignoreUnresolvablePlaceholders) { + } else if (this.ignoreUnresolvablePlaceholders) { // Proceed with unprocessed value. startIndex = result.indexOf(this.placeholderPrefix, endIndex + this.placeholderSuffix.length()); - } - else { + } else { throw new IllegalArgumentException("Could not resolve placeholder '" + placeholder + "'" + " in value \"" + value + "\""); } visitedPlaceholders.remove(originalPlaceholder); - } - else { + } else { startIndex = -1; } } @@ -191,16 +188,13 @@ private int findPlaceholderEndIndex(CharSequence buf, int startIndex) { if (withinNestedPlaceholder > 0) { withinNestedPlaceholder--; index = index + this.placeholderSuffix.length(); - } - else { + } else { return index; } - } - else if (substringMatch(buf, index, this.simplePrefix)) { + } else if (substringMatch(buf, index, this.simplePrefix)) { withinNestedPlaceholder++; index = index + this.simplePrefix.length(); - } - else { + } else { index++; } } @@ -252,4 +246,3 @@ public static void notNull(Object object, String message) { } } - diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/parser/TimePlaceholderUtils.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/parser/TimePlaceholderUtils.java index 4080d0d9c73e..518b5e52299b 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/parser/TimePlaceholderUtils.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/parser/TimePlaceholderUtils.java @@ -69,6 +69,7 @@ * time place holder utils */ public class TimePlaceholderUtils { + private static final Logger logger = LoggerFactory.getLogger(TimePlaceholderUtils.class); /** @@ -105,7 +106,8 @@ public static String replacePlaceholders(String value, Date date, boolean ignore * be ignored ({@code true}) or cause an exception ({@code false}) */ private static PropertyPlaceholderHelper getPropertyPlaceholderHelper(boolean ignoreUnresolvablePlaceholders) { - return new PropertyPlaceholderHelper(PLACEHOLDER_PREFIX, PLACEHOLDER_SUFFIX, null, ignoreUnresolvablePlaceholders); + return new PropertyPlaceholderHelper(PLACEHOLDER_PREFIX, PLACEHOLDER_SUFFIX, null, + ignoreUnresolvablePlaceholders); } /** @@ -139,7 +141,8 @@ private static String convert(String expression) { arr[i] = N; } else { char c = arr[i - 1]; - if (c == ADD_CHAR || c == SUBTRACT_CHAR || c == MULTIPLY_CHAR || c == DIVISION_CHAR || c == LEFT_BRACE_CHAR) { + if (c == ADD_CHAR || c == SUBTRACT_CHAR || c == MULTIPLY_CHAR || c == DIVISION_CHAR + || c == LEFT_BRACE_CHAR) { arr[i] = N; } } @@ -148,7 +151,8 @@ private static String convert(String expression) { arr[i] = P; } else { char c = arr[i - 1]; - if (c == ADD_CHAR || c == SUBTRACT_CHAR || c == MULTIPLY_CHAR || c == DIVISION_CHAR || c == LEFT_BRACE_CHAR) { + if (c == ADD_CHAR || c == SUBTRACT_CHAR || c == MULTIPLY_CHAR || c == DIVISION_CHAR + || c == LEFT_BRACE_CHAR) { arr[i] = P; } } @@ -285,9 +289,11 @@ private static List string2List(String expression) { * @return true or false */ private static boolean compare(String peek, String cur) { - if (MULTIPLY_STRING.equals(peek) && (DIVISION_STRING.equals(cur) || MULTIPLY_STRING.equals(cur) || ADD_STRING.equals(cur) || SUBTRACT_STRING.equals(cur))) { + if (MULTIPLY_STRING.equals(peek) && (DIVISION_STRING.equals(cur) || MULTIPLY_STRING.equals(cur) + || ADD_STRING.equals(cur) || SUBTRACT_STRING.equals(cur))) { return true; - } else if (DIVISION_STRING.equals(peek) && (DIVISION_STRING.equals(cur) || MULTIPLY_STRING.equals(cur) || ADD_STRING.equals(cur) || SUBTRACT_STRING.equals(cur))) { + } else if (DIVISION_STRING.equals(peek) && (DIVISION_STRING.equals(cur) || MULTIPLY_STRING.equals(cur) + || ADD_STRING.equals(cur) || SUBTRACT_STRING.equals(cur))) { return true; } else if (ADD_STRING.equals(peek) && (ADD_STRING.equals(cur) || SUBTRACT_STRING.equals(cur))) { return true; @@ -300,8 +306,9 @@ private static boolean compare(String peek, String cur) { /** * Placeholder replacement resolver */ - private static class TimePlaceholderResolver implements - PropertyPlaceholderHelper.PlaceholderResolver { + private static class TimePlaceholderResolver + implements + PropertyPlaceholderHelper.PlaceholderResolver { private final String value; @@ -362,7 +369,7 @@ private static String calculateTime(String expression, Date date) { value = String.valueOf(timestamp.getTime() / 1000); } else if (expression.startsWith(YEAR_WEEK)) { - value = calculateYearWeek(expression,date); + value = calculateYearWeek(expression, date); } else { Map.Entry entry = calcTimeExpression(expression, date); value = DateUtils.format(entry.getKey(), entry.getValue()); @@ -388,7 +395,7 @@ public static String calculateYearWeek(String expression, Date date) { String targetDate = ""; try { - if(dataFormat.contains(COMMA)) { + if (dataFormat.contains(COMMA)) { String param1 = dataFormat.split(COMMA)[0]; String param2 = dataFormat.split(COMMA)[1]; dataFormat = param1; @@ -414,10 +421,10 @@ public static String calculateYearWeek(String expression, Date date) { */ private static String transformYearWeek(Date date, String format, int weekDay) { Calendar calendar = Calendar.getInstance(); - //Minimum number of days required for the first week of the year + // Minimum number of days required for the first week of the year calendar.setMinimalDaysInFirstWeek(4); - //By default ,Set Monday as the first day of the week + // By default ,Set Monday as the first day of the week switch (weekDay) { case 2: calendar.setFirstDayOfWeek(Calendar.TUESDAY); @@ -449,13 +456,13 @@ private static String transformYearWeek(Date date, String format, int weekDay) { String weekYearStr = ""; if (weekOfYear < 10 && format.contains(HYPHEN)) { - weekYearStr = String.format("%d%s0%d",year,HYPHEN,weekOfYear); - } else if (weekOfYear >= 10 && format.contains(HYPHEN)){ - weekYearStr = String.format("%d%s%d",year,HYPHEN,weekOfYear); - } else if (weekOfYear < 10){ - weekYearStr = String.format("%d0%d",year,weekOfYear); + weekYearStr = String.format("%d%s0%d", year, HYPHEN, weekOfYear); + } else if (weekOfYear >= 10 && format.contains(HYPHEN)) { + weekYearStr = String.format("%d%s%d", year, HYPHEN, weekOfYear); + } else if (weekOfYear < 10) { + weekYearStr = String.format("%d0%d", year, weekOfYear); } else { - weekYearStr = String.format("%d%d",year,weekOfYear); + weekYearStr = String.format("%d%d", year, weekOfYear); } return weekYearStr; @@ -482,17 +489,17 @@ public static Map.Entry calcTimeExpression(String expression, Date } else if (expression.startsWith(WEEK_END)) { resultEntry = calcWeekEnd(expression, date); } else if (expression.startsWith(MONTH_FIRST_DAY)) { - resultEntry = calcCustomDay(expression,MONTH_FIRST_DAY, date); + resultEntry = calcCustomDay(expression, MONTH_FIRST_DAY, date); } else if (expression.startsWith(MONTH_LAST_DAY)) { - resultEntry = calcCustomDay(expression,MONTH_LAST_DAY, date); + resultEntry = calcCustomDay(expression, MONTH_LAST_DAY, date); } else if (expression.startsWith(THIS_DAY)) { - resultEntry = calcCustomDay(expression,THIS_DAY, date); + resultEntry = calcCustomDay(expression, THIS_DAY, date); } else if (expression.startsWith(LAST_DAY)) { - resultEntry = calcCustomDay(expression,LAST_DAY, date); + resultEntry = calcCustomDay(expression, LAST_DAY, date); } else if (expression.startsWith(WEEK_FIRST_DAY)) { - resultEntry = calcCustomDay(expression,WEEK_FIRST_DAY, date); + resultEntry = calcCustomDay(expression, WEEK_FIRST_DAY, date); } else if (expression.startsWith(WEEK_LAST_DAY)) { - resultEntry = calcCustomDay(expression,WEEK_LAST_DAY, date); + resultEntry = calcCustomDay(expression, WEEK_LAST_DAY, date); } else { resultEntry = calcMinutes(expression, date); } @@ -555,7 +562,7 @@ public static Map.Entry calcMonthEnd(String expression, Date date) * @param date date * @return calculate time expression with date,format */ - public static Map.Entry calcCustomDay(String expression,String keyDate, Date date) { + public static Map.Entry calcCustomDay(String expression, String keyDate, Date date) { String dataFormat = "yyyy-MM-dd"; Date targetDate = new Date(); @@ -563,12 +570,12 @@ public static Map.Entry calcCustomDay(String expression,String key case MONTH_FIRST_DAY: dataFormat = expression.substring(MONTH_FIRST_DAY.length() + 1, expression.length() - 1); - if(dataFormat.contains(COMMA)) { + if (dataFormat.contains(COMMA)) { String param1 = dataFormat.split(COMMA)[0]; String param2 = dataFormat.split(COMMA)[1]; dataFormat = param1; - targetDate = addMonths(DateUtils.getFirstDayOfMonth(date),calculate(param2)); + targetDate = addMonths(DateUtils.getFirstDayOfMonth(date), calculate(param2)); } else { targetDate = DateUtils.getFirstDayOfMonth(date); } @@ -577,12 +584,12 @@ public static Map.Entry calcCustomDay(String expression,String key case MONTH_LAST_DAY: dataFormat = expression.substring(MONTH_LAST_DAY.length() + 1, expression.length() - 1); - if(dataFormat.contains(COMMA)) { + if (dataFormat.contains(COMMA)) { String param1 = dataFormat.split(COMMA)[0]; String param2 = dataFormat.split(COMMA)[1]; dataFormat = param1; - Date lastMonthDay = addMonths(date,calculate(param2)); + Date lastMonthDay = addMonths(date, calculate(param2)); targetDate = DateUtils.getLastDayOfMonth(lastMonthDay); @@ -601,7 +608,7 @@ public static Map.Entry calcCustomDay(String expression,String key case WEEK_FIRST_DAY: dataFormat = expression.substring(WEEK_FIRST_DAY.length() + 1, expression.length() - 1); - if(dataFormat.contains(COMMA)) { + if (dataFormat.contains(COMMA)) { String param1 = dataFormat.split(COMMA)[0]; String param2 = dataFormat.split(COMMA)[1]; dataFormat = param1; @@ -614,7 +621,7 @@ public static Map.Entry calcCustomDay(String expression,String key case WEEK_LAST_DAY: dataFormat = expression.substring(WEEK_LAST_DAY.length() + 1, expression.length() - 1); - if(dataFormat.contains(COMMA)) { + if (dataFormat.contains(COMMA)) { String param1 = dataFormat.split(COMMA)[0]; String param2 = dataFormat.split(COMMA)[1]; dataFormat = param1; @@ -755,7 +762,7 @@ public static Integer calcMinutes(String minuteExpression) { } else { calcExpression = String.format("60*24*(%s)%s", minuteExpression.substring(0, index), - minuteExpression.substring(index)); + minuteExpression.substring(index)); } return calculate(calcExpression); diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/stream/StreamTask.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/stream/StreamTask.java index 51ab17e15661..15fb4d5ee4bd 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/stream/StreamTask.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/stream/StreamTask.java @@ -18,5 +18,6 @@ package org.apache.dolphinscheduler.plugin.task.api.stream; public interface StreamTask { + public void savePoint() throws Exception; } diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/stream/StreamTaskChannel.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/stream/StreamTaskChannel.java index a3bf9e5e23e3..6c35b3cec41f 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/stream/StreamTaskChannel.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/stream/StreamTaskChannel.java @@ -20,9 +20,6 @@ import org.apache.dolphinscheduler.plugin.task.api.AbstractTask; import org.apache.dolphinscheduler.plugin.task.api.TaskChannel; import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext; -import org.apache.dolphinscheduler.plugin.task.api.parameters.AbstractParameters; -import org.apache.dolphinscheduler.plugin.task.api.parameters.ParametersNode; -import org.apache.dolphinscheduler.plugin.task.api.parameters.resource.ResourceParametersHelper; public interface StreamTaskChannel extends TaskChannel { diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/utils/DependentDateUtils.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/utils/DependentDateUtils.java index abecf94a34d7..88fc080d4469 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/utils/DependentDateUtils.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/utils/DependentDateUtils.java @@ -179,7 +179,7 @@ public static List getLastWeekInterval(Date businessDate) { */ public static List getLastWeekOneDayInterval(Date businessDate, int dayOfWeek) { Date mondayThisWeek = DateUtils.getMonday(businessDate); - Date sunday = DateUtils.getSomeDay(mondayThisWeek, - 1); + Date sunday = DateUtils.getSomeDay(mondayThisWeek, -1); Date monday = DateUtils.getMonday(sunday); Date destDay = DateUtils.getSomeDay(monday, dayOfWeek - 1); return getDateIntervalListBetweenTwoDates(destDay, destDay); diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/utils/JsonPathUtils.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/utils/JsonPathUtils.java index e6e8cb219da2..452ed3af8604 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/utils/JsonPathUtils.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/utils/JsonPathUtils.java @@ -20,14 +20,14 @@ import java.util.Optional; import java.util.Set; -import com.jayway.jsonpath.JsonPath; -import com.jayway.jsonpath.Option; -import com.jayway.jsonpath.internal.DefaultsImpl; - import lombok.NonNull; import lombok.experimental.UtilityClass; import net.minidev.json.JSONArray; +import com.jayway.jsonpath.JsonPath; +import com.jayway.jsonpath.Option; +import com.jayway.jsonpath.internal.DefaultsImpl; + @UtilityClass public class JsonPathUtils { diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/utils/OkHttpUtils.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/utils/OkHttpUtils.java index 84cfe2a7308e..db6625d0c6a2 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/utils/OkHttpUtils.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/utils/OkHttpUtils.java @@ -58,7 +58,7 @@ public class OkHttpUtils { Request.Builder requestBuilder = new Request.Builder().url(finalUrl); if (requestBodyMap != null) { requestBuilder = requestBuilder.post(RequestBody.create(MediaType.parse("application/json"), - JSONUtils.toJsonString(requestBodyMap))); + JSONUtils.toJsonString(requestBodyMap))); } try (Response response = CLIENT.newCall(requestBuilder.build()).execute()) { return getResponseBody(response); @@ -91,8 +91,8 @@ private static void addHeader(@Nullable Map headers, @NonNull Re private static String getResponseBody(@NonNull Response response) throws IOException { if (response.code() != HttpStatus.SC_OK || response.body() == null) { throw new RuntimeException(String.format("Request execute failed, httpCode: %s, httpBody: %s", - response.code(), - response.body())); + response.code(), + response.body())); } return response.body().string(); } diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/utils/RetryUtils.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/utils/RetryUtils.java index 2dcf3efce4f0..1246d85f1548 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/utils/RetryUtils.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/utils/RetryUtils.java @@ -62,6 +62,7 @@ public static T retryFunction(@NonNull Supplier supplier, @NonNull RetryP @Data public static final class RetryPolicy { + /** * The max retry times */ diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/test/java/org/apache/dolphinscheduler/plugin/task/api/model/JSONUtilsTest.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/test/java/org/apache/dolphinscheduler/plugin/task/api/model/JSONUtilsTest.java index b52a9a099b39..c17902f56873 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/test/java/org/apache/dolphinscheduler/plugin/task/api/model/JSONUtilsTest.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/test/java/org/apache/dolphinscheduler/plugin/task/api/model/JSONUtilsTest.java @@ -156,7 +156,8 @@ public void testNodeString() { Assertions.assertEquals("", JSONUtils.getNodeString("{\"bar\":\"foo\"}", "key")); Assertions.assertEquals("foo", JSONUtils.getNodeString("{\"bar\":\"foo\"}", "bar")); Assertions.assertEquals("[1,2,3]", JSONUtils.getNodeString("{\"bar\": [1,2,3]}", "bar")); - Assertions.assertEquals("{\"1\":\"2\",\"2\":3}", JSONUtils.getNodeString("{\"bar\": {\"1\":\"2\",\"2\":3}}", "bar")); + Assertions.assertEquals("{\"1\":\"2\",\"2\":3}", + JSONUtils.getNodeString("{\"bar\": {\"1\":\"2\",\"2\":3}}", "bar")); } @Test @@ -216,7 +217,8 @@ public void testToJsonString() { Assertions.assertEquals("{\"foo\":\"bar\"}", JSONUtils.toJsonString(map)); Assertions.assertEquals(String.valueOf((Object) null), JSONUtils.toJsonString(null)); - Assertions.assertEquals("{\"foo\":\"bar\"}", JSONUtils.toJsonString(map, SerializationFeature.WRITE_NULL_MAP_VALUES)); + Assertions.assertEquals("{\"foo\":\"bar\"}", + JSONUtils.toJsonString(map, SerializationFeature.WRITE_NULL_MAP_VALUES)); } @Test diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/test/java/org/apache/dolphinscheduler/plugin/task/api/utils/ParameterUtilsTest.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/test/java/org/apache/dolphinscheduler/plugin/task/api/utils/ParameterUtilsTest.java index 65d8be407d2c..ea65116a5b09 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/test/java/org/apache/dolphinscheduler/plugin/task/api/utils/ParameterUtilsTest.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/test/java/org/apache/dolphinscheduler/plugin/task/api/utils/ParameterUtilsTest.java @@ -60,7 +60,8 @@ public void testConvertParameterPlaceholders() throws ParseException { // replace variable ${} form parameterMap.put("testParameter2", "${testParameter}"); - Assertions.assertEquals(parameterString, PlaceholderUtils.replacePlaceholders(parameterString, parameterMap, true)); + Assertions.assertEquals(parameterString, + PlaceholderUtils.replacePlaceholders(parameterString, parameterMap, true)); // replace time $[...] form, eg. $[yyyyMMdd] Date cronTime = new Date(); diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-blocking/pom.xml b/dolphinscheduler-task-plugin/dolphinscheduler-task-blocking/pom.xml index 6ed207715558..f57d1c0780c0 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-blocking/pom.xml +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-blocking/pom.xml @@ -15,15 +15,14 @@ ~ See the License for the specific language governing permissions and ~ limitations under the License. --> - + 4.0.0 - dolphinscheduler-task-plugin org.apache.dolphinscheduler + dolphinscheduler-task-plugin dev-SNAPSHOT - 4.0.0 dolphinscheduler-task-blocking jar diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-blocking/src/main/java/org/apache/dolphinscheduler/plugin/task/blocking/BlockingTaskChannelFactory.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-blocking/src/main/java/org/apache/dolphinscheduler/plugin/task/blocking/BlockingTaskChannelFactory.java index a8f0660cb64b..3398e1e405d4 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-blocking/src/main/java/org/apache/dolphinscheduler/plugin/task/blocking/BlockingTaskChannelFactory.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-blocking/src/main/java/org/apache/dolphinscheduler/plugin/task/blocking/BlockingTaskChannelFactory.java @@ -29,6 +29,7 @@ @AutoService(TaskChannelFactory.class) public class BlockingTaskChannelFactory implements TaskChannelFactory { + @Override public TaskChannel create() { return new BlockingTaskChannel(); diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-chunjun/pom.xml b/dolphinscheduler-task-plugin/dolphinscheduler-task-chunjun/pom.xml index 6929353a9b23..ee6db0c85fb1 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-chunjun/pom.xml +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-chunjun/pom.xml @@ -15,13 +15,14 @@ ~ See the License for the specific language governing permissions and ~ limitations under the License. --> - + + 4.0.0 - dolphinscheduler-task-plugin org.apache.dolphinscheduler + dolphinscheduler-task-plugin dev-SNAPSHOT - 4.0.0 dolphinscheduler-task-chunjun jar diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-chunjun/src/main/java/org/apache/dolphinscheduler/plugin/task/chunjun/ChunJunParameters.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-chunjun/src/main/java/org/apache/dolphinscheduler/plugin/task/chunjun/ChunJunParameters.java index 4d6cb06ab1b6..6980ca549af9 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-chunjun/src/main/java/org/apache/dolphinscheduler/plugin/task/chunjun/ChunJunParameters.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-chunjun/src/main/java/org/apache/dolphinscheduler/plugin/task/chunjun/ChunJunParameters.java @@ -113,8 +113,8 @@ public class ChunJunParameters extends AbstractParameters { public boolean checkParameters() { if (customConfig == Flag.NO.ordinal()) { return dataSource != 0 && dataTarget != 0 - && StringUtils.isNotEmpty(sql) - && StringUtils.isNotEmpty(targetTable); + && StringUtils.isNotEmpty(sql) + && StringUtils.isNotEmpty(targetTable); } else { return StringUtils.isNotEmpty(json); } diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-chunjun/src/main/java/org/apache/dolphinscheduler/plugin/task/chunjun/ChunJunTask.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-chunjun/src/main/java/org/apache/dolphinscheduler/plugin/task/chunjun/ChunJunTask.java index a690dcd3834c..f152417aa985 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-chunjun/src/main/java/org/apache/dolphinscheduler/plugin/task/chunjun/ChunJunTask.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-chunjun/src/main/java/org/apache/dolphinscheduler/plugin/task/chunjun/ChunJunTask.java @@ -17,8 +17,8 @@ package org.apache.dolphinscheduler.plugin.task.chunjun; -import org.apache.commons.io.FileUtils; -import org.apache.commons.lang.SystemUtils; +import static org.apache.dolphinscheduler.plugin.task.api.TaskConstants.EXIT_CODE_FAILURE; +import static org.apache.dolphinscheduler.plugin.task.api.TaskConstants.RWXR_XR_X; import org.apache.dolphinscheduler.plugin.task.api.AbstractTask; import org.apache.dolphinscheduler.plugin.task.api.ShellCommandExecutor; @@ -34,6 +34,9 @@ import org.apache.dolphinscheduler.spi.enums.Flag; import org.apache.dolphinscheduler.spi.utils.JSONUtils; +import org.apache.commons.io.FileUtils; +import org.apache.commons.lang.SystemUtils; + import java.io.File; import java.nio.charset.StandardCharsets; import java.nio.file.Files; @@ -48,13 +51,11 @@ import java.util.Map; import java.util.Set; -import static org.apache.dolphinscheduler.plugin.task.api.TaskConstants.EXIT_CODE_FAILURE; -import static org.apache.dolphinscheduler.plugin.task.api.TaskConstants.RWXR_XR_X; - /** * chunjun task */ public class ChunJunTask extends AbstractTask { + /** * chunjun path */ @@ -85,7 +86,7 @@ public ChunJunTask(TaskExecutionContext taskExecutionContext) { this.taskExecutionContext = taskExecutionContext; this.shellCommandExecutor = new ShellCommandExecutor(this::logHandle, - taskExecutionContext, logger); + taskExecutionContext, logger); } /** @@ -139,12 +140,11 @@ public void handle(TaskCallBack taskCallBack) throws TaskException { * @return * @throws Exception */ - private String buildChunJunJsonFile(Map paramsMap) - throws Exception { + private String buildChunJunJsonFile(Map paramsMap) throws Exception { // generate json String fileName = String.format("%s/%s_job.json", - taskExecutionContext.getExecutePath(), - taskExecutionContext.getTaskAppId()); + taskExecutionContext.getExecutePath(), + taskExecutionContext.getTaskAppId()); String json = null; @@ -173,13 +173,12 @@ private String buildChunJunJsonFile(Map paramsMap) * @return shell command file name * @throws Exception if error throws Exception */ - private String buildShellCommandFile(String jobConfigFilePath, Map paramsMap) - throws Exception { + private String buildShellCommandFile(String jobConfigFilePath, Map paramsMap) throws Exception { // generate scripts String fileName = String.format("%s/%s_node.%s", - taskExecutionContext.getExecutePath(), - taskExecutionContext.getTaskAppId(), - SystemUtils.IS_OS_WINDOWS ? "bat" : "sh"); + taskExecutionContext.getExecutePath(), + taskExecutionContext.getTaskAppId(), + SystemUtils.IS_OS_WINDOWS ? "bat" : "sh"); Path path = new File(fileName).toPath(); diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-conditions/pom.xml b/dolphinscheduler-task-plugin/dolphinscheduler-task-conditions/pom.xml index 97e6c0981528..d4266ae9f985 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-conditions/pom.xml +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-conditions/pom.xml @@ -15,15 +15,14 @@ ~ See the License for the specific language governing permissions and ~ limitations under the License. --> - + 4.0.0 - dolphinscheduler-task-plugin org.apache.dolphinscheduler + dolphinscheduler-task-plugin dev-SNAPSHOT - 4.0.0 dolphinscheduler-task-conditions jar @@ -41,4 +40,4 @@ - \ No newline at end of file + diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-conditions/src/main/java/org/apache/dolphinscheduler/plugin/task/conditions/ConditionsTaskChannelFactory.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-conditions/src/main/java/org/apache/dolphinscheduler/plugin/task/conditions/ConditionsTaskChannelFactory.java index e0a41c1d0d20..16b083d826c7 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-conditions/src/main/java/org/apache/dolphinscheduler/plugin/task/conditions/ConditionsTaskChannelFactory.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-conditions/src/main/java/org/apache/dolphinscheduler/plugin/task/conditions/ConditionsTaskChannelFactory.java @@ -34,6 +34,7 @@ @AutoService(TaskChannelFactory.class) public class ConditionsTaskChannelFactory implements TaskChannelFactory { + @Override public TaskChannel create() { return new ConditionsTaskChannel(); diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-dataquality/pom.xml b/dolphinscheduler-task-plugin/dolphinscheduler-task-dataquality/pom.xml index 4311fa80c84a..cc64b6cdcb49 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-dataquality/pom.xml +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-dataquality/pom.xml @@ -15,15 +15,14 @@ ~ See the License for the specific language governing permissions and ~ limitations under the License. --> - + 4.0.0 - dolphinscheduler-task-plugin org.apache.dolphinscheduler + dolphinscheduler-task-plugin dev-SNAPSHOT - 4.0.0 dolphinscheduler-task-dataquality jar diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-dataquality/src/main/java/org/apache/dolphinscheduler/plugin/task/dq/DataQualityTaskChannel.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-dataquality/src/main/java/org/apache/dolphinscheduler/plugin/task/dq/DataQualityTaskChannel.java index 327060584225..e6ceae86406e 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-dataquality/src/main/java/org/apache/dolphinscheduler/plugin/task/dq/DataQualityTaskChannel.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-dataquality/src/main/java/org/apache/dolphinscheduler/plugin/task/dq/DataQualityTaskChannel.java @@ -21,8 +21,8 @@ import org.apache.dolphinscheduler.plugin.task.api.TaskChannel; import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext; import org.apache.dolphinscheduler.plugin.task.api.parameters.AbstractParameters; -import org.apache.dolphinscheduler.plugin.task.api.parameters.dataquality.DataQualityParameters; import org.apache.dolphinscheduler.plugin.task.api.parameters.ParametersNode; +import org.apache.dolphinscheduler.plugin.task.api.parameters.dataquality.DataQualityParameters; import org.apache.dolphinscheduler.plugin.task.api.parameters.resource.ResourceParametersHelper; import org.apache.dolphinscheduler.spi.utils.JSONUtils; diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-dataquality/src/main/java/org/apache/dolphinscheduler/plugin/task/dq/exception/DataQualityException.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-dataquality/src/main/java/org/apache/dolphinscheduler/plugin/task/dq/exception/DataQualityException.java index 8f05e9cbc6de..8bc820f34c25 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-dataquality/src/main/java/org/apache/dolphinscheduler/plugin/task/dq/exception/DataQualityException.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-dataquality/src/main/java/org/apache/dolphinscheduler/plugin/task/dq/exception/DataQualityException.java @@ -38,7 +38,8 @@ public DataQualityException(Throwable cause) { super(cause); } - protected DataQualityException(String message, Throwable cause, boolean enableSuppression, boolean writableStackTrace) { + protected DataQualityException(String message, Throwable cause, boolean enableSuppression, + boolean writableStackTrace) { super(message, cause, enableSuppression, writableStackTrace); } } diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-dataquality/src/main/java/org/apache/dolphinscheduler/plugin/task/dq/rule/RuleManager.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-dataquality/src/main/java/org/apache/dolphinscheduler/plugin/task/dq/rule/RuleManager.java index b7a550a9de9d..1fb019d879bc 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-dataquality/src/main/java/org/apache/dolphinscheduler/plugin/task/dq/rule/RuleManager.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-dataquality/src/main/java/org/apache/dolphinscheduler/plugin/task/dq/rule/RuleManager.java @@ -67,15 +67,15 @@ public class RuleManager { + "${update_time} as update_time "; public static final String DEFAULT_COMPARISON_WRITER_SQL = - BASE_SQL + "from ${statistics_table} full join ${comparison_table}"; + BASE_SQL + "from ${statistics_table} full join ${comparison_table}"; public static final String MULTI_TABLE_COMPARISON_WRITER_SQL = - BASE_SQL + BASE_SQL + "from ( ${statistics_execute_sql} ) tmp1 " + "join ( ${comparison_execute_sql} ) tmp2"; public static final String SINGLE_TABLE_CUSTOM_SQL_WRITER_SQL = - BASE_SQL + BASE_SQL + "from ( ${statistics_table} ) tmp1 " + "join ${comparison_table}"; public static final String TASK_STATISTICS_VALUE_WRITER_SQL = @@ -91,7 +91,8 @@ public class RuleManager { + "${update_time} as update_time " + "from ${statistics_table}"; - public RuleManager(Map inputParameterValue, DataQualityTaskExecutionContext dataQualityTaskExecutionContext) { + public RuleManager(Map inputParameterValue, + DataQualityTaskExecutionContext dataQualityTaskExecutionContext) { this.inputParameterValue = inputParameterValue; this.dataQualityTaskExecutionContext = dataQualityTaskExecutionContext; } @@ -104,7 +105,8 @@ public DataQualityConfiguration generateDataQualityParameter() throws RuntimeExc Map inputParameterValueResult = RuleParserUtils.getInputParameterMapFromEntryList( - JSONUtils.toList(dataQualityTaskExecutionContext.getRuleInputEntryList(), DqRuleInputEntry.class)); + JSONUtils.toList(dataQualityTaskExecutionContext.getRuleInputEntryList(), + DqRuleInputEntry.class)); inputParameterValueResult.putAll(inputParameterValue); inputParameterValueResult.putAll(BusinessTimeUtils.getBusinessTime(CommandType.START_PROCESS, new Date())); inputParameterValueResult.putIfAbsent(COMPARISON_TYPE, NONE_COMPARISON_TYPE); @@ -131,4 +133,4 @@ public DataQualityConfiguration generateDataQualityParameter() throws RuntimeExc return ruleParser.parse(inputParameterValueResult, dataQualityTaskExecutionContext); } -} \ No newline at end of file +} diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-dataquality/src/main/java/org/apache/dolphinscheduler/plugin/task/dq/rule/parameter/BaseConfig.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-dataquality/src/main/java/org/apache/dolphinscheduler/plugin/task/dq/rule/parameter/BaseConfig.java index a775bde6d419..cbcf90846412 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-dataquality/src/main/java/org/apache/dolphinscheduler/plugin/task/dq/rule/parameter/BaseConfig.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-dataquality/src/main/java/org/apache/dolphinscheduler/plugin/task/dq/rule/parameter/BaseConfig.java @@ -30,12 +30,12 @@ public class BaseConfig { private String type; @JsonProperty("config") - private Map config; + private Map config; public BaseConfig() { } - public BaseConfig(String type, Map config) { + public BaseConfig(String type, Map config) { this.type = type; this.config = config; } diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-dataquality/src/main/java/org/apache/dolphinscheduler/plugin/task/dq/rule/parameter/DataQualityConfiguration.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-dataquality/src/main/java/org/apache/dolphinscheduler/plugin/task/dq/rule/parameter/DataQualityConfiguration.java index eff31329783b..3773932bac27 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-dataquality/src/main/java/org/apache/dolphinscheduler/plugin/task/dq/rule/parameter/DataQualityConfiguration.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-dataquality/src/main/java/org/apache/dolphinscheduler/plugin/task/dq/rule/parameter/DataQualityConfiguration.java @@ -44,7 +44,8 @@ public class DataQualityConfiguration { @JsonProperty("writers") private List writerConfigs; - public DataQualityConfiguration() {} + public DataQualityConfiguration() { + } public DataQualityConfiguration(String name, List readerConfigs, diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-dataquality/src/main/java/org/apache/dolphinscheduler/plugin/task/dq/rule/parameter/EnvConfig.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-dataquality/src/main/java/org/apache/dolphinscheduler/plugin/task/dq/rule/parameter/EnvConfig.java index b4c49b89d660..5da079e69c6f 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-dataquality/src/main/java/org/apache/dolphinscheduler/plugin/task/dq/rule/parameter/EnvConfig.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-dataquality/src/main/java/org/apache/dolphinscheduler/plugin/task/dq/rule/parameter/EnvConfig.java @@ -27,8 +27,8 @@ public class EnvConfig extends BaseConfig { public EnvConfig() { } - public EnvConfig(String type, Map config) { - super(type,config); + public EnvConfig(String type, Map config) { + super(type, config); } } diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-dataquality/src/main/java/org/apache/dolphinscheduler/plugin/task/dq/rule/parser/IRuleParser.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-dataquality/src/main/java/org/apache/dolphinscheduler/plugin/task/dq/rule/parser/IRuleParser.java index de7b6e6230a0..eb3801efc785 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-dataquality/src/main/java/org/apache/dolphinscheduler/plugin/task/dq/rule/parser/IRuleParser.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-dataquality/src/main/java/org/apache/dolphinscheduler/plugin/task/dq/rule/parser/IRuleParser.java @@ -28,6 +28,6 @@ */ public interface IRuleParser { - DataQualityConfiguration parse(Map inputParameterValue, + DataQualityConfiguration parse(Map inputParameterValue, DataQualityTaskExecutionContext context) throws DataQualityException; } diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-dataquality/src/main/java/org/apache/dolphinscheduler/plugin/task/dq/rule/parser/MappingColumn.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-dataquality/src/main/java/org/apache/dolphinscheduler/plugin/task/dq/rule/parser/MappingColumn.java index 666a16b4332e..05e09efe2762 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-dataquality/src/main/java/org/apache/dolphinscheduler/plugin/task/dq/rule/parser/MappingColumn.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-dataquality/src/main/java/org/apache/dolphinscheduler/plugin/task/dq/rule/parser/MappingColumn.java @@ -26,7 +26,8 @@ public class MappingColumn { private String operator; private String targetField; - public MappingColumn() {} + public MappingColumn() { + } public MappingColumn(String srcField, String operator, String targetField) { this.srcField = srcField; diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-dataquality/src/main/java/org/apache/dolphinscheduler/plugin/task/dq/rule/parser/MultiTableAccuracyRuleParser.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-dataquality/src/main/java/org/apache/dolphinscheduler/plugin/task/dq/rule/parser/MultiTableAccuracyRuleParser.java index 314a6272da2b..88c761690ad6 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-dataquality/src/main/java/org/apache/dolphinscheduler/plugin/task/dq/rule/parser/MultiTableAccuracyRuleParser.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-dataquality/src/main/java/org/apache/dolphinscheduler/plugin/task/dq/rule/parser/MultiTableAccuracyRuleParser.java @@ -45,28 +45,29 @@ public class MultiTableAccuracyRuleParser implements IRuleParser { public DataQualityConfiguration parse(Map inputParameterValue, DataQualityTaskExecutionContext context) throws DataQualityException { List dqRuleExecuteSqlList = - JSONUtils.toList(context.getExecuteSqlList(),DqRuleExecuteSql.class); + JSONUtils.toList(context.getExecuteSqlList(), DqRuleExecuteSql.class); DqRuleExecuteSql statisticsSql = RuleParserUtils.getExecuteSqlListByType( dqRuleExecuteSqlList, ExecuteSqlType.STATISTICS).get(0); - inputParameterValue.put(STATISTICS_TABLE,statisticsSql.getTableAlias()); + inputParameterValue.put(STATISTICS_TABLE, statisticsSql.getTableAlias()); int index = 1; List readerConfigList = - RuleParserUtils.getReaderConfigList(inputParameterValue,context); + RuleParserUtils.getReaderConfigList(inputParameterValue, context); - RuleParserUtils.addStatisticsValueTableReaderConfig(readerConfigList,context); + RuleParserUtils.addStatisticsValueTableReaderConfig(readerConfigList, context); List transformerConfigList = new ArrayList<>(); - List mappingColumnList = RuleParserUtils.getMappingColumnList(inputParameterValue.get(MAPPING_COLUMNS)); + List mappingColumnList = + RuleParserUtils.getMappingColumnList(inputParameterValue.get(MAPPING_COLUMNS)); - //get on clause - inputParameterValue.put(ON_CLAUSE, RuleParserUtils.getOnClause(mappingColumnList,inputParameterValue)); - //get where clause - inputParameterValue.put(WHERE_CLAUSE, RuleParserUtils.getWhereClause(mappingColumnList,inputParameterValue)); + // get on clause + inputParameterValue.put(ON_CLAUSE, RuleParserUtils.getOnClause(mappingColumnList, inputParameterValue)); + // get where clause + inputParameterValue.put(WHERE_CLAUSE, RuleParserUtils.getWhereClause(mappingColumnList, inputParameterValue)); index = RuleParserUtils.replaceExecuteSqlPlaceholder( dqRuleExecuteSqlList, @@ -76,11 +77,11 @@ public DataQualityConfiguration parse(Map inputParameterValue, String writerSql = RuleManager.DEFAULT_COMPARISON_WRITER_SQL; if (context.isCompareWithFixedValue()) { - writerSql = writerSql.replaceAll("full join \\$\\{comparison_table}",""); + writerSql = writerSql.replaceAll("full join \\$\\{comparison_table}", ""); } List writerConfigList = RuleParserUtils.getAllWriterConfigList(inputParameterValue, - context, index, transformerConfigList, writerSql,RuleManager.TASK_STATISTICS_VALUE_WRITER_SQL); + context, index, transformerConfigList, writerSql, RuleManager.TASK_STATISTICS_VALUE_WRITER_SQL); return new DataQualityConfiguration( context.getRuleName(), diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-dataquality/src/main/java/org/apache/dolphinscheduler/plugin/task/dq/rule/parser/MultiTableComparisonRuleParser.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-dataquality/src/main/java/org/apache/dolphinscheduler/plugin/task/dq/rule/parser/MultiTableComparisonRuleParser.java index bc04fd629eb5..b6a130366b1a 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-dataquality/src/main/java/org/apache/dolphinscheduler/plugin/task/dq/rule/parser/MultiTableComparisonRuleParser.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-dataquality/src/main/java/org/apache/dolphinscheduler/plugin/task/dq/rule/parser/MultiTableComparisonRuleParser.java @@ -39,13 +39,14 @@ public DataQualityConfiguration parse(Map inputParameterValue, DataQualityTaskExecutionContext context) throws DataQualityException { List readerConfigList = - RuleParserUtils.getReaderConfigList(inputParameterValue,context); - RuleParserUtils.addStatisticsValueTableReaderConfig(readerConfigList,context); + RuleParserUtils.getReaderConfigList(inputParameterValue, context); + RuleParserUtils.addStatisticsValueTableReaderConfig(readerConfigList, context); List transformerConfigList = new ArrayList<>(); List writerConfigList = RuleParserUtils.getWriterConfigList( - ParameterUtils.convertParameterPlaceholders(RuleManager.MULTI_TABLE_COMPARISON_WRITER_SQL,inputParameterValue), + ParameterUtils.convertParameterPlaceholders(RuleManager.MULTI_TABLE_COMPARISON_WRITER_SQL, + inputParameterValue), context); return new DataQualityConfiguration( diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-dataquality/src/main/java/org/apache/dolphinscheduler/plugin/task/dq/rule/parser/SingleTableCustomSqlRuleParser.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-dataquality/src/main/java/org/apache/dolphinscheduler/plugin/task/dq/rule/parser/SingleTableCustomSqlRuleParser.java index 09d9e63ac7c7..d4c8adc4e4d6 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-dataquality/src/main/java/org/apache/dolphinscheduler/plugin/task/dq/rule/parser/SingleTableCustomSqlRuleParser.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-dataquality/src/main/java/org/apache/dolphinscheduler/plugin/task/dq/rule/parser/SingleTableCustomSqlRuleParser.java @@ -38,31 +38,31 @@ public class SingleTableCustomSqlRuleParser implements IRuleParser { public DataQualityConfiguration parse(Map inputParameterValue, DataQualityTaskExecutionContext context) throws DataQualityException { List dqRuleExecuteSqlList = - JSONUtils.toList(context.getExecuteSqlList(),DqRuleExecuteSql.class); + JSONUtils.toList(context.getExecuteSqlList(), DqRuleExecuteSql.class); int index = 1; List readerConfigList = - RuleParserUtils.getReaderConfigList(inputParameterValue,context); - RuleParserUtils.addStatisticsValueTableReaderConfig(readerConfigList,context); + RuleParserUtils.getReaderConfigList(inputParameterValue, context); + RuleParserUtils.addStatisticsValueTableReaderConfig(readerConfigList, context); List transformerConfigList = RuleParserUtils - .getSingleTableCustomSqlTransformerConfigList(index,inputParameterValue); + .getSingleTableCustomSqlTransformerConfigList(index, inputParameterValue); - //replace the placeholder in execute sql list + // replace the placeholder in execute sql list index = RuleParserUtils.replaceExecuteSqlPlaceholder( - dqRuleExecuteSqlList, - index, - inputParameterValue, - transformerConfigList); + dqRuleExecuteSqlList, + index, + inputParameterValue, + transformerConfigList); String writerSql = RuleManager.SINGLE_TABLE_CUSTOM_SQL_WRITER_SQL; if (context.isCompareWithFixedValue()) { - writerSql = writerSql.replaceAll("join \\$\\{comparison_table}",""); + writerSql = writerSql.replaceAll("join \\$\\{comparison_table}", ""); } List writerConfigList = RuleParserUtils.getAllWriterConfigList(inputParameterValue, - context, index, transformerConfigList, writerSql,RuleManager.TASK_STATISTICS_VALUE_WRITER_SQL); + context, index, transformerConfigList, writerSql, RuleManager.TASK_STATISTICS_VALUE_WRITER_SQL); return new DataQualityConfiguration( context.getRuleName(), diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-dataquality/src/main/java/org/apache/dolphinscheduler/plugin/task/dq/rule/parser/SingleTableRuleParser.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-dataquality/src/main/java/org/apache/dolphinscheduler/plugin/task/dq/rule/parser/SingleTableRuleParser.java index 92ead2a0071f..456d9c7a379d 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-dataquality/src/main/java/org/apache/dolphinscheduler/plugin/task/dq/rule/parser/SingleTableRuleParser.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-dataquality/src/main/java/org/apache/dolphinscheduler/plugin/task/dq/rule/parser/SingleTableRuleParser.java @@ -42,35 +42,35 @@ public class SingleTableRuleParser implements IRuleParser { public DataQualityConfiguration parse(Map inputParameterValue, DataQualityTaskExecutionContext context) throws DataQualityException { List dqRuleExecuteSqlList = - JSONUtils.toList(context.getExecuteSqlList(),DqRuleExecuteSql.class); + JSONUtils.toList(context.getExecuteSqlList(), DqRuleExecuteSql.class); DqRuleExecuteSql statisticsSql = RuleParserUtils.getExecuteSqlListByType(dqRuleExecuteSqlList, ExecuteSqlType.STATISTICS).get(0); - inputParameterValue.put(STATISTICS_TABLE,statisticsSql.getTableAlias()); + inputParameterValue.put(STATISTICS_TABLE, statisticsSql.getTableAlias()); int index = 1; List readerConfigList = - RuleParserUtils.getReaderConfigList(inputParameterValue,context); - RuleParserUtils.addStatisticsValueTableReaderConfig(readerConfigList,context); + RuleParserUtils.getReaderConfigList(inputParameterValue, context); + RuleParserUtils.addStatisticsValueTableReaderConfig(readerConfigList, context); List transformerConfigList = new ArrayList<>(); - //replace the placeholder in execute sql list + // replace the placeholder in execute sql list index = RuleParserUtils.replaceExecuteSqlPlaceholder( - dqRuleExecuteSqlList, - index, - inputParameterValue, - transformerConfigList); + dqRuleExecuteSqlList, + index, + inputParameterValue, + transformerConfigList); String writerSql = RuleManager.DEFAULT_COMPARISON_WRITER_SQL; if (context.isCompareWithFixedValue()) { - writerSql = writerSql.replaceAll("full join \\$\\{comparison_table}",""); + writerSql = writerSql.replaceAll("full join \\$\\{comparison_table}", ""); } List writerConfigList = RuleParserUtils.getAllWriterConfigList(inputParameterValue, - context, index, transformerConfigList, writerSql,RuleManager.TASK_STATISTICS_VALUE_WRITER_SQL); + context, index, transformerConfigList, writerSql, RuleManager.TASK_STATISTICS_VALUE_WRITER_SQL); return new DataQualityConfiguration( context.getRuleName(), diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-dataquality/src/main/java/org/apache/dolphinscheduler/plugin/task/dq/utils/RuleParserUtils.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-dataquality/src/main/java/org/apache/dolphinscheduler/plugin/task/dq/utils/RuleParserUtils.java index f5b03b2eaf67..221a60933e38 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-dataquality/src/main/java/org/apache/dolphinscheduler/plugin/task/dq/utils/RuleParserUtils.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-dataquality/src/main/java/org/apache/dolphinscheduler/plugin/task/dq/utils/RuleParserUtils.java @@ -100,12 +100,12 @@ private RuleParserUtils() { private static final String WHERE_TARGET_FILTER = "WHERE (${target_filter})"; public static List getReaderConfigList( - Map inputParameterValue, - DataQualityTaskExecutionContext dataQualityTaskExecutionContext) throws DataQualityException { + Map inputParameterValue, + DataQualityTaskExecutionContext dataQualityTaskExecutionContext) throws DataQualityException { List readerConfigList = new ArrayList<>(); - //all the rule need the source config + // all the rule need the source config if (StringUtils.isNotEmpty(dataQualityTaskExecutionContext.getSourceConnectorType())) { BaseConnectionParam sourceDataSource = (BaseConnectionParam) DataSourceUtils.buildConnectionParams( @@ -113,7 +113,7 @@ public static List getReaderConfigList( dataQualityTaskExecutionContext.getSourceConnectionParams()); BaseConfig sourceBaseConfig = new BaseConfig(); sourceBaseConfig.setType(dataQualityTaskExecutionContext.getSourceConnectorType()); - Map config = new HashMap<>(); + Map config = new HashMap<>(); if (sourceDataSource != null) { String tableName = inputParameterValue.get(SRC_TABLE); String database = sourceDataSource.getDatabase(); @@ -140,17 +140,19 @@ public static List getReaderConfigList( dataQualityTaskExecutionContext.getTargetConnectionParams()); BaseConfig targetBaseConfig = new BaseConfig(); targetBaseConfig.setType(dataQualityTaskExecutionContext.getTargetConnectorType()); - Map config = new HashMap<>(); + Map config = new HashMap<>(); if (targetDataSource != null) { - config.put(DATABASE,targetDataSource.getDatabase()); - config.put(TABLE,inputParameterValue.get(TARGET_TABLE)); - config.put(URL,DataSourceUtils.getJdbcUrl(DbType.of(dataQualityTaskExecutionContext.getTargetType()),targetDataSource)); - config.put(USER,targetDataSource.getUser()); - config.put(PASSWORD,targetDataSource.getPassword()); - config.put(DRIVER, DataSourceUtils.getDatasourceDriver(DbType.of(dataQualityTaskExecutionContext.getTargetType()))); + config.put(DATABASE, targetDataSource.getDatabase()); + config.put(TABLE, inputParameterValue.get(TARGET_TABLE)); + config.put(URL, DataSourceUtils.getJdbcUrl(DbType.of(dataQualityTaskExecutionContext.getTargetType()), + targetDataSource)); + config.put(USER, targetDataSource.getUser()); + config.put(PASSWORD, targetDataSource.getPassword()); + config.put(DRIVER, DataSourceUtils + .getDatasourceDriver(DbType.of(dataQualityTaskExecutionContext.getTargetType()))); String outputTable = targetDataSource.getDatabase() + "_" + inputParameterValue.get(TARGET_TABLE); - config.put(OUTPUT_TABLE,outputTable); - inputParameterValue.put(TARGET_TABLE,outputTable); + config.put(OUTPUT_TABLE, outputTable); + inputParameterValue.put(TARGET_TABLE, outputTable); } targetBaseConfig.setConfig(config); @@ -161,26 +163,29 @@ public static List getReaderConfigList( } public static int replaceExecuteSqlPlaceholder(List executeSqlList, - int index, Map inputParameterValueResult, - List transformerConfigList) { - List midExecuteSqlDefinitionList - = getExecuteSqlListByType(executeSqlList, ExecuteSqlType.MIDDLE); - - List statisticsExecuteSqlDefinitionList - = getExecuteSqlListByType(executeSqlList, ExecuteSqlType.STATISTICS); - - checkAndReplace(midExecuteSqlDefinitionList,inputParameterValueResult.get(SRC_FILTER),AND_SRC_FILTER); - checkAndReplace(midExecuteSqlDefinitionList,inputParameterValueResult.get(SRC_FILTER),WHERE_SRC_FILTER); - checkAndReplace(statisticsExecuteSqlDefinitionList,inputParameterValueResult.get(SRC_FILTER),AND_SRC_FILTER); - checkAndReplace(statisticsExecuteSqlDefinitionList,inputParameterValueResult.get(SRC_FILTER),WHERE_SRC_FILTER); - - checkAndReplace(midExecuteSqlDefinitionList,inputParameterValueResult.get(TARGET_FILTER),AND_TARGET_FILTER); - checkAndReplace(midExecuteSqlDefinitionList,inputParameterValueResult.get(TARGET_FILTER),WHERE_TARGET_FILTER); - checkAndReplace(statisticsExecuteSqlDefinitionList,inputParameterValueResult.get(TARGET_FILTER),AND_TARGET_FILTER); - checkAndReplace(statisticsExecuteSqlDefinitionList,inputParameterValueResult.get(TARGET_FILTER),WHERE_TARGET_FILTER); + int index, Map inputParameterValueResult, + List transformerConfigList) { + List midExecuteSqlDefinitionList = + getExecuteSqlListByType(executeSqlList, ExecuteSqlType.MIDDLE); + + List statisticsExecuteSqlDefinitionList = + getExecuteSqlListByType(executeSqlList, ExecuteSqlType.STATISTICS); + + checkAndReplace(midExecuteSqlDefinitionList, inputParameterValueResult.get(SRC_FILTER), AND_SRC_FILTER); + checkAndReplace(midExecuteSqlDefinitionList, inputParameterValueResult.get(SRC_FILTER), WHERE_SRC_FILTER); + checkAndReplace(statisticsExecuteSqlDefinitionList, inputParameterValueResult.get(SRC_FILTER), AND_SRC_FILTER); + checkAndReplace(statisticsExecuteSqlDefinitionList, inputParameterValueResult.get(SRC_FILTER), + WHERE_SRC_FILTER); + + checkAndReplace(midExecuteSqlDefinitionList, inputParameterValueResult.get(TARGET_FILTER), AND_TARGET_FILTER); + checkAndReplace(midExecuteSqlDefinitionList, inputParameterValueResult.get(TARGET_FILTER), WHERE_TARGET_FILTER); + checkAndReplace(statisticsExecuteSqlDefinitionList, inputParameterValueResult.get(TARGET_FILTER), + AND_TARGET_FILTER); + checkAndReplace(statisticsExecuteSqlDefinitionList, inputParameterValueResult.get(TARGET_FILTER), + WHERE_TARGET_FILTER); if (CollectionUtils.isNotEmpty(midExecuteSqlDefinitionList)) { - for (DqRuleExecuteSql executeSqlDefinition:midExecuteSqlDefinitionList) { + for (DqRuleExecuteSql executeSqlDefinition : midExecuteSqlDefinitionList) { index = setTransformerConfig( index, inputParameterValueResult, @@ -190,7 +195,7 @@ public static int replaceExecuteSqlPlaceholder(List executeSql } if (CollectionUtils.isNotEmpty(statisticsExecuteSqlDefinitionList)) { - for (DqRuleExecuteSql executeSqlDefinition:statisticsExecuteSqlDefinitionList) { + for (DqRuleExecuteSql executeSqlDefinition : statisticsExecuteSqlDefinitionList) { index = setTransformerConfig( index, inputParameterValueResult, @@ -203,59 +208,61 @@ public static int replaceExecuteSqlPlaceholder(List executeSql } private static int setTransformerConfig(int index, - Map inputParameterValueResult, - List transformerConfigList, - DqRuleExecuteSql executeSqlDefinition) { - Map config = new HashMap<>(); - config.put(INDEX,index++); - config.put(SQL, ParameterUtils.convertParameterPlaceholders(executeSqlDefinition.getSql(),inputParameterValueResult)); - config.put(OUTPUT_TABLE,executeSqlDefinition.getTableAlias()); - - BaseConfig transformerConfig = new BaseConfig(SQL,config); + Map inputParameterValueResult, + List transformerConfigList, + DqRuleExecuteSql executeSqlDefinition) { + Map config = new HashMap<>(); + config.put(INDEX, index++); + config.put(SQL, + ParameterUtils.convertParameterPlaceholders(executeSqlDefinition.getSql(), inputParameterValueResult)); + config.put(OUTPUT_TABLE, executeSqlDefinition.getTableAlias()); + + BaseConfig transformerConfig = new BaseConfig(SQL, config); transformerConfigList.add(transformerConfig); return index; } public static List getSingleTableCustomSqlTransformerConfigList(int index, - Map inputParameterValueResult) { + Map inputParameterValueResult) { List list = new ArrayList<>(); - Map config = new HashMap<>(); - config.put(INDEX,index + 1); - config.put(SQL,ParameterUtils.convertParameterPlaceholders(inputParameterValueResult.get(STATISTICS_EXECUTE_SQL),inputParameterValueResult)); - config.put(OUTPUT_TABLE,inputParameterValueResult.get(SRC_TABLE)); - inputParameterValueResult.put(STATISTICS_TABLE,inputParameterValueResult.get(SRC_TABLE)); - BaseConfig transformerConfig = new BaseConfig(SQL,config); + Map config = new HashMap<>(); + config.put(INDEX, index + 1); + config.put(SQL, ParameterUtils.convertParameterPlaceholders( + inputParameterValueResult.get(STATISTICS_EXECUTE_SQL), inputParameterValueResult)); + config.put(OUTPUT_TABLE, inputParameterValueResult.get(SRC_TABLE)); + inputParameterValueResult.put(STATISTICS_TABLE, inputParameterValueResult.get(SRC_TABLE)); + BaseConfig transformerConfig = new BaseConfig(SQL, config); list.add(transformerConfig); - return list; + return list; } private static String getCoalesceString(String table, String column) { return "coalesce(" + table + "." + column + ", '')"; } - private static String getSrcColumnIsNullStr(String table,List columns) { + private static String getSrcColumnIsNullStr(String table, List columns) { String[] columnList = new String[columns.size()]; for (int i = 0; i < columns.size(); i++) { String column = columns.get(i); columnList[i] = table + "." + column + " IS NULL"; } - return String.join(AND, columnList); + return String.join(AND, columnList); } - public static Map getInputParameterMapFromEntryList(List defaultInputEntryList) { + public static Map getInputParameterMapFromEntryList(List defaultInputEntryList) { - Map defaultInputParameterValue = new HashMap<>(); - for (DqRuleInputEntry inputEntry:defaultInputEntryList) { - defaultInputParameterValue.put(inputEntry.getField(),inputEntry.getValue()); + Map defaultInputParameterValue = new HashMap<>(); + for (DqRuleInputEntry inputEntry : defaultInputEntryList) { + defaultInputParameterValue.put(inputEntry.getField(), inputEntry.getValue()); } return defaultInputParameterValue; } public static List getWriterConfigList( - String sql, - DataQualityTaskExecutionContext dataQualityTaskExecutionContext) throws DataQualityException { + String sql, + DataQualityTaskExecutionContext dataQualityTaskExecutionContext) throws DataQualityException { List writerConfigList = new ArrayList<>(); if (StringUtils.isNotEmpty(dataQualityTaskExecutionContext.getWriterConnectorType())) { @@ -266,15 +273,17 @@ public static List getWriterConfigList( BaseConfig writerConfig = new BaseConfig(); writerConfig.setType(dataQualityTaskExecutionContext.getWriterConnectorType()); - Map config = new HashMap<>(); + Map config = new HashMap<>(); if (writerDataSource != null) { - config.put(DATABASE,writerDataSource.getDatabase()); - config.put(TABLE,dataQualityTaskExecutionContext.getWriterTable()); - config.put(URL,DataSourceUtils.getJdbcUrl(DbType.of(dataQualityTaskExecutionContext.getWriterType()),writerDataSource)); - config.put(USER,writerDataSource.getUser()); - config.put(PASSWORD,writerDataSource.getPassword()); - config.put(DRIVER, DataSourceUtils.getDatasourceDriver(DbType.of(dataQualityTaskExecutionContext.getWriterType()))); - config.put(SQL,sql); + config.put(DATABASE, writerDataSource.getDatabase()); + config.put(TABLE, dataQualityTaskExecutionContext.getWriterTable()); + config.put(URL, DataSourceUtils.getJdbcUrl(DbType.of(dataQualityTaskExecutionContext.getWriterType()), + writerDataSource)); + config.put(USER, writerDataSource.getUser()); + config.put(PASSWORD, writerDataSource.getPassword()); + config.put(DRIVER, DataSourceUtils + .getDatasourceDriver(DbType.of(dataQualityTaskExecutionContext.getWriterType()))); + config.put(SQL, sql); } writerConfig.setConfig(config); writerConfigList.add(writerConfig); @@ -283,46 +292,48 @@ public static List getWriterConfigList( return writerConfigList; } - public static void addStatisticsValueTableReaderConfig (List readerConfigList, - DataQualityTaskExecutionContext dataQualityTaskExecutionContext) { + public static void addStatisticsValueTableReaderConfig(List readerConfigList, + DataQualityTaskExecutionContext dataQualityTaskExecutionContext) { if (dataQualityTaskExecutionContext.isComparisonNeedStatisticsValueTable()) { - List statisticsBaseConfigList = RuleParserUtils.getStatisticsValueConfigReaderList(dataQualityTaskExecutionContext); + List statisticsBaseConfigList = + RuleParserUtils.getStatisticsValueConfigReaderList(dataQualityTaskExecutionContext); readerConfigList.addAll(statisticsBaseConfigList); } } - public static List getStatisticsValueConfigWriterList ( - String sql, - Map inputParameterValueResult, - DataQualityTaskExecutionContext dataQualityTaskExecutionContext) throws DataQualityException { + public static List getStatisticsValueConfigWriterList( + String sql, + Map inputParameterValueResult, + DataQualityTaskExecutionContext dataQualityTaskExecutionContext) throws DataQualityException { List writerConfigList = new ArrayList<>(); if (StringUtils.isNotEmpty(dataQualityTaskExecutionContext.getStatisticsValueConnectorType())) { BaseConfig writerConfig = getStatisticsValueConfig(dataQualityTaskExecutionContext); if (writerConfig != null) { - writerConfig.getConfig().put(SQL,ParameterUtils.convertParameterPlaceholders(sql,inputParameterValueResult)); + writerConfig.getConfig().put(SQL, + ParameterUtils.convertParameterPlaceholders(sql, inputParameterValueResult)); } writerConfigList.add(writerConfig); } return writerConfigList; } - public static List getStatisticsValueConfigReaderList ( - DataQualityTaskExecutionContext dataQualityTaskExecutionContext) throws DataQualityException { + public static List getStatisticsValueConfigReaderList( + DataQualityTaskExecutionContext dataQualityTaskExecutionContext) throws DataQualityException { List readerConfigList = new ArrayList<>(); if (StringUtils.isNotEmpty(dataQualityTaskExecutionContext.getStatisticsValueConnectorType())) { BaseConfig readerConfig = getStatisticsValueConfig(dataQualityTaskExecutionContext); if (readerConfig != null) { - readerConfig.getConfig().put(OUTPUT_TABLE,dataQualityTaskExecutionContext.getStatisticsValueTable()); + readerConfig.getConfig().put(OUTPUT_TABLE, dataQualityTaskExecutionContext.getStatisticsValueTable()); } readerConfigList.add(readerConfig); } return readerConfigList; } - public static BaseConfig getStatisticsValueConfig ( - DataQualityTaskExecutionContext dataQualityTaskExecutionContext) throws DataQualityException { + public static BaseConfig getStatisticsValueConfig( + DataQualityTaskExecutionContext dataQualityTaskExecutionContext) throws DataQualityException { BaseConfig baseConfig = null; if (StringUtils.isNotEmpty(dataQualityTaskExecutionContext.getStatisticsValueConnectorType())) { BaseConnectionParam writerDataSource = @@ -332,14 +343,16 @@ public static BaseConfig getStatisticsValueConfig ( baseConfig = new BaseConfig(); baseConfig.setType(dataQualityTaskExecutionContext.getStatisticsValueConnectorType()); - Map config = new HashMap<>(); + Map config = new HashMap<>(); if (writerDataSource != null) { - config.put(DATABASE,writerDataSource.getDatabase()); - config.put(TABLE,dataQualityTaskExecutionContext.getStatisticsValueTable()); - config.put(URL,DataSourceUtils.getJdbcUrl(DbType.of(dataQualityTaskExecutionContext.getStatisticsValueType()),writerDataSource)); - config.put(USER,writerDataSource.getUser()); - config.put(PASSWORD,writerDataSource.getPassword()); - config.put(DRIVER, DataSourceUtils.getDatasourceDriver(DbType.of(dataQualityTaskExecutionContext.getWriterType()))); + config.put(DATABASE, writerDataSource.getDatabase()); + config.put(TABLE, dataQualityTaskExecutionContext.getStatisticsValueTable()); + config.put(URL, DataSourceUtils.getJdbcUrl( + DbType.of(dataQualityTaskExecutionContext.getStatisticsValueType()), writerDataSource)); + config.put(USER, writerDataSource.getUser()); + config.put(PASSWORD, writerDataSource.getPassword()); + config.put(DRIVER, DataSourceUtils + .getDatasourceDriver(DbType.of(dataQualityTaskExecutionContext.getWriterType()))); } baseConfig.setConfig(config); @@ -348,45 +361,53 @@ public static BaseConfig getStatisticsValueConfig ( return baseConfig; } - public static String getOnClause(List mappingColumnList,Map inputParameterValueResult) { - //get on clause + public static String getOnClause(List mappingColumnList, + Map inputParameterValueResult) { + // get on clause String[] columnList = new String[mappingColumnList.size()]; for (int i = 0; i < mappingColumnList.size(); i++) { MappingColumn column = mappingColumnList.get(i); - columnList[i] = getCoalesceString(inputParameterValueResult.get(SRC_TABLE),column.getSrcField()) + columnList[i] = getCoalesceString(inputParameterValueResult.get(SRC_TABLE), column.getSrcField()) + column.getOperator() - + getCoalesceString(inputParameterValueResult.get(TARGET_TABLE),column.getTargetField()); + + getCoalesceString(inputParameterValueResult.get(TARGET_TABLE), column.getTargetField()); } - return String.join(AND,columnList); + return String.join(AND, columnList); } - public static String getWhereClause(List mappingColumnList,Map inputParameterValueResult) { - String srcColumnNotNull = "( NOT (" + getSrcColumnIsNullStr(inputParameterValueResult.get(SRC_TABLE),getSrcColumnList(mappingColumnList)) + " ))"; - String targetColumnIsNull = "( " + getSrcColumnIsNullStr(inputParameterValueResult.get(TARGET_TABLE),getTargetColumnList(mappingColumnList)) + " )"; + public static String getWhereClause(List mappingColumnList, + Map inputParameterValueResult) { + String srcColumnNotNull = "( NOT (" + + getSrcColumnIsNullStr(inputParameterValueResult.get(SRC_TABLE), getSrcColumnList(mappingColumnList)) + + " ))"; + String targetColumnIsNull = "( " + getSrcColumnIsNullStr(inputParameterValueResult.get(TARGET_TABLE), + getTargetColumnList(mappingColumnList)) + " )"; return srcColumnNotNull + AND + targetColumnIsNull; } public static List getWriterConfigList( - int index, - Map inputParameterValueResult, - List transformerConfigList, - DataQualityTaskExecutionContext dataQualityTaskExecutionContext, - String writerSql) throws DataQualityException { + int index, + Map inputParameterValueResult, + List transformerConfigList, + DataQualityTaskExecutionContext dataQualityTaskExecutionContext, + String writerSql) throws DataQualityException { List comparisonExecuteSqlList = - getExecuteSqlListByType(JSONUtils.toList(dataQualityTaskExecutionContext.getExecuteSqlList(),DqRuleExecuteSql.class), ExecuteSqlType.COMPARISON); + getExecuteSqlListByType( + JSONUtils.toList(dataQualityTaskExecutionContext.getExecuteSqlList(), DqRuleExecuteSql.class), + ExecuteSqlType.COMPARISON); if (CollectionUtils.isNotEmpty(comparisonExecuteSqlList)) { DqRuleExecuteSql comparisonSql = comparisonExecuteSqlList.get(0); - inputParameterValueResult.put(COMPARISON_TABLE,comparisonSql.getTableAlias()); + inputParameterValueResult.put(COMPARISON_TABLE, comparisonSql.getTableAlias()); - checkAndReplace(comparisonExecuteSqlList,inputParameterValueResult.get(SRC_FILTER),AND_SRC_FILTER); - checkAndReplace(comparisonExecuteSqlList,inputParameterValueResult.get(SRC_FILTER),WHERE_SRC_FILTER); - checkAndReplace(comparisonExecuteSqlList,inputParameterValueResult.get(TARGET_FILTER),AND_TARGET_FILTER); - checkAndReplace(comparisonExecuteSqlList,inputParameterValueResult.get(TARGET_FILTER),WHERE_TARGET_FILTER); + checkAndReplace(comparisonExecuteSqlList, inputParameterValueResult.get(SRC_FILTER), AND_SRC_FILTER); + checkAndReplace(comparisonExecuteSqlList, inputParameterValueResult.get(SRC_FILTER), WHERE_SRC_FILTER); + checkAndReplace(comparisonExecuteSqlList, inputParameterValueResult.get(TARGET_FILTER), AND_TARGET_FILTER); + checkAndReplace(comparisonExecuteSqlList, inputParameterValueResult.get(TARGET_FILTER), + WHERE_TARGET_FILTER); - for (DqRuleExecuteSql executeSqlDefinition:comparisonExecuteSqlList) { + for (DqRuleExecuteSql executeSqlDefinition : comparisonExecuteSqlList) { index = setTransformerConfig( index, inputParameterValueResult, @@ -396,18 +417,17 @@ public static List getWriterConfigList( } return getWriterConfigList( - ParameterUtils.convertParameterPlaceholders(writerSql,inputParameterValueResult), - dataQualityTaskExecutionContext - ); + ParameterUtils.convertParameterPlaceholders(writerSql, inputParameterValueResult), + dataQualityTaskExecutionContext); } - public static List getAllWriterConfigList ( - Map inputParameterValue, - DataQualityTaskExecutionContext context, - int index, - List transformerConfigList, - String writerSql, - String statisticsValueWriterSql) { + public static List getAllWriterConfigList( + Map inputParameterValue, + DataQualityTaskExecutionContext context, + int index, + List transformerConfigList, + String writerSql, + String statisticsValueWriterSql) { List writerConfigList = RuleParserUtils.getWriterConfigList( index, @@ -422,7 +442,7 @@ public static List getAllWriterConfigList ( inputParameterValue, context)); - BaseConfig errorOutputWriter = RuleParserUtils.getErrorOutputWriter(inputParameterValue,context); + BaseConfig errorOutputWriter = RuleParserUtils.getErrorOutputWriter(inputParameterValue, context); if (errorOutputWriter != null) { writerConfigList.add(errorOutputWriter); } @@ -430,8 +450,9 @@ public static List getAllWriterConfigList ( return writerConfigList; } - public static List getExecuteSqlListByType ( - List allExecuteSqlList, ExecuteSqlType executeSqlType) { + public static List getExecuteSqlListByType( + List allExecuteSqlList, + ExecuteSqlType executeSqlType) { if (CollectionUtils.isEmpty(allExecuteSqlList)) { return allExecuteSqlList; } @@ -444,9 +465,9 @@ public static List getExecuteSqlListByType ( private static void checkAndReplace(List list, String checkValue, String replaceSrc) { if (StringUtils.isEmpty(checkValue) && CollectionUtils.isNotEmpty(list)) { - for (DqRuleExecuteSql executeSqlDefinition:list) { + for (DqRuleExecuteSql executeSqlDefinition : list) { String sql = executeSqlDefinition.getSql(); - sql = sql.replace(replaceSrc,""); + sql = sql.replace(replaceSrc, ""); executeSqlDefinition.setSql(sql); } } @@ -457,9 +478,9 @@ public static List getMappingColumnList(String mappingColumns) { List list = new ArrayList<>(); mappingColumnList.forEach(item -> { MappingColumn column = new MappingColumn( - String.valueOf(item.get(SRC_FIELD)).replace("\"",""), - String.valueOf(item.get(OPERATOR)).replace("\""," "), - String.valueOf(item.get(TARGET_FIELD)).replace("\"","")); + String.valueOf(item.get(SRC_FIELD)).replace("\"", ""), + String.valueOf(item.get(OPERATOR)).replace("\"", " "), + String.valueOf(item.get(TARGET_FIELD)).replace("\"", "")); list.add(column); }); @@ -468,18 +489,14 @@ public static List getMappingColumnList(String mappingColumns) { public static List getSrcColumnList(List mappingColumns) { List list = new ArrayList<>(); - mappingColumns.forEach(item -> - list.add(item.getSrcField()) - ); + mappingColumns.forEach(item -> list.add(item.getSrcField())); return list; } public static List getTargetColumnList(List mappingColumns) { List list = new ArrayList<>(); - mappingColumns.forEach(item -> - list.add(item.getTargetField()) - ); + mappingColumns.forEach(item -> list.add(item.getTargetField())); return list; } @@ -487,13 +504,13 @@ public static List getTargetColumnList(List mappingColumn public static EnvConfig getEnvConfig() { EnvConfig envConfig = new EnvConfig(); envConfig.setType(BATCH); - return envConfig; + return envConfig; } public static BaseConfig getErrorOutputWriter(Map inputParameterValueResult, - DataQualityTaskExecutionContext dataQualityTaskExecutionContext) { - List dqRuleExecuteSqlList = JSONUtils.toList(dataQualityTaskExecutionContext.getExecuteSqlList() - ,DqRuleExecuteSql.class); + DataQualityTaskExecutionContext dataQualityTaskExecutionContext) { + List dqRuleExecuteSqlList = + JSONUtils.toList(dataQualityTaskExecutionContext.getExecuteSqlList(), DqRuleExecuteSql.class); DqRuleExecuteSql errorOutputSql = null; if (CollectionUtils.isEmpty(dqRuleExecuteSqlList)) { @@ -511,9 +528,9 @@ public static BaseConfig getErrorOutputWriter(Map inputParameter if (StringUtils.isNotEmpty(inputParameterValueResult.get(ERROR_OUTPUT_PATH)) && errorOutputSql != null) { baseConfig = new BaseConfig(); - Map config = new HashMap<>(); - config.put(PATH,inputParameterValueResult.get(ERROR_OUTPUT_PATH)); - config.put(INPUT_TABLE,errorOutputSql.getTableAlias()); + Map config = new HashMap<>(); + config.put(PATH, inputParameterValueResult.get(ERROR_OUTPUT_PATH)); + config.put(INPUT_TABLE, errorOutputSql.getTableAlias()); baseConfig.setConfig(config); baseConfig.setType(HDFS_FILE); } @@ -532,7 +549,7 @@ public static String generateUniqueCode(Map inputParameterValue) return "-1"; } - Map newInputParameterValue = new HashMap<>(inputParameterValue); + Map newInputParameterValue = new HashMap<>(inputParameterValue); newInputParameterValue.remove(RULE_TYPE); newInputParameterValue.remove(RULE_NAME); @@ -559,6 +576,6 @@ public static String generateUniqueCode(Map inputParameterValue) sb.append(value); } - return Md5Utils.getMd5(sb.toString(),true); + return Md5Utils.getMd5(sb.toString(), true); } } diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-dataquality/src/main/java/org/apache/dolphinscheduler/plugin/task/dq/utils/SparkArgsUtils.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-dataquality/src/main/java/org/apache/dolphinscheduler/plugin/task/dq/utils/SparkArgsUtils.java index 4b9260326eb7..4bd824226b8f 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-dataquality/src/main/java/org/apache/dolphinscheduler/plugin/task/dq/utils/SparkArgsUtils.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-dataquality/src/main/java/org/apache/dolphinscheduler/plugin/task/dq/utils/SparkArgsUtils.java @@ -17,10 +17,10 @@ package org.apache.dolphinscheduler.plugin.task.dq.utils; +import org.apache.dolphinscheduler.plugin.task.api.model.ResourceInfo; import org.apache.dolphinscheduler.plugin.task.api.parameters.dataquality.spark.ProgramType; import org.apache.dolphinscheduler.plugin.task.api.parameters.dataquality.spark.SparkConstants; import org.apache.dolphinscheduler.plugin.task.api.parameters.dataquality.spark.SparkParameters; -import org.apache.dolphinscheduler.plugin.task.api.model.ResourceInfo; import org.apache.dolphinscheduler.plugin.task.api.utils.ArgsUtils; import org.apache.dolphinscheduler.spi.utils.StringUtils; @@ -103,7 +103,8 @@ public static List buildArgs(SparkParameters param) { } String others = param.getOthers(); - if (!SPARK_LOCAL.equals(deployMode) && (StringUtils.isEmpty(others) || !others.contains(SparkConstants.SPARK_QUEUE))) { + if (!SPARK_LOCAL.equals(deployMode) + && (StringUtils.isEmpty(others) || !others.contains(SparkConstants.SPARK_QUEUE))) { String queue = param.getQueue(); if (StringUtils.isNotEmpty(queue)) { args.add(SparkConstants.SPARK_QUEUE); diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-datasync/pom.xml b/dolphinscheduler-task-plugin/dolphinscheduler-task-datasync/pom.xml index 3aa7bcacb13f..b3e1edb79ff4 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-datasync/pom.xml +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-datasync/pom.xml @@ -15,15 +15,14 @@ ~ See the License for the specific language governing permissions and ~ limitations under the License. --> - + 4.0.0 - dolphinscheduler-task-plugin org.apache.dolphinscheduler + dolphinscheduler-task-plugin dev-SNAPSHOT - 4.0.0 dolphinscheduler-task-datasync jar @@ -51,4 +50,4 @@ - \ No newline at end of file + diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-datasync/src/main/java/org/apache/dolphinscheduler/plugin/task/datasync/DatasyncParameters.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-datasync/src/main/java/org/apache/dolphinscheduler/plugin/task/datasync/DatasyncParameters.java index 21e271ad7d1b..261c4a703b65 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-datasync/src/main/java/org/apache/dolphinscheduler/plugin/task/datasync/DatasyncParameters.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-datasync/src/main/java/org/apache/dolphinscheduler/plugin/task/datasync/DatasyncParameters.java @@ -17,15 +17,17 @@ package org.apache.dolphinscheduler.plugin.task.datasync; -import com.fasterxml.jackson.annotation.JsonProperty; +import org.apache.dolphinscheduler.plugin.task.api.parameters.AbstractParameters; +import org.apache.dolphinscheduler.spi.utils.StringUtils; + +import java.util.List; + import lombok.Getter; import lombok.NoArgsConstructor; import lombok.Setter; import lombok.ToString; -import org.apache.dolphinscheduler.plugin.task.api.parameters.AbstractParameters; -import org.apache.dolphinscheduler.spi.utils.StringUtils; -import java.util.List; +import com.fasterxml.jackson.annotation.JsonProperty; @Setter @Getter @@ -56,7 +58,8 @@ public boolean checkParameters() { if (jsonFormat) { return StringUtils.isNotEmpty(json); } else { - return StringUtils.isNotEmpty(destinationLocationArn) && StringUtils.isNotEmpty(sourceLocationArn) && StringUtils.isNotEmpty(name); + return StringUtils.isNotEmpty(destinationLocationArn) && StringUtils.isNotEmpty(sourceLocationArn) + && StringUtils.isNotEmpty(name); } } @@ -64,7 +67,8 @@ public boolean checkParameters() { @Getter @NoArgsConstructor @ToString - public class Options{ + public class Options { + private String verifyMode; private String overwriteMode; private String atime; @@ -85,14 +89,16 @@ public class Options{ @Getter @NoArgsConstructor @ToString - public static class TaskSchedule{ + public static class TaskSchedule { + private String scheduleExpression; } @Setter @Getter @NoArgsConstructor @ToString - public static class FilterRule{ + public static class FilterRule { + private String filterType; private String value; } @@ -100,8 +106,9 @@ public static class FilterRule{ @Getter @NoArgsConstructor @ToString - public static class TagListEntry{ + public static class TagListEntry { + private String key; private String value; } -} \ No newline at end of file +} diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-datasync/src/main/java/org/apache/dolphinscheduler/plugin/task/datasync/DatasyncTaskChannel.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-datasync/src/main/java/org/apache/dolphinscheduler/plugin/task/datasync/DatasyncTaskChannel.java index 555eda934e7c..3890f6d64148 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-datasync/src/main/java/org/apache/dolphinscheduler/plugin/task/datasync/DatasyncTaskChannel.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-datasync/src/main/java/org/apache/dolphinscheduler/plugin/task/datasync/DatasyncTaskChannel.java @@ -45,4 +45,4 @@ public AbstractParameters parseParameters(ParametersNode parametersNode) { public ResourceParametersHelper getResources(String parameters) { return null; } -} \ No newline at end of file +} diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-datasync/src/main/java/org/apache/dolphinscheduler/plugin/task/datasync/DatasyncTaskChannelFactory.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-datasync/src/main/java/org/apache/dolphinscheduler/plugin/task/datasync/DatasyncTaskChannelFactory.java index a14b24f13f61..5fac501bea71 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-datasync/src/main/java/org/apache/dolphinscheduler/plugin/task/datasync/DatasyncTaskChannelFactory.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-datasync/src/main/java/org/apache/dolphinscheduler/plugin/task/datasync/DatasyncTaskChannelFactory.java @@ -17,7 +17,6 @@ package org.apache.dolphinscheduler.plugin.task.datasync; -import com.google.auto.service.AutoService; import org.apache.dolphinscheduler.plugin.task.api.TaskChannel; import org.apache.dolphinscheduler.plugin.task.api.TaskChannelFactory; import org.apache.dolphinscheduler.spi.params.base.PluginParams; @@ -25,6 +24,8 @@ import java.util.Collections; import java.util.List; +import com.google.auto.service.AutoService; + @AutoService(TaskChannelFactory.class) public class DatasyncTaskChannelFactory implements TaskChannelFactory { @@ -42,4 +43,4 @@ public String getName() { public List getParams() { return Collections.emptyList(); } -} \ No newline at end of file +} diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-datax/pom.xml b/dolphinscheduler-task-plugin/dolphinscheduler-task-datax/pom.xml index 02b4428e8607..9cae62ad94b4 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-datax/pom.xml +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-datax/pom.xml @@ -15,15 +15,14 @@ ~ See the License for the specific language governing permissions and ~ limitations under the License. --> - + 4.0.0 - dolphinscheduler-task-plugin org.apache.dolphinscheduler + dolphinscheduler-task-plugin dev-SNAPSHOT - 4.0.0 dolphinscheduler-task-datax jar diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-datax/src/main/java/org/apache/dolphinscheduler/plugin/task/datax/DataxParameters.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-datax/src/main/java/org/apache/dolphinscheduler/plugin/task/datax/DataxParameters.java index 391e94658edb..e90e5a86be61 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-datax/src/main/java/org/apache/dolphinscheduler/plugin/task/datax/DataxParameters.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-datax/src/main/java/org/apache/dolphinscheduler/plugin/task/datax/DataxParameters.java @@ -26,7 +26,6 @@ import org.apache.dolphinscheduler.spi.utils.JSONUtils; import org.apache.dolphinscheduler.spi.utils.StringUtils; -import java.util.ArrayList; import java.util.List; import java.util.Objects; @@ -288,8 +287,10 @@ public DataxTaskExecutionContext generateExtendedContext(ResourceParametersHelpe return dataxTaskExecutionContext; } - DataSourceParameters dbSource = (DataSourceParameters) parametersHelper.getResourceParameters(ResourceType.DATASOURCE, dataSource); - DataSourceParameters dbTarget = (DataSourceParameters) parametersHelper.getResourceParameters(ResourceType.DATASOURCE, dataTarget); + DataSourceParameters dbSource = + (DataSourceParameters) parametersHelper.getResourceParameters(ResourceType.DATASOURCE, dataSource); + DataSourceParameters dbTarget = + (DataSourceParameters) parametersHelper.getResourceParameters(ResourceType.DATASOURCE, dataTarget); if (Objects.nonNull(dbSource)) { dataxTaskExecutionContext.setDataSourceId(dataSource); diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-datax/src/main/java/org/apache/dolphinscheduler/plugin/task/datax/DataxTask.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-datax/src/main/java/org/apache/dolphinscheduler/plugin/task/datax/DataxTask.java index 48a7f7f7439c..7df406f6a253 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-datax/src/main/java/org/apache/dolphinscheduler/plugin/task/datax/DataxTask.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-datax/src/main/java/org/apache/dolphinscheduler/plugin/task/datax/DataxTask.java @@ -17,26 +17,15 @@ package org.apache.dolphinscheduler.plugin.task.datax; -import com.alibaba.druid.sql.ast.SQLStatement; -import com.alibaba.druid.sql.ast.expr.SQLIdentifierExpr; -import com.alibaba.druid.sql.ast.expr.SQLPropertyExpr; -import com.alibaba.druid.sql.ast.statement.SQLSelect; -import com.alibaba.druid.sql.ast.statement.SQLSelectItem; -import com.alibaba.druid.sql.ast.statement.SQLSelectQueryBlock; -import com.alibaba.druid.sql.ast.statement.SQLSelectStatement; -import com.alibaba.druid.sql.ast.statement.SQLUnionQuery; -import com.alibaba.druid.sql.parser.SQLStatementParser; -import com.fasterxml.jackson.databind.node.ArrayNode; -import com.fasterxml.jackson.databind.node.ObjectNode; -import org.apache.commons.collections4.CollectionUtils; -import org.apache.commons.io.FileUtils; -import org.apache.commons.lang3.SystemUtils; +import static org.apache.dolphinscheduler.plugin.datasource.api.utils.PasswordUtils.decodePassword; +import static org.apache.dolphinscheduler.plugin.task.api.TaskConstants.EXIT_CODE_FAILURE; +import static org.apache.dolphinscheduler.plugin.task.api.TaskConstants.RWXR_XR_X; + import org.apache.dolphinscheduler.plugin.datasource.api.plugin.DataSourceClientProvider; import org.apache.dolphinscheduler.plugin.datasource.api.utils.DataSourceUtils; import org.apache.dolphinscheduler.plugin.task.api.AbstractTask; import org.apache.dolphinscheduler.plugin.task.api.ShellCommandExecutor; import org.apache.dolphinscheduler.plugin.task.api.TaskCallBack; -import org.apache.dolphinscheduler.plugin.task.api.TaskConstants; import org.apache.dolphinscheduler.plugin.task.api.TaskException; import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext; import org.apache.dolphinscheduler.plugin.task.api.model.Property; @@ -50,6 +39,10 @@ import org.apache.dolphinscheduler.spi.utils.JSONUtils; import org.apache.dolphinscheduler.spi.utils.StringUtils; +import org.apache.commons.collections4.CollectionUtils; +import org.apache.commons.io.FileUtils; +import org.apache.commons.lang3.SystemUtils; + import java.io.File; import java.nio.charset.StandardCharsets; import java.nio.file.Files; @@ -72,11 +65,20 @@ import java.util.regex.Matcher; import java.util.regex.Pattern; -import static org.apache.dolphinscheduler.plugin.datasource.api.utils.PasswordUtils.decodePassword; -import static org.apache.dolphinscheduler.plugin.task.api.TaskConstants.EXIT_CODE_FAILURE; -import static org.apache.dolphinscheduler.plugin.task.api.TaskConstants.RWXR_XR_X; +import com.alibaba.druid.sql.ast.SQLStatement; +import com.alibaba.druid.sql.ast.expr.SQLIdentifierExpr; +import com.alibaba.druid.sql.ast.expr.SQLPropertyExpr; +import com.alibaba.druid.sql.ast.statement.SQLSelect; +import com.alibaba.druid.sql.ast.statement.SQLSelectItem; +import com.alibaba.druid.sql.ast.statement.SQLSelectQueryBlock; +import com.alibaba.druid.sql.ast.statement.SQLSelectStatement; +import com.alibaba.druid.sql.ast.statement.SQLUnionQuery; +import com.alibaba.druid.sql.parser.SQLStatementParser; +import com.fasterxml.jackson.databind.node.ArrayNode; +import com.fasterxml.jackson.databind.node.ObjectNode; public class DataxTask extends AbstractTask { + /** * jvm parameters */ @@ -139,7 +141,8 @@ public void init() { throw new RuntimeException("datax task params is not valid"); } - dataxTaskExecutionContext = dataXParameters.generateExtendedContext(taskExecutionContext.getResourceParametersHelper()); + dataxTaskExecutionContext = + dataXParameters.generateExtendedContext(taskExecutionContext.getResourceParametersHelper()); } /** @@ -193,8 +196,7 @@ public void cancel() throws TaskException { * @return datax json file name * @throws Exception if error throws Exception */ - private String buildDataxJsonFile(Map paramsMap) - throws Exception { + private String buildDataxJsonFile(Map paramsMap) throws Exception { // generate json String fileName = String.format("%s/%s_job.json", taskExecutionContext.getExecutePath(), @@ -271,7 +273,8 @@ private List buildDataxJobContentJson() { ArrayNode tableArr = writerConn.putArray("table"); tableArr.add(dataXParameters.getTargetTable()); - writerConn.put("jdbcUrl", DataSourceUtils.getJdbcUrl(DbType.valueOf(dataXParameters.getDtType()), dataTargetCfg)); + writerConn.put("jdbcUrl", + DataSourceUtils.getJdbcUrl(DbType.valueOf(dataXParameters.getDtType()), dataTargetCfg)); writerConnArr.add(writerConn); ObjectNode writerParam = JSONUtils.createObjectNode(); @@ -377,8 +380,7 @@ private ObjectNode buildDataxCoreJson() { * @return shell command file name * @throws Exception if error throws Exception */ - private String buildShellCommandFile(String jobConfigFilePath, Map paramsMap) - throws Exception { + private String buildShellCommandFile(String jobConfigFilePath, Map paramsMap) throws Exception { // generate scripts String fileName = String.format("%s/%s_node.%s", taskExecutionContext.getExecutePath(), @@ -466,7 +468,8 @@ public String loadJvmEnv(DataxParameters dataXParameters) { * @param sql sql for data synchronization * @return Keyword converted column names */ - private String[] parsingSqlColumnNames(DbType sourceType, DbType targetType, BaseConnectionParam dataSourceCfg, String sql) { + private String[] parsingSqlColumnNames(DbType sourceType, DbType targetType, BaseConnectionParam dataSourceCfg, + String sql) { String[] columnNames = tryGrammaticalAnalysisSqlColumnNames(sourceType, sql); if (columnNames == null || columnNames.length == 0) { @@ -563,7 +566,8 @@ public String[] tryExecuteSqlResolveColumnNames(DbType sourceType, BaseConnectio sql = sql.replace(";", ""); try ( - Connection connection = DataSourceClientProvider.getInstance().getConnection(sourceType, baseDataSource); + Connection connection = + DataSourceClientProvider.getInstance().getConnection(sourceType, baseDataSource); PreparedStatement stmt = connection.prepareStatement(sql); ResultSet resultSet = stmt.executeQuery()) { @@ -571,7 +575,7 @@ public String[] tryExecuteSqlResolveColumnNames(DbType sourceType, BaseConnectio int num = md.getColumnCount(); columnNames = new String[num]; for (int i = 1; i <= num; i++) { - columnNames[i - 1] = md.getColumnName(i).replace("t.",""); + columnNames[i - 1] = md.getColumnName(i).replace("t.", ""); } } catch (SQLException | ExecutionException e) { logger.error(e.getMessage(), e); diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-datax/src/main/java/org/apache/dolphinscheduler/plugin/task/datax/DataxUtils.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-datax/src/main/java/org/apache/dolphinscheduler/plugin/task/datax/DataxUtils.java index d2c4b1d67ed8..9ecb0bbb864e 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-datax/src/main/java/org/apache/dolphinscheduler/plugin/task/datax/DataxUtils.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-datax/src/main/java/org/apache/dolphinscheduler/plugin/task/datax/DataxUtils.java @@ -17,7 +17,6 @@ package org.apache.dolphinscheduler.plugin.task.datax; -import com.alibaba.druid.sql.dialect.presto.parser.PrestoStatementParser; import org.apache.dolphinscheduler.spi.enums.DbType; import com.alibaba.druid.sql.dialect.clickhouse.parser.ClickhouseStatementParser; @@ -25,6 +24,7 @@ import com.alibaba.druid.sql.dialect.mysql.parser.MySqlStatementParser; import com.alibaba.druid.sql.dialect.oracle.parser.OracleStatementParser; import com.alibaba.druid.sql.dialect.postgresql.parser.PGSQLStatementParser; +import com.alibaba.druid.sql.dialect.presto.parser.PrestoStatementParser; import com.alibaba.druid.sql.dialect.sqlserver.parser.SQLServerStatementParser; import com.alibaba.druid.sql.parser.SQLStatementParser; diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-datax/src/test/java/org/apache/dolphinscheduler/plugin/task/datax/DataxTaskTest.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-datax/src/test/java/org/apache/dolphinscheduler/plugin/task/datax/DataxTaskTest.java index 636ba839c06e..77b89e65ab83 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-datax/src/test/java/org/apache/dolphinscheduler/plugin/task/datax/DataxTaskTest.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-datax/src/test/java/org/apache/dolphinscheduler/plugin/task/datax/DataxTaskTest.java @@ -17,7 +17,12 @@ package org.apache.dolphinscheduler.plugin.task.datax; -import org.apache.commons.lang3.SystemUtils; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.mockStatic; +import static org.mockito.Mockito.when; + import org.apache.dolphinscheduler.common.utils.FileUtils; import org.apache.dolphinscheduler.plugin.datasource.api.plugin.DataSourceClientProvider; import org.apache.dolphinscheduler.plugin.task.api.ShellCommandExecutor; @@ -33,13 +38,8 @@ import org.apache.dolphinscheduler.spi.datasource.BaseConnectionParam; import org.apache.dolphinscheduler.spi.enums.DbType; import org.apache.dolphinscheduler.spi.utils.JSONUtils; -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.extension.ExtendWith; -import org.mockito.MockedStatic; -import org.mockito.Mockito; -import org.mockito.junit.jupiter.MockitoExtension; + +import org.apache.commons.lang3.SystemUtils; import java.io.File; import java.io.IOException; @@ -54,19 +54,21 @@ import java.util.HashMap; import java.util.Map; -import static org.mockito.ArgumentMatchers.eq; -import static org.mockito.ArgumentMatchers.anyString; -import static org.mockito.Mockito.mockStatic; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; - +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.MockedStatic; +import org.mockito.Mockito; +import org.mockito.junit.jupiter.MockitoExtension; @ExtendWith(MockitoExtension.class) public class DataxTaskTest { private DataxTask dataxTask; - private final TaskCallBack taskCallBack = (taskInstanceId, appIds) -> {}; + private final TaskCallBack taskCallBack = (taskInstanceId, appIds) -> { + }; @BeforeEach public void before() throws Exception { @@ -109,9 +111,8 @@ public void testHandleNullParamsMap() throws Exception { boolean delete = jsonFile.delete(); Assertions.assertTrue(delete); - File shellCommandFile = SystemUtils.IS_OS_WINDOWS ? - new File("/tmp/execution/app-id_node.bat") : - new File("/tmp/execution/app-id_node.sh"); + File shellCommandFile = SystemUtils.IS_OS_WINDOWS ? new File("/tmp/execution/app-id_node.bat") + : new File("/tmp/execution/app-id_node.sh"); InputStream shellCommandInputStream = Files.newInputStream(shellCommandFile.toPath()); String shellCommandStr = FileUtils.readFile2Str(shellCommandInputStream); Assertions.assertEquals(shellCommandStr, "python2.7 ${DATAX_HOME}/bin/datax.py --jvm=\"-Xms1G -Xmx1G\" " + @@ -151,9 +152,8 @@ public void testHandleParamsMap() throws Exception { boolean delete = jsonFile.delete(); Assertions.assertTrue(delete); - File shellCommandFile = SystemUtils.IS_OS_WINDOWS ? - new File("/tmp/execution/app-id_node.bat") : - new File("/tmp/execution/app-id_node.sh"); + File shellCommandFile = SystemUtils.IS_OS_WINDOWS ? new File("/tmp/execution/app-id_node.bat") + : new File("/tmp/execution/app-id_node.sh"); InputStream shellCommandInputStream = Files.newInputStream(shellCommandFile.toPath()); String shellCommandStr = FileUtils.readFile2Str(shellCommandInputStream); Assertions.assertEquals(shellCommandStr, "python2.7 ${DATAX_HOME}/bin/datax.py --jvm=\"-Xms1G -Xmx1G\" " + @@ -222,12 +222,12 @@ public void testTryExecuteSqlResolveColumnNames() throws Exception { when(resultSet.getMetaData()).thenReturn(md); when(stmt.executeQuery()).thenReturn(resultSet); - String[] rows = this.dataxTask.tryExecuteSqlResolveColumnNames(DbType.MYSQL,baseConnectionParam, ""); + String[] rows = this.dataxTask.tryExecuteSqlResolveColumnNames(DbType.MYSQL, baseConnectionParam, ""); Assertions.assertEquals(rows.length, 1); Assertions.assertEquals(rows[0], "something"); when(connection.prepareStatement(anyString())).thenThrow(new SQLException("Connection failed")); - String[] nullRows = this.dataxTask.tryExecuteSqlResolveColumnNames(DbType.MYSQL,baseConnectionParam, ""); + String[] nullRows = this.dataxTask.tryExecuteSqlResolveColumnNames(DbType.MYSQL, baseConnectionParam, ""); Assertions.assertNull(nullRows); } } @@ -324,4 +324,4 @@ private String getJsonString() { " }\n" + "}"; } -} \ No newline at end of file +} diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-dependent/pom.xml b/dolphinscheduler-task-plugin/dolphinscheduler-task-dependent/pom.xml index fc7a4eb28618..6fe7635444a7 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-dependent/pom.xml +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-dependent/pom.xml @@ -15,15 +15,14 @@ ~ See the License for the specific language governing permissions and ~ limitations under the License. --> - + 4.0.0 - dolphinscheduler-task-plugin org.apache.dolphinscheduler + dolphinscheduler-task-plugin dev-SNAPSHOT - 4.0.0 dolphinscheduler-task-dependent jar @@ -41,4 +40,4 @@ - \ No newline at end of file + diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-dependent/src/main/java/org/apache/dolphinscheduler/plugin/task/dependent/DependentTaskChannel.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-dependent/src/main/java/org/apache/dolphinscheduler/plugin/task/dependent/DependentTaskChannel.java index 1d24e8527a0f..1d7e62422ab4 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-dependent/src/main/java/org/apache/dolphinscheduler/plugin/task/dependent/DependentTaskChannel.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-dependent/src/main/java/org/apache/dolphinscheduler/plugin/task/dependent/DependentTaskChannel.java @@ -42,7 +42,8 @@ public AbstractTask createTask(TaskExecutionContext taskRequest) { @Override public AbstractParameters parseParameters(ParametersNode parametersNode) { return JSONUtils.parseObject(StringUtils.isEmpty(parametersNode.getDependence()) - ? parametersNode.getTaskParams() : parametersNode.getDependence(), DependentParameters.class); + ? parametersNode.getTaskParams() + : parametersNode.getDependence(), DependentParameters.class); } @Override diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-dependent/src/main/java/org/apache/dolphinscheduler/plugin/task/dependent/DependentTaskChannelFactory.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-dependent/src/main/java/org/apache/dolphinscheduler/plugin/task/dependent/DependentTaskChannelFactory.java index ad976d5857f0..4d323f338871 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-dependent/src/main/java/org/apache/dolphinscheduler/plugin/task/dependent/DependentTaskChannelFactory.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-dependent/src/main/java/org/apache/dolphinscheduler/plugin/task/dependent/DependentTaskChannelFactory.java @@ -29,6 +29,7 @@ @AutoService(TaskChannelFactory.class) public class DependentTaskChannelFactory implements TaskChannelFactory { + @Override public TaskChannel create() { return new DependentTaskChannel(); diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-dinky/pom.xml b/dolphinscheduler-task-plugin/dolphinscheduler-task-dinky/pom.xml index ec3259c0bcdf..647fbfab5be8 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-dinky/pom.xml +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-dinky/pom.xml @@ -15,18 +15,17 @@ ~ See the License for the specific language governing permissions and ~ limitations under the License. --> - + 4.0.0 - dolphinscheduler-task-plugin org.apache.dolphinscheduler + dolphinscheduler-task-plugin dev-SNAPSHOT - 4.0.0 - jar dolphinscheduler-task-dinky + jar org.apache.dolphinscheduler @@ -47,4 +46,4 @@ httpcore - \ No newline at end of file + diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-dinky/src/main/java/org/apache/dolphinscheduler/plugin/task/dinky/DinkyParameters.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-dinky/src/main/java/org/apache/dolphinscheduler/plugin/task/dinky/DinkyParameters.java index 2b0d6c1d3271..2349a39b5a65 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-dinky/src/main/java/org/apache/dolphinscheduler/plugin/task/dinky/DinkyParameters.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-dinky/src/main/java/org/apache/dolphinscheduler/plugin/task/dinky/DinkyParameters.java @@ -72,9 +72,9 @@ public void setOnline(boolean online) { @Override public String toString() { return "DinkyParameters{" + - "address='" + address + '\'' + - ", taskId='" + taskId + '\'' + - ", online='" + online + '\'' + - '}'; + "address='" + address + '\'' + + ", taskId='" + taskId + '\'' + + ", online='" + online + '\'' + + '}'; } } diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-dinky/src/main/java/org/apache/dolphinscheduler/plugin/task/dinky/DinkyTaskChannelFactory.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-dinky/src/main/java/org/apache/dolphinscheduler/plugin/task/dinky/DinkyTaskChannelFactory.java index 6c80a9769d14..ddaa53d233a2 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-dinky/src/main/java/org/apache/dolphinscheduler/plugin/task/dinky/DinkyTaskChannelFactory.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-dinky/src/main/java/org/apache/dolphinscheduler/plugin/task/dinky/DinkyTaskChannelFactory.java @@ -17,8 +17,6 @@ package org.apache.dolphinscheduler.plugin.task.dinky; -import com.google.auto.service.AutoService; - import org.apache.dolphinscheduler.plugin.task.api.TaskChannel; import org.apache.dolphinscheduler.plugin.task.api.TaskChannelFactory; import org.apache.dolphinscheduler.spi.params.base.PluginParams; @@ -26,8 +24,11 @@ import java.util.ArrayList; import java.util.List; +import com.google.auto.service.AutoService; + @AutoService(TaskChannelFactory.class) public class DinkyTaskChannelFactory implements TaskChannelFactory { + @Override public String getName() { return "DINKY"; diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-dinky/src/main/java/org/apache/dolphinscheduler/plugin/task/dinky/DinkyTaskConstants.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-dinky/src/main/java/org/apache/dolphinscheduler/plugin/task/dinky/DinkyTaskConstants.java index 847d4e3a31ac..8b12b8c259be 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-dinky/src/main/java/org/apache/dolphinscheduler/plugin/task/dinky/DinkyTaskConstants.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-dinky/src/main/java/org/apache/dolphinscheduler/plugin/task/dinky/DinkyTaskConstants.java @@ -25,14 +25,15 @@ public class DinkyTaskConstants { private DinkyTaskConstants() { throw new IllegalStateException("Utility class"); } - + private static final String API_ROUTE = "/openapi/"; public static final String SUBMIT_TASK = API_ROUTE + "submitTask"; public static final String ONLINE_TASK = API_ROUTE + "onLineTask"; public static final String SAVEPOINT_TASK = API_ROUTE + "savepointTask"; public static final String GET_JOB_INFO = API_ROUTE + "getJobInstance"; public static final int API_ERROR = 1; - public static final String API_VERSION_ERROR_TIPS = "Please check that the dinky version is greater than or equal to 0.6.5"; + public static final String API_VERSION_ERROR_TIPS = + "Please check that the dinky version is greater than or equal to 0.6.5"; public static final String API_RESULT_DATAS = "datas"; public static final String SAVEPOINT_CANCEL = "cancel"; diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-dms/pom.xml b/dolphinscheduler-task-plugin/dolphinscheduler-task-dms/pom.xml index e291594a08b9..8a3bb854134c 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-dms/pom.xml +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-dms/pom.xml @@ -15,15 +15,14 @@ ~ See the License for the specific language governing permissions and ~ limitations under the License. --> - + 4.0.0 - dolphinscheduler-task-plugin org.apache.dolphinscheduler + dolphinscheduler-task-plugin dev-SNAPSHOT - 4.0.0 dolphinscheduler-task-dms jar diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-dms/src/main/java/org/apache/dolphinscheduler/plugin/task/dms/DmsParameters.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-dms/src/main/java/org/apache/dolphinscheduler/plugin/task/dms/DmsParameters.java index de30df70c55f..c53dac33580f 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-dms/src/main/java/org/apache/dolphinscheduler/plugin/task/dms/DmsParameters.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-dms/src/main/java/org/apache/dolphinscheduler/plugin/task/dms/DmsParameters.java @@ -22,10 +22,10 @@ import java.util.Date; import java.util.List; -import com.amazonaws.services.databasemigrationservice.model.Tag; - import lombok.Data; +import com.amazonaws.services.databasemigrationservice.model.Tag; + @Data public class DmsParameters extends AbstractParameters { @@ -57,7 +57,7 @@ public boolean checkParameters() { flag = replicationTaskArn != null; } else { flag = sourceEndpointArn != null && targetEndpointArn != null && replicationInstanceArn != null - && migrationType != null && replicationTaskIdentifier != null && tableMappings != null; + && migrationType != null && replicationTaskIdentifier != null && tableMappings != null; } return flag; } diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-dms/src/main/java/org/apache/dolphinscheduler/plugin/task/dms/DmsTask.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-dms/src/main/java/org/apache/dolphinscheduler/plugin/task/dms/DmsTask.java index 177ca9bf1974..a54e0aad4697 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-dms/src/main/java/org/apache/dolphinscheduler/plugin/task/dms/DmsTask.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-dms/src/main/java/org/apache/dolphinscheduler/plugin/task/dms/DmsTask.java @@ -44,11 +44,11 @@ public class DmsTask extends AbstractRemoteTask { private static final ObjectMapper objectMapper = - new ObjectMapper().configure(FAIL_ON_UNKNOWN_PROPERTIES, false) - .configure(ACCEPT_EMPTY_ARRAY_AS_NULL_OBJECT, true) - .configure(READ_UNKNOWN_ENUM_VALUES_AS_NULL, true) - .configure(REQUIRE_SETTERS_FOR_GETTERS, true) - .setPropertyNamingStrategy(new PropertyNamingStrategy.UpperCamelCaseStrategy()); + new ObjectMapper().configure(FAIL_ON_UNKNOWN_PROPERTIES, false) + .configure(ACCEPT_EMPTY_ARRAY_AS_NULL_OBJECT, true) + .configure(READ_UNKNOWN_ENUM_VALUES_AS_NULL, true) + .configure(REQUIRE_SETTERS_FOR_GETTERS, true) + .setPropertyNamingStrategy(new PropertyNamingStrategy.UpperCamelCaseStrategy()); /** * taskExecutionContext */ @@ -102,7 +102,8 @@ public void trackApplicationStatus() { dmsHook.setReplicationTaskArn(appId.getReplicationTaskArn()); // if CdcStopPosition is not set, the task will not continue to check the running status if (isStopTaskWhenCdc()) { - logger.info("This is a cdc task and cdcStopPosition is not set, the task will not continue to check the running status"); + logger.info( + "This is a cdc task and cdcStopPosition is not set, the task will not continue to check the running status"); exitStatusCode = TaskConstants.EXIT_CODE_SUCCESS; return; } @@ -166,7 +167,8 @@ public int startReplicationTask() { } catch (InvalidResourceStateException e) { logger.error("Failed to start a task, error message: {}", e.getErrorMessage()); - // Only restart task when the error contains "Test connection", means instance can not connect to source or target + // Only restart task when the error contains "Test connection", means instance can not connect to source or + // target if (!e.getErrorMessage().contains("Test connection")) { return TaskConstants.EXIT_CODE_FAILURE; } @@ -231,7 +233,8 @@ public void convertJsonParameters() throws TaskException { // create a new parameter object using the json data if the json data is not empty if (parameters.getIsJsonFormat() && parameters.getJsonData() != null) { // combining local and global parameters - String jsonData = ParameterUtils.convertParameterPlaceholders(parameters.getJsonData(), ParamUtils.convert(taskExecutionContext.getPrepareParamsMap())); + String jsonData = ParameterUtils.convertParameterPlaceholders(parameters.getJsonData(), + ParamUtils.convert(taskExecutionContext.getPrepareParamsMap())); boolean isRestartTask = parameters.getIsRestartTask(); try { diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-dms/src/main/java/org/apache/dolphinscheduler/plugin/task/dms/DmsTaskChannelFactory.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-dms/src/main/java/org/apache/dolphinscheduler/plugin/task/dms/DmsTaskChannelFactory.java index addc00a02855..a8564ff8a178 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-dms/src/main/java/org/apache/dolphinscheduler/plugin/task/dms/DmsTaskChannelFactory.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-dms/src/main/java/org/apache/dolphinscheduler/plugin/task/dms/DmsTaskChannelFactory.java @@ -15,7 +15,6 @@ * limitations under the License. */ - package org.apache.dolphinscheduler.plugin.task.dms; import org.apache.dolphinscheduler.plugin.task.api.TaskChannel; diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-dvc/pom.xml b/dolphinscheduler-task-plugin/dolphinscheduler-task-dvc/pom.xml index a0435647ca13..93825996f453 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-dvc/pom.xml +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-dvc/pom.xml @@ -15,15 +15,14 @@ ~ See the License for the specific language governing permissions and ~ limitations under the License. --> - + 4.0.0 - dolphinscheduler-task-plugin org.apache.dolphinscheduler + dolphinscheduler-task-plugin dev-SNAPSHOT - 4.0.0 dolphinscheduler-task-dvc jar diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-dvc/src/main/java/org/apache/dolphinscheduler/plugin/task/dvc/DvcConstants.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-dvc/src/main/java/org/apache/dolphinscheduler/plugin/task/dvc/DvcConstants.java index 6f65bedc7466..d43ecbedd567 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-dvc/src/main/java/org/apache/dolphinscheduler/plugin/task/dvc/DvcConstants.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-dvc/src/main/java/org/apache/dolphinscheduler/plugin/task/dvc/DvcConstants.java @@ -18,17 +18,20 @@ package org.apache.dolphinscheduler.plugin.task.dvc; public class DvcConstants { + private DvcConstants() { throw new IllegalStateException("Utility class"); } public static final class DVC_TASK_TYPE { + public static final String UPLOAD = "Upload"; public static final String DOWNLOAD = "Download"; public static final String INIT = "Init DVC"; }; - public static final String CHECK_AND_SET_DVC_REPO = "which dvc || { echo \"dvc does not exist\"; exit 1; }; DVC_REPO=%s"; + public static final String CHECK_AND_SET_DVC_REPO = + "which dvc || { echo \"dvc does not exist\"; exit 1; }; DVC_REPO=%s"; public static final String SET_DATA_PATH = "DVC_DATA_PATH=%s"; @@ -38,19 +41,19 @@ public static final class DVC_TASK_TYPE { public static final String SET_MESSAGE = "DVC_MESSAGE=\"%s\""; - public static final String GIT_CLONE_DVC_REPO = "git clone $DVC_REPO dvc-repository; cd dvc-repository; pwd"; + public static final String GIT_CLONE_DVC_REPO = "git clone $DVC_REPO dvc-repository; cd dvc-repository; pwd"; - public static final String DVC_AUTOSTAGE = "dvc config core.autostage true --local || exit 1"; + public static final String DVC_AUTOSTAGE = "dvc config core.autostage true --local || exit 1"; - public static final String DVC_ADD_DATA = "dvc add $DVC_DATA_PATH -v -o $DVC_DATA_LOCATION --to-remote || exit 1"; + public static final String DVC_ADD_DATA = "dvc add $DVC_DATA_PATH -v -o $DVC_DATA_LOCATION --to-remote || exit 1"; - public static final String GIT_UPDATE_FOR_UPDATE_DATA = "git commit -am \"$DVC_MESSAGE\"\n" + + public static final String GIT_UPDATE_FOR_UPDATE_DATA = "git commit -am \"$DVC_MESSAGE\"\n" + "git tag \"$DVC_VERSION\" -m \"$DVC_MESSAGE\"\n" + "git push --all\n" + "git push --tags"; - public static final String DVC_DOWNLOAD = "dvc get $DVC_REPO $DVC_DATA_LOCATION -o $DVC_DATA_PATH -v --rev $DVC_VERSION"; - + public static final String DVC_DOWNLOAD = + "dvc get $DVC_REPO $DVC_DATA_LOCATION -o $DVC_DATA_PATH -v --rev $DVC_VERSION"; public static final String DVC_INIT = "dvc init || exit 1"; @@ -59,4 +62,3 @@ public static final class DVC_TASK_TYPE { public static final String GIT_UPDATE_FOR_INIT_DVC = "git commit -am \"init dvc project and add remote\"; git push"; } - diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-dvc/src/main/java/org/apache/dolphinscheduler/plugin/task/dvc/DvcParameters.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-dvc/src/main/java/org/apache/dolphinscheduler/plugin/task/dvc/DvcParameters.java index ec93e482843c..932abf83b7b6 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-dvc/src/main/java/org/apache/dolphinscheduler/plugin/task/dvc/DvcParameters.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-dvc/src/main/java/org/apache/dolphinscheduler/plugin/task/dvc/DvcParameters.java @@ -17,11 +17,11 @@ package org.apache.dolphinscheduler.plugin.task.dvc; -import lombok.Data; - import org.apache.dolphinscheduler.plugin.task.api.parameters.AbstractParameters; import org.apache.dolphinscheduler.spi.utils.StringUtils; +import lombok.Data; + @Data public class DvcParameters extends AbstractParameters { @@ -53,24 +53,23 @@ public boolean checkParameters() { switch (dvcTaskType) { case DvcConstants.DVC_TASK_TYPE.UPLOAD: return StringUtils.isNotEmpty(dvcRepository) && - StringUtils.isNotEmpty(dvcDataLocation) && - StringUtils.isNotEmpty(dvcLoadSaveDataPath) && - StringUtils.isNotEmpty(dvcVersion) && - StringUtils.isNotEmpty(dvcMessage); + StringUtils.isNotEmpty(dvcDataLocation) && + StringUtils.isNotEmpty(dvcLoadSaveDataPath) && + StringUtils.isNotEmpty(dvcVersion) && + StringUtils.isNotEmpty(dvcMessage); case DvcConstants.DVC_TASK_TYPE.DOWNLOAD: return StringUtils.isNotEmpty(dvcRepository) && - StringUtils.isNotEmpty(dvcDataLocation) && - StringUtils.isNotEmpty(dvcLoadSaveDataPath) && - StringUtils.isNotEmpty(dvcVersion); + StringUtils.isNotEmpty(dvcDataLocation) && + StringUtils.isNotEmpty(dvcLoadSaveDataPath) && + StringUtils.isNotEmpty(dvcVersion); case DvcConstants.DVC_TASK_TYPE.INIT: return StringUtils.isNotEmpty(dvcRepository) && - StringUtils.isNotEmpty(dvcStoreUrl); + StringUtils.isNotEmpty(dvcStoreUrl); default: return false; } } } - diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-dvc/src/main/java/org/apache/dolphinscheduler/plugin/task/dvc/DvcTaskChannel.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-dvc/src/main/java/org/apache/dolphinscheduler/plugin/task/dvc/DvcTaskChannel.java index adccccede793..107b21b3b366 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-dvc/src/main/java/org/apache/dolphinscheduler/plugin/task/dvc/DvcTaskChannel.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-dvc/src/main/java/org/apache/dolphinscheduler/plugin/task/dvc/DvcTaskChannel.java @@ -24,7 +24,6 @@ import org.apache.dolphinscheduler.plugin.task.api.parameters.resource.ResourceParametersHelper; import org.apache.dolphinscheduler.spi.utils.JSONUtils; - public class DvcTaskChannel implements TaskChannel { @Override @@ -48,4 +47,3 @@ public ResourceParametersHelper getResources(String parameters) { } } - diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-dvc/src/main/java/org/apache/dolphinscheduler/plugin/task/dvc/DvcTaskChannelFactory.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-dvc/src/main/java/org/apache/dolphinscheduler/plugin/task/dvc/DvcTaskChannelFactory.java index f028d4e0d609..a29ffded5b7f 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-dvc/src/main/java/org/apache/dolphinscheduler/plugin/task/dvc/DvcTaskChannelFactory.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-dvc/src/main/java/org/apache/dolphinscheduler/plugin/task/dvc/DvcTaskChannelFactory.java @@ -32,6 +32,7 @@ @AutoService(TaskChannelFactory.class) public class DvcTaskChannelFactory implements TaskChannelFactory { + @Override public TaskChannel create() { return new DvcTaskChannel(); @@ -46,13 +47,15 @@ public String getName() { public List getParams() { List paramsList = new ArrayList<>(); - InputParam nodeName = InputParam.newBuilder("name", "$t('Node name')").addValidate(Validate.newBuilder().setRequired(true).build()).build(); + InputParam nodeName = InputParam.newBuilder("name", "$t('Node name')") + .addValidate(Validate.newBuilder().setRequired(true).build()).build(); - RadioParam runFlag = RadioParam.newBuilder("runFlag", "RUN_FLAG").addParamsOptions(new ParamsOptions("NORMAL", "NORMAL", false)).addParamsOptions(new ParamsOptions("FORBIDDEN", "FORBIDDEN", false)).build(); + RadioParam runFlag = RadioParam.newBuilder("runFlag", "RUN_FLAG") + .addParamsOptions(new ParamsOptions("NORMAL", "NORMAL", false)) + .addParamsOptions(new ParamsOptions("FORBIDDEN", "FORBIDDEN", false)).build(); paramsList.add(nodeName); paramsList.add(runFlag); return paramsList; } } - diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-emr/pom.xml b/dolphinscheduler-task-plugin/dolphinscheduler-task-emr/pom.xml index 7a4e9913a251..969756a99146 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-emr/pom.xml +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-emr/pom.xml @@ -15,15 +15,14 @@ ~ See the License for the specific language governing permissions and ~ limitations under the License. --> - + 4.0.0 - dolphinscheduler-task-plugin org.apache.dolphinscheduler + dolphinscheduler-task-plugin dev-SNAPSHOT - 4.0.0 dolphinscheduler-task-emr jar diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-emr/src/main/java/org/apache/dolphinscheduler/plugin/task/emr/AbstractEmrTask.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-emr/src/main/java/org/apache/dolphinscheduler/plugin/task/emr/AbstractEmrTask.java index 3adce9e1b5bf..c6be686347e6 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-emr/src/main/java/org/apache/dolphinscheduler/plugin/task/emr/AbstractEmrTask.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-emr/src/main/java/org/apache/dolphinscheduler/plugin/task/emr/AbstractEmrTask.java @@ -21,6 +21,7 @@ import static com.fasterxml.jackson.databind.DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES; import static com.fasterxml.jackson.databind.DeserializationFeature.READ_UNKNOWN_ENUM_VALUES_AS_NULL; import static com.fasterxml.jackson.databind.MapperFeature.REQUIRE_SETTERS_FOR_GETTERS; + import org.apache.dolphinscheduler.common.utils.PropertyUtils; import org.apache.dolphinscheduler.plugin.task.api.AbstractRemoteTask; import org.apache.dolphinscheduler.plugin.task.api.TaskConstants; diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-emr/src/main/java/org/apache/dolphinscheduler/plugin/task/emr/EmrParameters.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-emr/src/main/java/org/apache/dolphinscheduler/plugin/task/emr/EmrParameters.java index 1e38a66d8138..8b045d6edc62 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-emr/src/main/java/org/apache/dolphinscheduler/plugin/task/emr/EmrParameters.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-emr/src/main/java/org/apache/dolphinscheduler/plugin/task/emr/EmrParameters.java @@ -55,10 +55,11 @@ public class EmrParameters extends AbstractParameters { public boolean checkParameters() { /* * When saving a task, the programType cannot be empty and jobFlowDefineJson or stepsDefineJson cannot be empty: - * (1) When ProgramType is RUN_JOB_FLOW, jobFlowDefineJson cannot be empty. - * (2) When ProgramType is ADD_JOB_FLOW_STEPS, stepsDefineJson cannot be empty. + * (1) When ProgramType is RUN_JOB_FLOW, jobFlowDefineJson cannot be empty. (2) When ProgramType is + * ADD_JOB_FLOW_STEPS, stepsDefineJson cannot be empty. */ - return programType != null && (StringUtils.isNotEmpty(jobFlowDefineJson) || StringUtils.isNotEmpty(stepsDefineJson)); + return programType != null + && (StringUtils.isNotEmpty(jobFlowDefineJson) || StringUtils.isNotEmpty(stepsDefineJson)); } @Override @@ -70,9 +71,9 @@ public List getResourceFilesList() { @Override public String toString() { return "EmrParameters{" - + "programType=" + programType - + ", jobFlowDefineJson='" + jobFlowDefineJson + '\'' - + ", stepsDefineJson='" + stepsDefineJson + '\'' - + '}'; + + "programType=" + programType + + ", jobFlowDefineJson='" + jobFlowDefineJson + '\'' + + ", stepsDefineJson='" + stepsDefineJson + '\'' + + '}'; } } diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-emr/src/main/java/org/apache/dolphinscheduler/plugin/task/emr/EmrTaskChannel.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-emr/src/main/java/org/apache/dolphinscheduler/plugin/task/emr/EmrTaskChannel.java index e59b4e361359..812f7ed12166 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-emr/src/main/java/org/apache/dolphinscheduler/plugin/task/emr/EmrTaskChannel.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-emr/src/main/java/org/apache/dolphinscheduler/plugin/task/emr/EmrTaskChannel.java @@ -26,6 +26,7 @@ import org.apache.dolphinscheduler.spi.utils.JSONUtils; public class EmrTaskChannel implements TaskChannel { + @Override public void cancelApplication(boolean status) { // no need diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-emr/src/main/java/org/apache/dolphinscheduler/plugin/task/emr/EmrTaskChannelFactory.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-emr/src/main/java/org/apache/dolphinscheduler/plugin/task/emr/EmrTaskChannelFactory.java index daf67cb2d231..cad5583f07a1 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-emr/src/main/java/org/apache/dolphinscheduler/plugin/task/emr/EmrTaskChannelFactory.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-emr/src/main/java/org/apache/dolphinscheduler/plugin/task/emr/EmrTaskChannelFactory.java @@ -28,6 +28,7 @@ @AutoService(TaskChannelFactory.class) public class EmrTaskChannelFactory implements TaskChannelFactory { + @Override public String getName() { return "EMR"; diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-flink-stream/pom.xml b/dolphinscheduler-task-plugin/dolphinscheduler-task-flink-stream/pom.xml index e2dd5753f8a9..2cd1a9f4f22f 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-flink-stream/pom.xml +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-flink-stream/pom.xml @@ -15,15 +15,14 @@ ~ See the License for the specific language governing permissions and ~ limitations under the License. --> - + 4.0.0 - dolphinscheduler-task-plugin org.apache.dolphinscheduler + dolphinscheduler-task-plugin dev-SNAPSHOT - 4.0.0 dolphinscheduler-task-flink-stream jar @@ -45,4 +44,4 @@ ${project.version} - \ No newline at end of file + diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-flink-stream/src/main/java/org/apache/dolphinscheduler/plugin/task/flink/FlinkStreamTaskChannel.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-flink-stream/src/main/java/org/apache/dolphinscheduler/plugin/task/flink/FlinkStreamTaskChannel.java index 5a86dbe3f2b3..2bf37e326c5d 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-flink-stream/src/main/java/org/apache/dolphinscheduler/plugin/task/flink/FlinkStreamTaskChannel.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-flink-stream/src/main/java/org/apache/dolphinscheduler/plugin/task/flink/FlinkStreamTaskChannel.java @@ -18,7 +18,6 @@ package org.apache.dolphinscheduler.plugin.task.flink; import org.apache.dolphinscheduler.plugin.task.api.AbstractTask; -import org.apache.dolphinscheduler.plugin.task.api.TaskChannel; import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext; import org.apache.dolphinscheduler.plugin.task.api.parameters.AbstractParameters; import org.apache.dolphinscheduler.plugin.task.api.parameters.ParametersNode; @@ -27,6 +26,7 @@ import org.apache.dolphinscheduler.spi.utils.JSONUtils; public class FlinkStreamTaskChannel implements StreamTaskChannel { + @Override public void cancelApplication(boolean status) { diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-flink-stream/src/main/java/org/apache/dolphinscheduler/plugin/task/flink/FlinkStreamTaskChannelFactory.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-flink-stream/src/main/java/org/apache/dolphinscheduler/plugin/task/flink/FlinkStreamTaskChannelFactory.java index f90a1e47b765..475386bf3654 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-flink-stream/src/main/java/org/apache/dolphinscheduler/plugin/task/flink/FlinkStreamTaskChannelFactory.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-flink-stream/src/main/java/org/apache/dolphinscheduler/plugin/task/flink/FlinkStreamTaskChannelFactory.java @@ -27,6 +27,7 @@ @AutoService(TaskChannelFactory.class) public class FlinkStreamTaskChannelFactory implements TaskChannelFactory { + @Override public TaskChannel create() { return new FlinkStreamTaskChannel(); diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-flink/pom.xml b/dolphinscheduler-task-plugin/dolphinscheduler-task-flink/pom.xml index c43d507d3cc6..5a208a89ac5c 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-flink/pom.xml +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-flink/pom.xml @@ -15,29 +15,28 @@ ~ See the License for the specific language governing permissions and ~ limitations under the License. --> - + 4.0.0 - dolphinscheduler-task-plugin org.apache.dolphinscheduler + dolphinscheduler-task-plugin dev-SNAPSHOT - 4.0.0 dolphinscheduler-task-flink jar - - org.apache.dolphinscheduler - dolphinscheduler-spi - provided - - - org.apache.dolphinscheduler - dolphinscheduler-task-api - ${project.version} - + + org.apache.dolphinscheduler + dolphinscheduler-spi + provided + + + org.apache.dolphinscheduler + dolphinscheduler-task-api + ${project.version} + diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-flink/src/main/java/org/apache/dolphinscheduler/plugin/task/flink/FileUtils.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-flink/src/main/java/org/apache/dolphinscheduler/plugin/task/flink/FileUtils.java index 33f0fecbfbde..4672fad31b14 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-flink/src/main/java/org/apache/dolphinscheduler/plugin/task/flink/FileUtils.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-flink/src/main/java/org/apache/dolphinscheduler/plugin/task/flink/FileUtils.java @@ -17,11 +17,12 @@ package org.apache.dolphinscheduler.plugin.task.flink; -import org.apache.commons.lang3.SystemUtils; +import static org.apache.dolphinscheduler.plugin.task.api.TaskConstants.RWXR_XR_X; + import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext; import org.apache.dolphinscheduler.spi.utils.StringUtils; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; + +import org.apache.commons.lang3.SystemUtils; import java.io.File; import java.io.IOException; @@ -34,18 +35,23 @@ import java.nio.file.attribute.PosixFilePermissions; import java.util.Set; -import static org.apache.dolphinscheduler.plugin.task.api.TaskConstants.RWXR_XR_X; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; public class FileUtils { + private static final Logger LOGGER = LoggerFactory.getLogger(FileUtils.class); - private FileUtils() {} + private FileUtils() { + } public static String getInitScriptFilePath(TaskExecutionContext taskExecutionContext) { - return String.format("%s/%s_init.sql", taskExecutionContext.getExecutePath(), taskExecutionContext.getTaskAppId()); + return String.format("%s/%s_init.sql", taskExecutionContext.getExecutePath(), + taskExecutionContext.getTaskAppId()); } public static String getScriptFilePath(TaskExecutionContext taskExecutionContext) { - return String.format("%s/%s_node.sql", taskExecutionContext.getExecutePath(), taskExecutionContext.getTaskAppId()); + return String.format("%s/%s_node.sql", taskExecutionContext.getExecutePath(), + taskExecutionContext.getTaskAppId()); } public static void generateScriptFile(TaskExecutionContext taskExecutionContext, FlinkParameters flinkParameters) { @@ -53,8 +59,7 @@ public static void generateScriptFile(TaskExecutionContext taskExecutionContext, String scriptFilePath = FileUtils.getScriptFilePath(taskExecutionContext); String initOptionsString = StringUtils.join( FlinkArgsUtils.buildInitOptionsForSql(flinkParameters), - FlinkConstants.FLINK_SQL_NEWLINE - ).concat(FlinkConstants.FLINK_SQL_NEWLINE); + FlinkConstants.FLINK_SQL_NEWLINE).concat(FlinkConstants.FLINK_SQL_NEWLINE); writeScriptFile(initScriptFilePath, initOptionsString + flinkParameters.getInitScript()); writeScriptFile(scriptFilePath, flinkParameters.getRawScript()); } @@ -66,7 +71,8 @@ private static void writeScriptFile(String scriptFileFullPath, String script) { try { Files.delete(path); } catch (IOException e) { - throw new RuntimeException(String.format("Flink Script file exists in path: %s before creation and cannot be deleted", path), e); + throw new RuntimeException(String + .format("Flink Script file exists in path: %s before creation and cannot be deleted", path), e); } } @@ -96,7 +102,7 @@ private static void writeStringToFile(File file, String content, StandardOpenOpt LOGGER.info("Writing content: " + content); LOGGER.info("To file: " + file.getAbsolutePath()); Files.write(file.getAbsoluteFile().toPath(), content.getBytes(StandardCharsets.UTF_8), standardOpenOption); - } catch(IOException e) { + } catch (IOException e) { throw new RuntimeException("Error writing file: " + file.getAbsoluteFile(), e); } } diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-flink/src/main/java/org/apache/dolphinscheduler/plugin/task/flink/FlinkArgsUtils.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-flink/src/main/java/org/apache/dolphinscheduler/plugin/task/flink/FlinkArgsUtils.java index 403a9bc63224..9508008eb057 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-flink/src/main/java/org/apache/dolphinscheduler/plugin/task/flink/FlinkArgsUtils.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-flink/src/main/java/org/apache/dolphinscheduler/plugin/task/flink/FlinkArgsUtils.java @@ -24,8 +24,6 @@ import org.apache.dolphinscheduler.plugin.task.api.parser.ParameterUtils; import org.apache.dolphinscheduler.plugin.task.api.utils.ArgsUtils; import org.apache.dolphinscheduler.spi.utils.StringUtils; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import java.util.ArrayList; import java.util.List; @@ -95,7 +93,8 @@ public static List buildSavePointCommandLine(TaskExecutionContext taskEx * * @return argument list */ - private static List buildRunCommandLineForSql(TaskExecutionContext taskExecutionContext, FlinkParameters flinkParameters) { + private static List buildRunCommandLineForSql(TaskExecutionContext taskExecutionContext, + FlinkParameters flinkParameters) { List args = new ArrayList<>(); args.add(FlinkConstants.FLINK_SQL_COMMAND); @@ -120,7 +119,8 @@ private static List buildRunCommandLineForSql(TaskExecutionContext taskE public static List buildInitOptionsForSql(FlinkParameters flinkParameters) { List initOptions = new ArrayList<>(); - FlinkDeployMode deployMode = Optional.ofNullable(flinkParameters.getDeployMode()).orElse(FlinkDeployMode.CLUSTER); + FlinkDeployMode deployMode = + Optional.ofNullable(flinkParameters.getDeployMode()).orElse(FlinkDeployMode.CLUSTER); /** * Currently flink sql on yarn only supports yarn-per-job mode @@ -130,7 +130,8 @@ public static List buildInitOptionsForSql(FlinkParameters flinkParameter initOptions.add(String.format(FlinkConstants.FLINK_FORMAT_EXECUTION_TARGET, "local")); } else { // execution.target - initOptions.add(String.format(FlinkConstants.FLINK_FORMAT_EXECUTION_TARGET, FlinkConstants.FLINK_YARN_PER_JOB)); + initOptions.add( + String.format(FlinkConstants.FLINK_FORMAT_EXECUTION_TARGET, FlinkConstants.FLINK_YARN_PER_JOB)); // taskmanager.numberOfTaskSlots int slot = flinkParameters.getSlot(); @@ -141,19 +142,22 @@ public static List buildInitOptionsForSql(FlinkParameters flinkParameter // yarn.application.name String appName = flinkParameters.getAppName(); if (StringUtils.isNotEmpty(appName)) { - initOptions.add(String.format(FlinkConstants.FLINK_FORMAT_YARN_APPLICATION_NAME, ArgsUtils.escape(appName))); + initOptions.add( + String.format(FlinkConstants.FLINK_FORMAT_YARN_APPLICATION_NAME, ArgsUtils.escape(appName))); } // jobmanager.memory.process.size String jobManagerMemory = flinkParameters.getJobManagerMemory(); if (StringUtils.isNotEmpty(jobManagerMemory)) { - initOptions.add(String.format(FlinkConstants.FLINK_FORMAT_JOBMANAGER_MEMORY_PROCESS_SIZE, jobManagerMemory)); + initOptions.add( + String.format(FlinkConstants.FLINK_FORMAT_JOBMANAGER_MEMORY_PROCESS_SIZE, jobManagerMemory)); } // taskmanager.memory.process.size String taskManagerMemory = flinkParameters.getTaskManagerMemory(); if (StringUtils.isNotEmpty(taskManagerMemory)) { - initOptions.add(String.format(FlinkConstants.FLINK_FORMAT_TASKMANAGER_MEMORY_PROCESS_SIZE, taskManagerMemory)); + initOptions.add( + String.format(FlinkConstants.FLINK_FORMAT_TASKMANAGER_MEMORY_PROCESS_SIZE, taskManagerMemory)); } // yarn.application.queue @@ -175,7 +179,8 @@ public static List buildInitOptionsForSql(FlinkParameters flinkParameter return initOptions; } - private static List buildRunCommandLineForOthers(TaskExecutionContext taskExecutionContext, FlinkParameters flinkParameters) { + private static List buildRunCommandLineForOthers(TaskExecutionContext taskExecutionContext, + FlinkParameters flinkParameters) { List args = new ArrayList<>(); args.add(FlinkConstants.FLINK_COMMAND); @@ -184,23 +189,24 @@ private static List buildRunCommandLineForOthers(TaskExecutionContext ta // build run command switch (deployMode) { case CLUSTER: - if (FLINK_VERSION_AFTER_OR_EQUALS_1_12.equals(flinkVersion) || FLINK_VERSION_AFTER_OR_EQUALS_1_13.equals(flinkVersion)) { - args.add(FlinkConstants.FLINK_RUN); //run - args.add(FlinkConstants.FLINK_EXECUTION_TARGET); //-t - args.add(FlinkConstants.FLINK_YARN_PER_JOB); //yarn-per-job + if (FLINK_VERSION_AFTER_OR_EQUALS_1_12.equals(flinkVersion) + || FLINK_VERSION_AFTER_OR_EQUALS_1_13.equals(flinkVersion)) { + args.add(FlinkConstants.FLINK_RUN); // run + args.add(FlinkConstants.FLINK_EXECUTION_TARGET); // -t + args.add(FlinkConstants.FLINK_YARN_PER_JOB); // yarn-per-job } else { - args.add(FlinkConstants.FLINK_RUN); //run - args.add(FlinkConstants.FLINK_RUN_MODE); //-m - args.add(FlinkConstants.FLINK_YARN_CLUSTER); //yarn-cluster + args.add(FlinkConstants.FLINK_RUN); // run + args.add(FlinkConstants.FLINK_RUN_MODE); // -m + args.add(FlinkConstants.FLINK_YARN_CLUSTER); // yarn-cluster } break; case APPLICATION: - args.add(FlinkConstants.FLINK_RUN_APPLICATION); //run-application - args.add(FlinkConstants.FLINK_EXECUTION_TARGET); //-t - args.add(FlinkConstants.FLINK_YARN_APPLICATION); //yarn-application + args.add(FlinkConstants.FLINK_RUN_APPLICATION); // run-application + args.add(FlinkConstants.FLINK_EXECUTION_TARGET); // -t + args.add(FlinkConstants.FLINK_YARN_APPLICATION); // yarn-application break; case LOCAL: - args.add(FlinkConstants.FLINK_RUN); //run + args.add(FlinkConstants.FLINK_RUN); // run break; } @@ -213,11 +219,11 @@ private static List buildRunCommandLineForOthers(TaskExecutionContext ta int slot = flinkParameters.getSlot(); if (slot > 0) { args.add(FlinkConstants.FLINK_YARN_SLOT); - args.add(String.format("%d", slot)); //-ys + args.add(String.format("%d", slot)); // -ys } String appName = flinkParameters.getAppName(); - if (StringUtils.isNotEmpty(appName)) { //-ynm + if (StringUtils.isNotEmpty(appName)) { // -ynm args.add(FlinkConstants.FLINK_APP_NAME); args.add(ArgsUtils.escape(appName)); } @@ -225,7 +231,7 @@ private static List buildRunCommandLineForOthers(TaskExecutionContext ta // judge flink version, the parameter -yn has removed from flink 1.10 if (flinkVersion == null || FLINK_VERSION_BEFORE_1_10.equals(flinkVersion)) { int taskManager = flinkParameters.getTaskManager(); - if (taskManager > 0) { //-yn + if (taskManager > 0) { // -yn args.add(FlinkConstants.FLINK_TASK_MANAGE); args.add(String.format("%d", taskManager)); } @@ -233,7 +239,7 @@ private static List buildRunCommandLineForOthers(TaskExecutionContext ta String jobManagerMemory = flinkParameters.getJobManagerMemory(); if (StringUtils.isNotEmpty(jobManagerMemory)) { args.add(FlinkConstants.FLINK_JOB_MANAGE_MEM); - args.add(jobManagerMemory); //-yjm + args.add(jobManagerMemory); // -yjm } String taskManagerMemory = flinkParameters.getTaskManagerMemory(); @@ -257,10 +263,11 @@ private static List buildRunCommandLineForOthers(TaskExecutionContext ta int parallelism = flinkParameters.getParallelism(); if (parallelism > 0) { args.add(FlinkConstants.FLINK_PARALLELISM); - args.add(String.format("%d", parallelism)); // -p + args.add(String.format("%d", parallelism)); // -p } - // If the job is submitted in attached mode, perform a best-effort cluster shutdown when the CLI is terminated abruptly + // If the job is submitted in attached mode, perform a best-effort cluster shutdown when the CLI is terminated + // abruptly // The task status will be synchronized with the cluster job status args.add(FlinkConstants.FLINK_SHUTDOWN_ON_ATTACHED_EXIT); // -sae @@ -272,14 +279,14 @@ private static List buildRunCommandLineForOthers(TaskExecutionContext ta ProgramType programType = flinkParameters.getProgramType(); String mainClass = flinkParameters.getMainClass(); if (programType != null && programType != ProgramType.PYTHON && StringUtils.isNotEmpty(mainClass)) { - args.add(FlinkConstants.FLINK_MAIN_CLASS); //-c - args.add(flinkParameters.getMainClass()); //main class + args.add(FlinkConstants.FLINK_MAIN_CLASS); // -c + args.add(flinkParameters.getMainClass()); // main class } ResourceInfo mainJar = flinkParameters.getMainJar(); if (mainJar != null) { // -py - if(ProgramType.PYTHON == programType) { + if (ProgramType.PYTHON == programType) { args.add(FlinkConstants.FLINK_PYTHON); } args.add(mainJar.getRes()); diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-flink/src/main/java/org/apache/dolphinscheduler/plugin/task/flink/FlinkTaskChannel.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-flink/src/main/java/org/apache/dolphinscheduler/plugin/task/flink/FlinkTaskChannel.java index 2f1daf3bde5a..e41a1f1a6bfa 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-flink/src/main/java/org/apache/dolphinscheduler/plugin/task/flink/FlinkTaskChannel.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-flink/src/main/java/org/apache/dolphinscheduler/plugin/task/flink/FlinkTaskChannel.java @@ -25,6 +25,7 @@ import org.apache.dolphinscheduler.spi.utils.JSONUtils; public class FlinkTaskChannel implements TaskChannel { + @Override public void cancelApplication(boolean status) { diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-flink/src/main/java/org/apache/dolphinscheduler/plugin/task/flink/FlinkTaskChannelFactory.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-flink/src/main/java/org/apache/dolphinscheduler/plugin/task/flink/FlinkTaskChannelFactory.java index 48363803c599..953f013fcec2 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-flink/src/main/java/org/apache/dolphinscheduler/plugin/task/flink/FlinkTaskChannelFactory.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-flink/src/main/java/org/apache/dolphinscheduler/plugin/task/flink/FlinkTaskChannelFactory.java @@ -27,6 +27,7 @@ @AutoService(TaskChannelFactory.class) public class FlinkTaskChannelFactory implements TaskChannelFactory { + @Override public TaskChannel create() { return new FlinkTaskChannel(); diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-http/pom.xml b/dolphinscheduler-task-plugin/dolphinscheduler-task-http/pom.xml index 015d3193e001..fb66bf47064a 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-http/pom.xml +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-http/pom.xml @@ -23,10 +23,10 @@ dolphinscheduler-task-plugin dev-SNAPSHOT - + dolphinscheduler-task-http jar - + org.apache.dolphinscheduler @@ -38,7 +38,7 @@ dolphinscheduler-task-api ${project.version} - + org.apache.commons commons-collections4 @@ -51,7 +51,7 @@ org.apache.httpcomponents httpcore - + com.squareup.okhttp3 mockwebserver diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-http/src/main/java/org/apache/dolphinscheduler/plugin/task/http/HttpCheckCondition.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-http/src/main/java/org/apache/dolphinscheduler/plugin/task/http/HttpCheckCondition.java index b2f423b22fde..479540a1f509 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-http/src/main/java/org/apache/dolphinscheduler/plugin/task/http/HttpCheckCondition.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-http/src/main/java/org/apache/dolphinscheduler/plugin/task/http/HttpCheckCondition.java @@ -27,6 +27,6 @@ public enum HttpCheckCondition { * 2 body_contains * 3 body_not_contains */ - STATUS_CODE_DEFAULT,STATUS_CODE_CUSTOM, BODY_CONTAINS, BODY_NOT_CONTAINS + STATUS_CODE_DEFAULT, STATUS_CODE_CUSTOM, BODY_CONTAINS, BODY_NOT_CONTAINS } diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-http/src/main/java/org/apache/dolphinscheduler/plugin/task/http/HttpParameters.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-http/src/main/java/org/apache/dolphinscheduler/plugin/task/http/HttpParameters.java index 2676dd675c7c..ce76ed1d9a48 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-http/src/main/java/org/apache/dolphinscheduler/plugin/task/http/HttpParameters.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-http/src/main/java/org/apache/dolphinscheduler/plugin/task/http/HttpParameters.java @@ -28,6 +28,7 @@ * http parameter */ public class HttpParameters extends AbstractParameters { + /** * url */ diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-http/src/main/java/org/apache/dolphinscheduler/plugin/task/http/HttpParametersType.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-http/src/main/java/org/apache/dolphinscheduler/plugin/task/http/HttpParametersType.java index 5e0437b6986c..b027fd874c28 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-http/src/main/java/org/apache/dolphinscheduler/plugin/task/http/HttpParametersType.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-http/src/main/java/org/apache/dolphinscheduler/plugin/task/http/HttpParametersType.java @@ -26,5 +26,5 @@ public enum HttpParametersType { * 1 body; * 2 headers; */ - PARAMETER,BODY,HEADERS + PARAMETER, BODY, HEADERS } diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-http/src/main/java/org/apache/dolphinscheduler/plugin/task/http/HttpProperty.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-http/src/main/java/org/apache/dolphinscheduler/plugin/task/http/HttpProperty.java index 4044b2c57e7d..5041417b9380 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-http/src/main/java/org/apache/dolphinscheduler/plugin/task/http/HttpProperty.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-http/src/main/java/org/apache/dolphinscheduler/plugin/task/http/HttpProperty.java @@ -20,6 +20,7 @@ import java.util.Objects; public class HttpProperty { + /** * key */ diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-http/src/main/java/org/apache/dolphinscheduler/plugin/task/http/HttpTask.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-http/src/main/java/org/apache/dolphinscheduler/plugin/task/http/HttpTask.java index 2562d5dcc184..3b16a920f140 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-http/src/main/java/org/apache/dolphinscheduler/plugin/task/http/HttpTask.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-http/src/main/java/org/apache/dolphinscheduler/plugin/task/http/HttpTask.java @@ -95,13 +95,15 @@ public void handle(TaskCallBack taskCallBack) throws TaskException { String statusCode = null; String body = null; - try (CloseableHttpClient client = createHttpClient(); - CloseableHttpResponse response = sendRequest(client)) { + try ( + CloseableHttpClient client = createHttpClient(); + CloseableHttpResponse response = sendRequest(client)) { statusCode = String.valueOf(getStatusCode(response)); body = getResponseBody(response); exitStatusCode = validResponse(body, statusCode); long costTime = System.currentTimeMillis() - startTime; - logger.info("startTime: {}, httpUrl: {}, httpMethod: {}, costTime : {} milliseconds, statusCode : {}, body : {}, log : {}", + logger.info( + "startTime: {}, httpUrl: {}, httpMethod: {}, costTime : {} milliseconds, statusCode : {}, body : {}, log : {}", formatTimeStamp, httpParameters.getUrl(), httpParameters.getHttpMethod(), costTime, statusCode, body, output); } catch (Exception e) { @@ -141,7 +143,8 @@ protected CloseableHttpResponse sendRequest(CloseableHttpClient client) throws I } } addRequestParams(builder, httpPropertyList); - String requestUrl = ParameterUtils.convertParameterPlaceholders(httpParameters.getUrl(), ParamUtils.convert(paramsMap)); + String requestUrl = + ParameterUtils.convertParameterPlaceholders(httpParameters.getUrl(), ParamUtils.convert(paramsMap)); HttpUriRequest request = builder.setUri(requestUrl).build(); setHeaders(request, httpPropertyList); return client.execute(request); @@ -202,7 +205,8 @@ protected int validResponse(String body, String statusCode) { break; case STATUS_CODE_CUSTOM: if (!statusCode.equals(httpParameters.getCondition())) { - appendMessage(httpParameters.getUrl() + " statuscode: " + statusCode + ", Must be: " + httpParameters.getCondition()); + appendMessage(httpParameters.getUrl() + " statuscode: " + statusCode + ", Must be: " + + httpParameters.getCondition()); exitStatusCode = -1; } break; @@ -293,7 +297,8 @@ protected CloseableHttpClient createHttpClient() { * @return RequestConfig */ private RequestConfig requestConfig() { - return RequestConfig.custom().setSocketTimeout(httpParameters.getSocketTimeout()).setConnectTimeout(httpParameters.getConnectTimeout()).build(); + return RequestConfig.custom().setSocketTimeout(httpParameters.getSocketTimeout()) + .setConnectTimeout(httpParameters.getConnectTimeout()).build(); } /** diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-java/pom.xml b/dolphinscheduler-task-plugin/dolphinscheduler-task-java/pom.xml index 36aa9afeb2a2..b28effa47b99 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-java/pom.xml +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-java/pom.xml @@ -15,16 +15,15 @@ ~ See the License for the specific language governing permissions and ~ limitations under the License. --> - + + 4.0.0 - dolphinscheduler-task-plugin org.apache.dolphinscheduler + dolphinscheduler-task-plugin dev-SNAPSHOT - - 4.0.0 dolphinscheduler-task-java jar @@ -42,4 +41,4 @@ - \ No newline at end of file + diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-java/src/main/java/org/apache/dolphinscheduler/plugin/task/java/JavaConstants.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-java/src/main/java/org/apache/dolphinscheduler/plugin/task/java/JavaConstants.java index e4d941db309d..e9bfd581c649 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-java/src/main/java/org/apache/dolphinscheduler/plugin/task/java/JavaConstants.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-java/src/main/java/org/apache/dolphinscheduler/plugin/task/java/JavaConstants.java @@ -17,8 +17,6 @@ package org.apache.dolphinscheduler.plugin.task.java; -import java.io.File; - public class JavaConstants { private JavaConstants() { diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-java/src/main/java/org/apache/dolphinscheduler/plugin/task/java/JavaParameters.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-java/src/main/java/org/apache/dolphinscheduler/plugin/task/java/JavaParameters.java index 527441b634cc..de0da08358a1 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-java/src/main/java/org/apache/dolphinscheduler/plugin/task/java/JavaParameters.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-java/src/main/java/org/apache/dolphinscheduler/plugin/task/java/JavaParameters.java @@ -27,6 +27,7 @@ @Data public class JavaParameters extends AbstractParameters { + /** * origin java script */ diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-java/src/main/java/org/apache/dolphinscheduler/plugin/task/java/JavaTask.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-java/src/main/java/org/apache/dolphinscheduler/plugin/task/java/JavaTask.java index e23eb5124c8b..3f0f6b2ec565 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-java/src/main/java/org/apache/dolphinscheduler/plugin/task/java/JavaTask.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-java/src/main/java/org/apache/dolphinscheduler/plugin/task/java/JavaTask.java @@ -125,8 +125,10 @@ public String getPreScript() { public void handle(TaskCallBack taskCallBack) throws TaskException { try { // Step 1: judge if is java or jar run type. - // Step 2 case1: the jar run type builds the command directly, adding resource to the java -jar class when building the command - // Step 2 case2: the java run type, first replace the custom parameters, then compile the code, and then build the command will add resource + // Step 2 case1: the jar run type builds the command directly, adding resource to the java -jar class when + // building the command + // Step 2 case2: the java run type, first replace the custom parameters, then compile the code, and then + // build the command will add resource // Step 3: to run the command String command = null; switch (javaParameters.getRunType()) { @@ -236,7 +238,6 @@ public AbstractParameters getParameters() { return javaParameters; } - /** * Replaces placeholders such as local variables in source files * @@ -280,7 +281,7 @@ protected static String convertJavaSourceCodePlaceholders(String rawScript) thro protected void createJavaSourceFileIfNotExists(String sourceCode, String fileName) throws IOException { logger.info("tenantCode: {}, task dir:{}", taskRequest.getTenantCode(), taskRequest.getExecutePath()); if (!Files.exists(Paths.get(fileName))) { - logger.info("the java source code:{}, will be write to the file: {}", fileName,sourceCode); + logger.info("the java source code:{}, will be write to the file: {}", fileName, sourceCode); // write data to file FileUtils.writeStringToFile(new File(fileName), sourceCode, @@ -296,7 +297,8 @@ protected void createJavaSourceFileIfNotExists(String sourceCode, String fileNam * @return String **/ protected String buildJavaSourceCodeFileFullName(String publicClassName) { - return String.format(JavaConstants.JAVA_SOURCE_CODE_NAME_TEMPLATE, taskRequest.getExecutePath(), publicClassName); + return String.format(JavaConstants.JAVA_SOURCE_CODE_NAME_TEMPLATE, taskRequest.getExecutePath(), + publicClassName); } /** @@ -331,7 +333,7 @@ protected String buildResourcePath() { **/ protected String buildJavaCompileCommand(String sourceCode) throws IOException { String publicClassName = getPublicClassName(sourceCode); - String fileName = buildJavaSourceCodeFileFullName(publicClassName); + String fileName = buildJavaSourceCodeFileFullName(publicClassName); createJavaSourceFileIfNotExists(sourceCode, fileName); StringBuilder compilerCommand = new StringBuilder() diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-java/src/main/java/org/apache/dolphinscheduler/plugin/task/java/JavaTaskChannelFactory.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-java/src/main/java/org/apache/dolphinscheduler/plugin/task/java/JavaTaskChannelFactory.java index c374960d4546..e72f62e29eee 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-java/src/main/java/org/apache/dolphinscheduler/plugin/task/java/JavaTaskChannelFactory.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-java/src/main/java/org/apache/dolphinscheduler/plugin/task/java/JavaTaskChannelFactory.java @@ -27,6 +27,7 @@ @AutoService(TaskChannelFactory.class) public class JavaTaskChannelFactory implements TaskChannelFactory { + /** * Construct a channel for a Java task * diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-java/src/main/java/org/apache/dolphinscheduler/plugin/task/java/exception/JavaSourceFileExistException.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-java/src/main/java/org/apache/dolphinscheduler/plugin/task/java/exception/JavaSourceFileExistException.java index 421d77de6687..6599ff24cf51 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-java/src/main/java/org/apache/dolphinscheduler/plugin/task/java/exception/JavaSourceFileExistException.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-java/src/main/java/org/apache/dolphinscheduler/plugin/task/java/exception/JavaSourceFileExistException.java @@ -18,6 +18,7 @@ package org.apache.dolphinscheduler.plugin.task.java.exception; public class JavaSourceFileExistException extends RuntimeException { + public JavaSourceFileExistException() { } @@ -33,7 +34,8 @@ public JavaSourceFileExistException(Throwable cause) { super(cause); } - public JavaSourceFileExistException(String message, Throwable cause, boolean enableSuppression, boolean writableStackTrace) { + public JavaSourceFileExistException(String message, Throwable cause, boolean enableSuppression, + boolean writableStackTrace) { super(message, cause, enableSuppression, writableStackTrace); } } diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-java/src/main/java/org/apache/dolphinscheduler/plugin/task/java/exception/PublicClassNotFoundException.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-java/src/main/java/org/apache/dolphinscheduler/plugin/task/java/exception/PublicClassNotFoundException.java index 16b0070c97f1..103c4433780d 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-java/src/main/java/org/apache/dolphinscheduler/plugin/task/java/exception/PublicClassNotFoundException.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-java/src/main/java/org/apache/dolphinscheduler/plugin/task/java/exception/PublicClassNotFoundException.java @@ -18,6 +18,7 @@ package org.apache.dolphinscheduler.plugin.task.java.exception; public class PublicClassNotFoundException extends RuntimeException { + public PublicClassNotFoundException() { } @@ -33,7 +34,8 @@ public PublicClassNotFoundException(Throwable cause) { super(cause); } - public PublicClassNotFoundException(String message, Throwable cause, boolean enableSuppression, boolean writableStackTrace) { + public PublicClassNotFoundException(String message, Throwable cause, boolean enableSuppression, + boolean writableStackTrace) { super(message, cause, enableSuppression, writableStackTrace); } } diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-jupyter/pom.xml b/dolphinscheduler-task-plugin/dolphinscheduler-task-jupyter/pom.xml index c81ffc613dfb..8e23678a1367 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-jupyter/pom.xml +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-jupyter/pom.xml @@ -15,15 +15,14 @@ ~ See the License for the specific language governing permissions and ~ limitations under the License. --> - + 4.0.0 - dolphinscheduler-task-plugin org.apache.dolphinscheduler + dolphinscheduler-task-plugin dev-SNAPSHOT - 4.0.0 dolphinscheduler-task-jupyter jar diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-jupyter/src/main/java/org/apache/dolphinscheduler/plugin/task/jupyter/JupyterTaskChannelFactory.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-jupyter/src/main/java/org/apache/dolphinscheduler/plugin/task/jupyter/JupyterTaskChannelFactory.java index 7ecbb29ccabc..41bff2bf805d 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-jupyter/src/main/java/org/apache/dolphinscheduler/plugin/task/jupyter/JupyterTaskChannelFactory.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-jupyter/src/main/java/org/apache/dolphinscheduler/plugin/task/jupyter/JupyterTaskChannelFactory.java @@ -27,6 +27,7 @@ @AutoService(TaskChannelFactory.class) public class JupyterTaskChannelFactory implements TaskChannelFactory { + @Override public String getName() { return "JUPYTER"; diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-k8s/pom.xml b/dolphinscheduler-task-plugin/dolphinscheduler-task-k8s/pom.xml index b517bd6bfba2..a6b167a218b7 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-k8s/pom.xml +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-k8s/pom.xml @@ -15,13 +15,14 @@ ~ See the License for the specific language governing permissions and ~ limitations under the License. --> - + + 4.0.0 - dolphinscheduler-task-plugin org.apache.dolphinscheduler + dolphinscheduler-task-plugin dev-SNAPSHOT - 4.0.0 dolphinscheduler-task-k8s jar diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-k8s/src/main/java/org/apache/dolphinscheduler/plugin/task/k8s/K8sTaskChannel.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-k8s/src/main/java/org/apache/dolphinscheduler/plugin/task/k8s/K8sTaskChannel.java index a1a613b6af2b..e6d94fa7acda 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-k8s/src/main/java/org/apache/dolphinscheduler/plugin/task/k8s/K8sTaskChannel.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-k8s/src/main/java/org/apache/dolphinscheduler/plugin/task/k8s/K8sTaskChannel.java @@ -27,6 +27,7 @@ import org.apache.dolphinscheduler.spi.utils.JSONUtils; public class K8sTaskChannel implements TaskChannel { + @Override public void cancelApplication(boolean status) { diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-k8s/src/main/java/org/apache/dolphinscheduler/plugin/task/k8s/K8sTaskChannelFactory.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-k8s/src/main/java/org/apache/dolphinscheduler/plugin/task/k8s/K8sTaskChannelFactory.java index dd8caac035e1..4af560b5d295 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-k8s/src/main/java/org/apache/dolphinscheduler/plugin/task/k8s/K8sTaskChannelFactory.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-k8s/src/main/java/org/apache/dolphinscheduler/plugin/task/k8s/K8sTaskChannelFactory.java @@ -27,6 +27,7 @@ @AutoService(TaskChannelFactory.class) public class K8sTaskChannelFactory implements TaskChannelFactory { + @Override public String getName() { return "K8S"; diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-mlflow/pom.xml b/dolphinscheduler-task-plugin/dolphinscheduler-task-mlflow/pom.xml index 7e21ccab77ec..8e3132839070 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-mlflow/pom.xml +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-mlflow/pom.xml @@ -15,15 +15,14 @@ ~ See the License for the specific language governing permissions and ~ limitations under the License. --> - + 4.0.0 - dolphinscheduler-task-plugin org.apache.dolphinscheduler + dolphinscheduler-task-plugin dev-SNAPSHOT - 4.0.0 dolphinscheduler-task-mlflow jar diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-mlflow/src/main/java/org/apache/dolphinscheduler/plugin/task/mlflow/MlflowConstants.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-mlflow/src/main/java/org/apache/dolphinscheduler/plugin/task/mlflow/MlflowConstants.java index a13741230f29..c14fb3041c9b 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-mlflow/src/main/java/org/apache/dolphinscheduler/plugin/task/mlflow/MlflowConstants.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-mlflow/src/main/java/org/apache/dolphinscheduler/plugin/task/mlflow/MlflowConstants.java @@ -18,6 +18,7 @@ package org.apache.dolphinscheduler.plugin.task.mlflow; public class MlflowConstants { + private MlflowConstants() { throw new IllegalStateException("Utility class"); } @@ -59,23 +60,23 @@ private MlflowConstants() { public static final String SET_REPOSITORY = "repo=%s"; public static final String MLFLOW_RUN_BASIC_ALGORITHM = "mlflow run $repo " - + "-P algorithm=%s " - + "-P data_path=$data_path " - + "-P params=\"%s\" " - + "-P search_params=\"%s\" " - + "-P model_name=\"%s\" " - + "--experiment-name=\"%s\""; + + "-P algorithm=%s " + + "-P data_path=$data_path " + + "-P params=\"%s\" " + + "-P search_params=\"%s\" " + + "-P model_name=\"%s\" " + + "--experiment-name=\"%s\""; public static final String MLFLOW_RUN_AUTOML_PROJECT = "mlflow run $repo " - + "-P tool=%s " - + "-P data_path=$data_path " - + "-P params=\"%s\" " - + "-P model_name=\"%s\" " - + "--experiment-name=\"%s\""; + + "-P tool=%s " + + "-P data_path=$data_path " + + "-P params=\"%s\" " + + "-P model_name=\"%s\" " + + "--experiment-name=\"%s\""; public static final String MLFLOW_RUN_CUSTOM_PROJECT = "mlflow run $repo " - + "%s " - + "--experiment-name=\"%s\""; + + "%s " + + "--experiment-name=\"%s\""; public static final String MLFLOW_MODELS_SERVE = "mlflow models serve -m %s --port %s -h 0.0.0.0"; @@ -84,8 +85,8 @@ private MlflowConstants() { public static final String DOCKER_RREMOVE_CONTAINER = "docker rm -f %s"; public static final String DOCKER_RUN = "docker run -d --name=%s -p=%s:8080 " - + "--health-cmd \"curl --fail http://127.0.0.1:8080/ping || exit 1\" --health-interval 5s --health-retries 20" - + " %s"; + + "--health-cmd \"curl --fail http://127.0.0.1:8080/ping || exit 1\" --health-interval 5s --health-retries 20" + + " %s"; public static final String DOCKER_HEALTH_CHECK = "docker inspect --format \"{{json .State.Health.Status }}\" %s"; diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-mlflow/src/main/java/org/apache/dolphinscheduler/plugin/task/mlflow/MlflowParameters.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-mlflow/src/main/java/org/apache/dolphinscheduler/plugin/task/mlflow/MlflowParameters.java index 819cb27bd8ad..61400dc19da1 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-mlflow/src/main/java/org/apache/dolphinscheduler/plugin/task/mlflow/MlflowParameters.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-mlflow/src/main/java/org/apache/dolphinscheduler/plugin/task/mlflow/MlflowParameters.java @@ -17,13 +17,13 @@ package org.apache.dolphinscheduler.plugin.task.mlflow; -import lombok.Data; - import org.apache.dolphinscheduler.plugin.task.api.parameters.AbstractParameters; import org.apache.dolphinscheduler.spi.utils.StringUtils; import java.util.HashMap; +import lombok.Data; + @Data public class MlflowParameters extends AbstractParameters { @@ -47,7 +47,6 @@ public class MlflowParameters extends AbstractParameters { */ private String automlTool = "FLAML"; - /** * basic algorithm parameters */ @@ -93,7 +92,7 @@ public HashMap getParamsMap() { paramsMap.put("experiment_name", experimentName); paramsMap.put("model_name", modelName); paramsMap.put("MLFLOW_TRACKING_URI", mlflowTrackingUri); - switch (mlflowJobType){ + switch (mlflowJobType) { case MlflowConstants.JOB_TYPE_BASIC_ALGORITHM: addParamsMapForBasicAlgorithm(paramsMap); break; @@ -134,11 +133,11 @@ public String getModelKeyName(String tag) throws IllegalArgumentException { return imageName; } - public String getContainerName(){ + public String getContainerName() { return "ds-mlflow-" + getModelKeyName("-"); } - public boolean getIsDeployDocker(){ + public boolean getIsDeployDocker() { if (StringUtils.isEmpty(deployType)) { return false; } diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-mlflow/src/main/java/org/apache/dolphinscheduler/plugin/task/mlflow/MlflowTaskChannel.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-mlflow/src/main/java/org/apache/dolphinscheduler/plugin/task/mlflow/MlflowTaskChannel.java index fbed85e94cd9..250e8c3b8253 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-mlflow/src/main/java/org/apache/dolphinscheduler/plugin/task/mlflow/MlflowTaskChannel.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-mlflow/src/main/java/org/apache/dolphinscheduler/plugin/task/mlflow/MlflowTaskChannel.java @@ -24,7 +24,6 @@ import org.apache.dolphinscheduler.plugin.task.api.parameters.resource.ResourceParametersHelper; import org.apache.dolphinscheduler.spi.utils.JSONUtils; - public class MlflowTaskChannel implements TaskChannel { @Override diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-mlflow/src/main/java/org/apache/dolphinscheduler/plugin/task/mlflow/MlflowTaskChannelFactory.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-mlflow/src/main/java/org/apache/dolphinscheduler/plugin/task/mlflow/MlflowTaskChannelFactory.java index 5e58fb1915b3..dc18814d1d08 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-mlflow/src/main/java/org/apache/dolphinscheduler/plugin/task/mlflow/MlflowTaskChannelFactory.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-mlflow/src/main/java/org/apache/dolphinscheduler/plugin/task/mlflow/MlflowTaskChannelFactory.java @@ -32,6 +32,7 @@ @AutoService(TaskChannelFactory.class) public class MlflowTaskChannelFactory implements TaskChannelFactory { + @Override public TaskChannel create() { return new MlflowTaskChannel(); @@ -46,9 +47,12 @@ public String getName() { public List getParams() { List paramsList = new ArrayList<>(); - InputParam nodeName = InputParam.newBuilder("name", "$t('Node name')").addValidate(Validate.newBuilder().setRequired(true).build()).build(); + InputParam nodeName = InputParam.newBuilder("name", "$t('Node name')") + .addValidate(Validate.newBuilder().setRequired(true).build()).build(); - RadioParam runFlag = RadioParam.newBuilder("runFlag", "RUN_FLAG").addParamsOptions(new ParamsOptions("NORMAL", "NORMAL", false)).addParamsOptions(new ParamsOptions("FORBIDDEN", "FORBIDDEN", false)).build(); + RadioParam runFlag = RadioParam.newBuilder("runFlag", "RUN_FLAG") + .addParamsOptions(new ParamsOptions("NORMAL", "NORMAL", false)) + .addParamsOptions(new ParamsOptions("FORBIDDEN", "FORBIDDEN", false)).build(); paramsList.add(nodeName); paramsList.add(runFlag); diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-mr/pom.xml b/dolphinscheduler-task-plugin/dolphinscheduler-task-mr/pom.xml index 2e00765735b2..d83329e66ac8 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-mr/pom.xml +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-mr/pom.xml @@ -15,15 +15,14 @@ ~ See the License for the specific language governing permissions and ~ limitations under the License. --> - + 4.0.0 - dolphinscheduler-task-plugin org.apache.dolphinscheduler + dolphinscheduler-task-plugin dev-SNAPSHOT - 4.0.0 dolphinscheduler-task-mr jar diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-mr/src/main/java/org/apache/dolphinscheduler/plugin/task/mr/MapReduceParameters.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-mr/src/main/java/org/apache/dolphinscheduler/plugin/task/mr/MapReduceParameters.java index e6a67065f465..8a8296fa2651 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-mr/src/main/java/org/apache/dolphinscheduler/plugin/task/mr/MapReduceParameters.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-mr/src/main/java/org/apache/dolphinscheduler/plugin/task/mr/MapReduceParameters.java @@ -153,7 +153,6 @@ public String toString() { + "mainClass=" + mainClass + "mainArgs=" + mainArgs + "queue=" + queue - + "other mainArgs=" + others - ; + + "other mainArgs=" + others; } } diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-mr/src/main/java/org/apache/dolphinscheduler/plugin/task/mr/MapReduceTask.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-mr/src/main/java/org/apache/dolphinscheduler/plugin/task/mr/MapReduceTask.java index 0a519dc65fab..a37d2f9e34c0 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-mr/src/main/java/org/apache/dolphinscheduler/plugin/task/mr/MapReduceTask.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-mr/src/main/java/org/apache/dolphinscheduler/plugin/task/mr/MapReduceTask.java @@ -25,11 +25,9 @@ import org.apache.dolphinscheduler.plugin.task.api.parameters.AbstractParameters; import org.apache.dolphinscheduler.plugin.task.api.parser.ParamUtils; import org.apache.dolphinscheduler.plugin.task.api.parser.ParameterUtils; -import org.apache.dolphinscheduler.plugin.task.api.utils.MapUtils; import org.apache.dolphinscheduler.spi.utils.JSONUtils; import java.util.ArrayList; -import java.util.HashMap; import java.util.List; import java.util.Map; @@ -68,7 +66,8 @@ public void init() { logger.info("mapreduce task params {}", taskExecutionContext.getTaskParams()); - this.mapreduceParameters = JSONUtils.parseObject(taskExecutionContext.getTaskParams(), MapReduceParameters.class); + this.mapreduceParameters = + JSONUtils.parseObject(taskExecutionContext.getTaskParams(), MapReduceParameters.class); // check parameters if (mapreduceParameters == null || !mapreduceParameters.checkParameters()) { @@ -81,10 +80,13 @@ public void init() { // replace placeholder,and combine local and global parameters Map paramsMap = taskExecutionContext.getPrepareParamsMap(); - String args = ParameterUtils.convertParameterPlaceholders(mapreduceParameters.getMainArgs(), ParamUtils.convert(paramsMap)); + String args = ParameterUtils.convertParameterPlaceholders(mapreduceParameters.getMainArgs(), + ParamUtils.convert(paramsMap)); mapreduceParameters.setMainArgs(args); - if (mapreduceParameters.getProgramType() != null && mapreduceParameters.getProgramType() == ProgramType.PYTHON) { - String others = ParameterUtils.convertParameterPlaceholders(mapreduceParameters.getOthers(), ParamUtils.convert(paramsMap)); + if (mapreduceParameters.getProgramType() != null + && mapreduceParameters.getProgramType() == ProgramType.PYTHON) { + String others = ParameterUtils.convertParameterPlaceholders(mapreduceParameters.getOthers(), + ParamUtils.convert(paramsMap)); mapreduceParameters.setOthers(others); } } diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-mr/src/main/java/org/apache/dolphinscheduler/plugin/task/mr/MapReduceTaskChannel.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-mr/src/main/java/org/apache/dolphinscheduler/plugin/task/mr/MapReduceTaskChannel.java index 931adbf50907..efa03545e9b8 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-mr/src/main/java/org/apache/dolphinscheduler/plugin/task/mr/MapReduceTaskChannel.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-mr/src/main/java/org/apache/dolphinscheduler/plugin/task/mr/MapReduceTaskChannel.java @@ -26,6 +26,7 @@ import org.apache.dolphinscheduler.spi.utils.JSONUtils; public class MapReduceTaskChannel implements TaskChannel { + @Override public void cancelApplication(boolean status) { diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-openmldb/pom.xml b/dolphinscheduler-task-plugin/dolphinscheduler-task-openmldb/pom.xml index 065d99386f2c..6c5063ee283d 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-openmldb/pom.xml +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-openmldb/pom.xml @@ -15,15 +15,14 @@ ~ See the License for the specific language governing permissions and ~ limitations under the License. --> - + 4.0.0 - dolphinscheduler-task-plugin org.apache.dolphinscheduler + dolphinscheduler-task-plugin dev-SNAPSHOT - 4.0.0 dolphinscheduler-task-openmldb jar diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-openmldb/src/main/java/org/apache/dolphinscheduler/plugin/task/openmldb/OpenmldbTaskChannel.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-openmldb/src/main/java/org/apache/dolphinscheduler/plugin/task/openmldb/OpenmldbTaskChannel.java index 27a1e349d330..592a5dad326b 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-openmldb/src/main/java/org/apache/dolphinscheduler/plugin/task/openmldb/OpenmldbTaskChannel.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-openmldb/src/main/java/org/apache/dolphinscheduler/plugin/task/openmldb/OpenmldbTaskChannel.java @@ -25,6 +25,7 @@ import org.apache.dolphinscheduler.spi.utils.JSONUtils; public class OpenmldbTaskChannel implements TaskChannel { + @Override public void cancelApplication(boolean status) { diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-openmldb/src/main/java/org/apache/dolphinscheduler/plugin/task/openmldb/OpenmldbTaskChannelFactory.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-openmldb/src/main/java/org/apache/dolphinscheduler/plugin/task/openmldb/OpenmldbTaskChannelFactory.java index b4698c9461d2..32e3eaa04df9 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-openmldb/src/main/java/org/apache/dolphinscheduler/plugin/task/openmldb/OpenmldbTaskChannelFactory.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-openmldb/src/main/java/org/apache/dolphinscheduler/plugin/task/openmldb/OpenmldbTaskChannelFactory.java @@ -27,6 +27,7 @@ @AutoService(TaskChannelFactory.class) public class OpenmldbTaskChannelFactory implements TaskChannelFactory { + @Override public TaskChannel create() { return new OpenmldbTaskChannel(); diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-pigeon/src/main/java/org/apache/dolphinscheduler/plugin/task/pigeon/PigeonConfig.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-pigeon/src/main/java/org/apache/dolphinscheduler/plugin/task/pigeon/PigeonConfig.java index a8a70ce0a5a9..9345909ac903 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-pigeon/src/main/java/org/apache/dolphinscheduler/plugin/task/pigeon/PigeonConfig.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-pigeon/src/main/java/org/apache/dolphinscheduler/plugin/task/pigeon/PigeonConfig.java @@ -41,7 +41,8 @@ public static synchronized PigeonConfig getInstance() { } private PigeonConfig() { - ResourceBundle bundle = ResourceBundle.getBundle(PigeonConfig.class.getPackage().getName().replace(".", "/") + "/config"); + ResourceBundle bundle = + ResourceBundle.getBundle(PigeonConfig.class.getPackage().getName().replace(".", "/") + "/config"); this.jobTriggerUrl = bundle.getString("job.trigger.url"); this.jobStatusUrl = bundle.getString("job.status.url"); this.jobTriggerPostBody = bundle.getString("job.trigger.post.body"); diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-pigeon/src/main/java/org/apache/dolphinscheduler/plugin/task/pigeon/PigeonTaskChannel.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-pigeon/src/main/java/org/apache/dolphinscheduler/plugin/task/pigeon/PigeonTaskChannel.java index 72fff0277cea..998544124c35 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-pigeon/src/main/java/org/apache/dolphinscheduler/plugin/task/pigeon/PigeonTaskChannel.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-pigeon/src/main/java/org/apache/dolphinscheduler/plugin/task/pigeon/PigeonTaskChannel.java @@ -29,6 +29,7 @@ import org.slf4j.LoggerFactory; public class PigeonTaskChannel implements TaskChannel { + private static final Logger logger = LoggerFactory.getLogger(PigeonTaskChannel.class); @Override diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-pigeon/src/main/java/org/apache/dolphinscheduler/plugin/task/pigeon/PigeonTaskChannelFactory.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-pigeon/src/main/java/org/apache/dolphinscheduler/plugin/task/pigeon/PigeonTaskChannelFactory.java index e95f1330bbed..a7ce31692192 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-pigeon/src/main/java/org/apache/dolphinscheduler/plugin/task/pigeon/PigeonTaskChannelFactory.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-pigeon/src/main/java/org/apache/dolphinscheduler/plugin/task/pigeon/PigeonTaskChannelFactory.java @@ -43,10 +43,10 @@ public String getName() { @Override public List getParams() { - InputParam webHookParam - = InputParam.newBuilder(PigeonParamsConstants.NAME_TARGET_JOB_NAME, PigeonParamsConstants.TARGET_JOB_NAME) - .addValidate(Validate.newBuilder().setRequired(true).build()) - .build(); + InputParam webHookParam = + InputParam.newBuilder(PigeonParamsConstants.NAME_TARGET_JOB_NAME, PigeonParamsConstants.TARGET_JOB_NAME) + .addValidate(Validate.newBuilder().setRequired(true).build()) + .build(); return Arrays.asList(webHookParam); } } diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-procedure/pom.xml b/dolphinscheduler-task-plugin/dolphinscheduler-task-procedure/pom.xml index 586c47e3ec01..2f2e67397a31 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-procedure/pom.xml +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-procedure/pom.xml @@ -15,15 +15,14 @@ ~ See the License for the specific language governing permissions and ~ limitations under the License. --> - + 4.0.0 - dolphinscheduler-task-plugin org.apache.dolphinscheduler + dolphinscheduler-task-plugin dev-SNAPSHOT - 4.0.0 dolphinscheduler-task-procedure jar diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-procedure/src/main/java/org/apache/dolphinscheduler/plugin/task/procedure/ProcedureParameters.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-procedure/src/main/java/org/apache/dolphinscheduler/plugin/task/procedure/ProcedureParameters.java index b3a15a539e01..bdb3ff4a27a9 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-procedure/src/main/java/org/apache/dolphinscheduler/plugin/task/procedure/ProcedureParameters.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-procedure/src/main/java/org/apache/dolphinscheduler/plugin/task/procedure/ProcedureParameters.java @@ -135,9 +135,11 @@ public ResourceParametersHelper getResources() { } public ProcedureTaskExecutionContext generateExtendedContext(ResourceParametersHelper parametersHelper) { - DataSourceParameters dataSourceParameters = (DataSourceParameters) parametersHelper.getResourceParameters(ResourceType.DATASOURCE, datasource); + DataSourceParameters dataSourceParameters = + (DataSourceParameters) parametersHelper.getResourceParameters(ResourceType.DATASOURCE, datasource); ProcedureTaskExecutionContext procedureTaskExecutionContext = new ProcedureTaskExecutionContext(); - procedureTaskExecutionContext.setConnectionParams(Objects.nonNull(dataSourceParameters) ? dataSourceParameters.getConnectionParams() : null); + procedureTaskExecutionContext.setConnectionParams( + Objects.nonNull(dataSourceParameters) ? dataSourceParameters.getConnectionParams() : null); return procedureTaskExecutionContext; } } diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-python/pom.xml b/dolphinscheduler-task-plugin/dolphinscheduler-task-python/pom.xml index be3fc479e2d1..7145437bc7b1 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-python/pom.xml +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-python/pom.xml @@ -15,15 +15,14 @@ ~ See the License for the specific language governing permissions and ~ limitations under the License. --> - + 4.0.0 - dolphinscheduler-task-plugin org.apache.dolphinscheduler + dolphinscheduler-task-plugin dev-SNAPSHOT - 4.0.0 dolphinscheduler-task-python jar diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-python/src/main/java/org/apache/dolphinscheduler/plugin/task/python/PythonParameters.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-python/src/main/java/org/apache/dolphinscheduler/plugin/task/python/PythonParameters.java index 68e6c23e4b5e..d9fb27870d54 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-python/src/main/java/org/apache/dolphinscheduler/plugin/task/python/PythonParameters.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-python/src/main/java/org/apache/dolphinscheduler/plugin/task/python/PythonParameters.java @@ -23,6 +23,7 @@ import java.util.List; public class PythonParameters extends AbstractParameters { + /** * origin python script */ diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-python/src/main/java/org/apache/dolphinscheduler/plugin/task/python/PythonTaskChannel.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-python/src/main/java/org/apache/dolphinscheduler/plugin/task/python/PythonTaskChannel.java index 56dab9b2f29b..d4b0b0b5f462 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-python/src/main/java/org/apache/dolphinscheduler/plugin/task/python/PythonTaskChannel.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-python/src/main/java/org/apache/dolphinscheduler/plugin/task/python/PythonTaskChannel.java @@ -25,6 +25,7 @@ import org.apache.dolphinscheduler.spi.utils.JSONUtils; public class PythonTaskChannel implements TaskChannel { + @Override public void cancelApplication(boolean status) { diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-python/src/main/java/org/apache/dolphinscheduler/plugin/task/python/PythonTaskChannelFactory.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-python/src/main/java/org/apache/dolphinscheduler/plugin/task/python/PythonTaskChannelFactory.java index 00c44ee310e6..ae84f50bcda4 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-python/src/main/java/org/apache/dolphinscheduler/plugin/task/python/PythonTaskChannelFactory.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-python/src/main/java/org/apache/dolphinscheduler/plugin/task/python/PythonTaskChannelFactory.java @@ -1,19 +1,19 @@ package org.apache.dolphinscheduler.plugin.task.python;/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ import org.apache.dolphinscheduler.plugin.task.api.TaskChannel; import org.apache.dolphinscheduler.plugin.task.api.TaskChannelFactory; @@ -25,6 +25,7 @@ @AutoService(TaskChannelFactory.class) public class PythonTaskChannelFactory implements TaskChannelFactory { + @Override public TaskChannel create() { return new PythonTaskChannel(); diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-pytorch/pom.xml b/dolphinscheduler-task-plugin/dolphinscheduler-task-pytorch/pom.xml index 0a0107d0e34d..128c6f21990d 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-pytorch/pom.xml +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-pytorch/pom.xml @@ -15,15 +15,14 @@ ~ See the License for the specific language governing permissions and ~ limitations under the License. --> - + 4.0.0 - dolphinscheduler-task-plugin org.apache.dolphinscheduler + dolphinscheduler-task-plugin dev-SNAPSHOT - 4.0.0 dolphinscheduler-task-pytorch jar diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-pytorch/src/main/java/org/apache/dolphinscheduler/plugin/task/pytorch/GitProjectManager.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-pytorch/src/main/java/org/apache/dolphinscheduler/plugin/task/pytorch/GitProjectManager.java index d65e5e1150e4..bd11bb35464b 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-pytorch/src/main/java/org/apache/dolphinscheduler/plugin/task/pytorch/GitProjectManager.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-pytorch/src/main/java/org/apache/dolphinscheduler/plugin/task/pytorch/GitProjectManager.java @@ -26,16 +26,18 @@ import java.nio.file.Paths; import java.util.regex.Pattern; +import lombok.Data; + import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import lombok.Data; - @Data public class GitProjectManager { + public static final String GIT_PATH_LOCAL = "GIT_PROJECT"; private static final Pattern GIT_CHECK_PATTERN = Pattern.compile("^(git@|https?://)"); - protected final Logger logger = LoggerFactory.getLogger(String.format(TaskConstants.TASK_LOG_LOGGER_NAME_FORMAT, getClass())); + protected final Logger logger = + LoggerFactory.getLogger(String.format(TaskConstants.TASK_LOG_LOGGER_NAME_FORMAT, getClass())); private String path; private String baseDir = "."; diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-pytorch/src/main/java/org/apache/dolphinscheduler/plugin/task/pytorch/PytorchParameters.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-pytorch/src/main/java/org/apache/dolphinscheduler/plugin/task/pytorch/PytorchParameters.java index cbac9b548e5b..7fbc669ae63b 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-pytorch/src/main/java/org/apache/dolphinscheduler/plugin/task/pytorch/PytorchParameters.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-pytorch/src/main/java/org/apache/dolphinscheduler/plugin/task/pytorch/PytorchParameters.java @@ -54,7 +54,6 @@ public boolean checkParameters() { return true; } - public String getRequirementPath() { return getPossiblePath(requirements); } @@ -69,7 +68,6 @@ public String getPythonCommand() { return command; } - public String getScriptPath() { return getPossiblePath(script); } diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-pytorch/src/main/java/org/apache/dolphinscheduler/plugin/task/pytorch/PytorchTask.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-pytorch/src/main/java/org/apache/dolphinscheduler/plugin/task/pytorch/PytorchTask.java index c362097e3855..92aed842f62b 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-pytorch/src/main/java/org/apache/dolphinscheduler/plugin/task/pytorch/PytorchTask.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-pytorch/src/main/java/org/apache/dolphinscheduler/plugin/task/pytorch/PytorchTask.java @@ -46,8 +46,8 @@ public PytorchTask(TaskExecutionContext taskExecutionContext) { this.taskExecutionContext = taskExecutionContext; this.shellCommandExecutor = new ShellCommandExecutor(this::logHandle, - taskExecutionContext, - logger); + taskExecutionContext, + logger); } @Override @@ -89,7 +89,6 @@ public void cancel() throws TaskException { } - public String buildPythonExecuteCommand() throws Exception { List args = new ArrayList<>(); @@ -112,7 +111,8 @@ public String buildPythonExecuteCommand() throws Exception { String scriptParams = pytorchParameters.getScriptParams(); if (scriptParams != null && !scriptParams.isEmpty()) { - args.add(String.format("%s %s %s", getPythonCommand(), pytorchParameters.getScriptPath(), pytorchParameters.getScriptParams())); + args.add(String.format("%s %s %s", getPythonCommand(), pytorchParameters.getScriptPath(), + pytorchParameters.getScriptParams())); } else { args.add(String.format("%s %s", getPythonCommand(), pytorchParameters.getScriptPath())); @@ -133,10 +133,8 @@ private String getPythonCommand() { } - @Override public AbstractParameters getParameters() { return pytorchParameters; } } - diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-pytorch/src/main/java/org/apache/dolphinscheduler/plugin/task/pytorch/PytorchTaskChannelFactory.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-pytorch/src/main/java/org/apache/dolphinscheduler/plugin/task/pytorch/PytorchTaskChannelFactory.java index be7e3d65458b..bf1bda8b2dbd 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-pytorch/src/main/java/org/apache/dolphinscheduler/plugin/task/pytorch/PytorchTaskChannelFactory.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-pytorch/src/main/java/org/apache/dolphinscheduler/plugin/task/pytorch/PytorchTaskChannelFactory.java @@ -28,6 +28,7 @@ @AutoService(TaskChannelFactory.class) public class PytorchTaskChannelFactory implements TaskChannelFactory { + @Override public TaskChannel create() { return new PytorchTaskChannel(); diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-sagemaker/src/main/java/org/apache/dolphinscheduler/plugin/task/sagemaker/SagemakerConstants.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-sagemaker/src/main/java/org/apache/dolphinscheduler/plugin/task/sagemaker/SagemakerConstants.java index 143abbb64460..73339d8da323 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-sagemaker/src/main/java/org/apache/dolphinscheduler/plugin/task/sagemaker/SagemakerConstants.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-sagemaker/src/main/java/org/apache/dolphinscheduler/plugin/task/sagemaker/SagemakerConstants.java @@ -18,6 +18,7 @@ package org.apache.dolphinscheduler.plugin.task.sagemaker; public class SagemakerConstants { + public static final int CHECK_PIPELINE_EXECUTION_STATUS_INTERVAL = 5000; public static final int PIPELINE_MAX_RESULTS = 100; diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-sagemaker/src/main/java/org/apache/dolphinscheduler/plugin/task/sagemaker/SagemakerTaskChannelFactory.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-sagemaker/src/main/java/org/apache/dolphinscheduler/plugin/task/sagemaker/SagemakerTaskChannelFactory.java index 0c4afec462fd..ccad195e0ed6 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-sagemaker/src/main/java/org/apache/dolphinscheduler/plugin/task/sagemaker/SagemakerTaskChannelFactory.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-sagemaker/src/main/java/org/apache/dolphinscheduler/plugin/task/sagemaker/SagemakerTaskChannelFactory.java @@ -28,6 +28,7 @@ @AutoService(TaskChannelFactory.class) public class SagemakerTaskChannelFactory implements TaskChannelFactory { + @Override public TaskChannel create() { return new SagemakerTaskChannel(); diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-seatunnel/pom.xml b/dolphinscheduler-task-plugin/dolphinscheduler-task-seatunnel/pom.xml index 5a0da75c7d82..581635316846 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-seatunnel/pom.xml +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-seatunnel/pom.xml @@ -15,15 +15,14 @@ ~ See the License for the specific language governing permissions and ~ limitations under the License. --> - + 4.0.0 - dolphinscheduler-task-plugin org.apache.dolphinscheduler + dolphinscheduler-task-plugin dev-SNAPSHOT - 4.0.0 dolphinscheduler-task-seatunnel jar diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-seatunnel/src/main/java/org/apache/dolphinscheduler/plugin/task/seatunnel/DeployModeEnum.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-seatunnel/src/main/java/org/apache/dolphinscheduler/plugin/task/seatunnel/DeployModeEnum.java index 28fb7685ba72..019887735419 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-seatunnel/src/main/java/org/apache/dolphinscheduler/plugin/task/seatunnel/DeployModeEnum.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-seatunnel/src/main/java/org/apache/dolphinscheduler/plugin/task/seatunnel/DeployModeEnum.java @@ -18,6 +18,7 @@ package org.apache.dolphinscheduler.plugin.task.seatunnel; public enum DeployModeEnum { + cluster("cluster"), client("client"), local("client"); diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-seatunnel/src/main/java/org/apache/dolphinscheduler/plugin/task/seatunnel/SeatunnelParameters.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-seatunnel/src/main/java/org/apache/dolphinscheduler/plugin/task/seatunnel/SeatunnelParameters.java index 38591ba8a48b..6fa19fc0c601 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-seatunnel/src/main/java/org/apache/dolphinscheduler/plugin/task/seatunnel/SeatunnelParameters.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-seatunnel/src/main/java/org/apache/dolphinscheduler/plugin/task/seatunnel/SeatunnelParameters.java @@ -34,8 +34,8 @@ @JsonTypeInfo(use = JsonTypeInfo.Id.NAME, visible = true, property = "engine") @JsonSubTypes({ - @JsonSubTypes.Type(value = SeatunnelFlinkParameters.class, name = "FLINK"), - @JsonSubTypes.Type(value = SeatunnelSparkParameters.class, name = "SPARK") + @JsonSubTypes.Type(value = SeatunnelFlinkParameters.class, name = "FLINK"), + @JsonSubTypes.Type(value = SeatunnelSparkParameters.class, name = "SPARK") }) public class SeatunnelParameters extends AbstractParameters { @@ -86,7 +86,8 @@ public void setResourceList(List resourceList) { public boolean checkParameters() { return Objects.nonNull(engine) && ((BooleanUtils.isTrue(useCustom) && StringUtils.isNotBlank(rawScript)) - || (BooleanUtils.isFalse(useCustom) && CollectionUtils.isNotEmpty(resourceList) && resourceList.size() == 1)); + || (BooleanUtils.isFalse(useCustom) && CollectionUtils.isNotEmpty(resourceList) + && resourceList.size() == 1)); } @Override diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-seatunnel/src/main/java/org/apache/dolphinscheduler/plugin/task/seatunnel/SeatunnelTaskChannel.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-seatunnel/src/main/java/org/apache/dolphinscheduler/plugin/task/seatunnel/SeatunnelTaskChannel.java index b91c4f1228be..b4dbc649029e 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-seatunnel/src/main/java/org/apache/dolphinscheduler/plugin/task/seatunnel/SeatunnelTaskChannel.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-seatunnel/src/main/java/org/apache/dolphinscheduler/plugin/task/seatunnel/SeatunnelTaskChannel.java @@ -35,7 +35,8 @@ public void cancelApplication(boolean status) { @Override public SeatunnelTask createTask(TaskExecutionContext taskRequest) { - SeatunnelParameters seatunnelParameters = JSONUtils.parseObject(taskRequest.getTaskParams(), SeatunnelParameters.class); + SeatunnelParameters seatunnelParameters = + JSONUtils.parseObject(taskRequest.getTaskParams(), SeatunnelParameters.class); if (EngineEnum.FLINK == seatunnelParameters.getEngine()) { return new SeatunnelFlinkTask(taskRequest); } else if (EngineEnum.SPARK == seatunnelParameters.getEngine()) { diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-seatunnel/src/main/java/org/apache/dolphinscheduler/plugin/task/seatunnel/flink/SeatunnelFlinkParameters.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-seatunnel/src/main/java/org/apache/dolphinscheduler/plugin/task/seatunnel/flink/SeatunnelFlinkParameters.java index 3f0b0eee40e3..0527bdaad289 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-seatunnel/src/main/java/org/apache/dolphinscheduler/plugin/task/seatunnel/flink/SeatunnelFlinkParameters.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-seatunnel/src/main/java/org/apache/dolphinscheduler/plugin/task/seatunnel/flink/SeatunnelFlinkParameters.java @@ -26,6 +26,7 @@ public class SeatunnelFlinkParameters extends SeatunnelParameters { private String others; public static enum RunModeEnum { + RUN("--run-mode run"), RUN_APPLICATION("--run-mode run-application"); diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-seatunnel/src/main/java/org/apache/dolphinscheduler/plugin/task/seatunnel/flink/SeatunnelFlinkTask.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-seatunnel/src/main/java/org/apache/dolphinscheduler/plugin/task/seatunnel/flink/SeatunnelFlinkTask.java index efaa135334cc..07190375f9c8 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-seatunnel/src/main/java/org/apache/dolphinscheduler/plugin/task/seatunnel/flink/SeatunnelFlinkTask.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-seatunnel/src/main/java/org/apache/dolphinscheduler/plugin/task/seatunnel/flink/SeatunnelFlinkTask.java @@ -35,7 +35,8 @@ public SeatunnelFlinkTask(TaskExecutionContext taskExecutionContext) { @Override public void init() { - seatunnelParameters = JSONUtils.parseObject(taskExecutionContext.getTaskParams(), SeatunnelFlinkParameters.class); + seatunnelParameters = + JSONUtils.parseObject(taskExecutionContext.getTaskParams(), SeatunnelFlinkParameters.class); setSeatunnelParameters(seatunnelParameters); super.init(); } @@ -43,7 +44,9 @@ public void init() { @Override public List buildOptions() throws Exception { List args = super.buildOptions(); - args.add(Objects.isNull(seatunnelParameters.getRunMode()) ? SeatunnelFlinkParameters.RunModeEnum.RUN.getCommand() : seatunnelParameters.getRunMode().getCommand()); + args.add( + Objects.isNull(seatunnelParameters.getRunMode()) ? SeatunnelFlinkParameters.RunModeEnum.RUN.getCommand() + : seatunnelParameters.getRunMode().getCommand()); if (StringUtils.isNotBlank(seatunnelParameters.getOthers())) { args.add(seatunnelParameters.getOthers()); } diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-seatunnel/src/main/java/org/apache/dolphinscheduler/plugin/task/seatunnel/spark/SeatunnelSparkParameters.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-seatunnel/src/main/java/org/apache/dolphinscheduler/plugin/task/seatunnel/spark/SeatunnelSparkParameters.java index e17675a8458b..79c543a74dd7 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-seatunnel/src/main/java/org/apache/dolphinscheduler/plugin/task/seatunnel/spark/SeatunnelSparkParameters.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-seatunnel/src/main/java/org/apache/dolphinscheduler/plugin/task/seatunnel/spark/SeatunnelSparkParameters.java @@ -43,6 +43,7 @@ public boolean checkParameters() { } public static enum MasterTypeEnum { + YARN("yarn"), LOCAL("local"), SPARK("spark://"), diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-seatunnel/src/main/java/org/apache/dolphinscheduler/plugin/task/seatunnel/spark/SeatunnelSparkTask.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-seatunnel/src/main/java/org/apache/dolphinscheduler/plugin/task/seatunnel/spark/SeatunnelSparkTask.java index 9fcb5a424a40..e6f4b4417aed 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-seatunnel/src/main/java/org/apache/dolphinscheduler/plugin/task/seatunnel/spark/SeatunnelSparkTask.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-seatunnel/src/main/java/org/apache/dolphinscheduler/plugin/task/seatunnel/spark/SeatunnelSparkTask.java @@ -19,7 +19,6 @@ import static org.apache.dolphinscheduler.plugin.task.seatunnel.Constants.DEPLOY_MODE_OPTIONS; import static org.apache.dolphinscheduler.plugin.task.seatunnel.Constants.MASTER_OPTIONS; -import static org.apache.dolphinscheduler.plugin.task.seatunnel.Constants.QUEUE_OPTIONS; import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext; import org.apache.dolphinscheduler.plugin.task.seatunnel.DeployModeEnum; @@ -27,8 +26,6 @@ import org.apache.dolphinscheduler.plugin.task.seatunnel.spark.SeatunnelSparkParameters.MasterTypeEnum; import org.apache.dolphinscheduler.spi.utils.JSONUtils; -import org.apache.commons.lang3.StringUtils; - import java.util.List; public class SeatunnelSparkTask extends SeatunnelTask { @@ -45,7 +42,8 @@ public SeatunnelSparkTask(TaskExecutionContext taskExecutionContext) { @Override public void init() { - seatunnelParameters = JSONUtils.parseObject(taskExecutionContext.getTaskParams(), SeatunnelSparkParameters.class); + seatunnelParameters = + JSONUtils.parseObject(taskExecutionContext.getTaskParams(), SeatunnelSparkParameters.class); setSeatunnelParameters(seatunnelParameters); super.init(); } @@ -56,7 +54,8 @@ public List buildOptions() throws Exception { args.add(DEPLOY_MODE_OPTIONS); args.add(seatunnelParameters.getDeployMode().getCommand()); - MasterTypeEnum master = DeployModeEnum.local == seatunnelParameters.getDeployMode() ? MasterTypeEnum.LOCAL : seatunnelParameters.getMaster(); + MasterTypeEnum master = DeployModeEnum.local == seatunnelParameters.getDeployMode() ? MasterTypeEnum.LOCAL + : seatunnelParameters.getMaster(); args.add(MASTER_OPTIONS); args.add(master.getCommand()); diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-shell/pom.xml b/dolphinscheduler-task-plugin/dolphinscheduler-task-shell/pom.xml index e7ddd6dd19bd..73fc5066f09a 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-shell/pom.xml +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-shell/pom.xml @@ -15,15 +15,14 @@ ~ See the License for the specific language governing permissions and ~ limitations under the License. --> - + 4.0.0 - dolphinscheduler-task-plugin org.apache.dolphinscheduler + dolphinscheduler-task-plugin dev-SNAPSHOT - 4.0.0 dolphinscheduler-task-shell jar diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-shell/src/main/java/org/apache/dolphinscheduler/plugin/task/shell/ShellTask.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-shell/src/main/java/org/apache/dolphinscheduler/plugin/task/shell/ShellTask.java index 7c7c4b24974c..6e83c5c268f7 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-shell/src/main/java/org/apache/dolphinscheduler/plugin/task/shell/ShellTask.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-shell/src/main/java/org/apache/dolphinscheduler/plugin/task/shell/ShellTask.java @@ -17,12 +17,12 @@ package org.apache.dolphinscheduler.plugin.task.shell; -import org.apache.commons.lang3.SystemUtils; +import static org.apache.dolphinscheduler.plugin.task.api.TaskConstants.EXIT_CODE_FAILURE; +import static org.apache.dolphinscheduler.plugin.task.api.TaskConstants.RWXR_XR_X; import org.apache.dolphinscheduler.plugin.task.api.AbstractTask; import org.apache.dolphinscheduler.plugin.task.api.ShellCommandExecutor; import org.apache.dolphinscheduler.plugin.task.api.TaskCallBack; -import org.apache.dolphinscheduler.plugin.task.api.TaskConstants; import org.apache.dolphinscheduler.plugin.task.api.TaskException; import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext; import org.apache.dolphinscheduler.plugin.task.api.model.Property; @@ -32,6 +32,8 @@ import org.apache.dolphinscheduler.plugin.task.api.parser.ParameterUtils; import org.apache.dolphinscheduler.spi.utils.JSONUtils; +import org.apache.commons.lang3.SystemUtils; + import java.io.File; import java.nio.file.FileAlreadyExistsException; import java.nio.file.Files; @@ -43,9 +45,6 @@ import java.util.Map; import java.util.Set; -import static org.apache.dolphinscheduler.plugin.task.api.TaskConstants.EXIT_CODE_FAILURE; -import static org.apache.dolphinscheduler.plugin.task.api.TaskConstants.RWXR_XR_X; - /** * shell task */ diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-shell/src/main/java/org/apache/dolphinscheduler/plugin/task/shell/ShellTaskChannelFactory.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-shell/src/main/java/org/apache/dolphinscheduler/plugin/task/shell/ShellTaskChannelFactory.java index c960a0d5c16a..801826ffd183 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-shell/src/main/java/org/apache/dolphinscheduler/plugin/task/shell/ShellTaskChannelFactory.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-shell/src/main/java/org/apache/dolphinscheduler/plugin/task/shell/ShellTaskChannelFactory.java @@ -32,6 +32,7 @@ @AutoService(TaskChannelFactory.class) public class ShellTaskChannelFactory implements TaskChannelFactory { + @Override public TaskChannel create() { return new ShellTaskChannel(); diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-spark/pom.xml b/dolphinscheduler-task-plugin/dolphinscheduler-task-spark/pom.xml index 45c2923c28d8..6d0d6d79b178 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-spark/pom.xml +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-spark/pom.xml @@ -15,15 +15,14 @@ ~ See the License for the specific language governing permissions and ~ limitations under the License. --> - + 4.0.0 - dolphinscheduler-task-plugin org.apache.dolphinscheduler + dolphinscheduler-task-plugin dev-SNAPSHOT - 4.0.0 dolphinscheduler-task-spark jar diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-spark/src/main/java/org/apache/dolphinscheduler/plugin/task/spark/SparkTaskChannelFactory.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-spark/src/main/java/org/apache/dolphinscheduler/plugin/task/spark/SparkTaskChannelFactory.java index 0d307bde55c5..92db56ad2813 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-spark/src/main/java/org/apache/dolphinscheduler/plugin/task/spark/SparkTaskChannelFactory.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-spark/src/main/java/org/apache/dolphinscheduler/plugin/task/spark/SparkTaskChannelFactory.java @@ -27,6 +27,7 @@ @AutoService(TaskChannelFactory.class) public class SparkTaskChannelFactory implements TaskChannelFactory { + @Override public String getName() { return "SPARK"; diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-sql/pom.xml b/dolphinscheduler-task-plugin/dolphinscheduler-task-sql/pom.xml index 4896aecc4fb2..501c6be998d5 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-sql/pom.xml +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-sql/pom.xml @@ -15,15 +15,14 @@ ~ See the License for the specific language governing permissions and ~ limitations under the License. --> - + 4.0.0 - dolphinscheduler-task-plugin org.apache.dolphinscheduler + dolphinscheduler-task-plugin dev-SNAPSHOT - 4.0.0 dolphinscheduler-task-sql jar diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-sql/src/main/java/org/apache/dolphinscheduler/plugin/task/sql/SqlBinds.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-sql/src/main/java/org/apache/dolphinscheduler/plugin/task/sql/SqlBinds.java index 5f2741d952b8..3b61a53626f1 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-sql/src/main/java/org/apache/dolphinscheduler/plugin/task/sql/SqlBinds.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-sql/src/main/java/org/apache/dolphinscheduler/plugin/task/sql/SqlBinds.java @@ -25,6 +25,7 @@ * Used to contains both prepared sql string and its to-be-bind parameters */ public class SqlBinds { + private final String sql; private final Map paramsMap; diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-sql/src/main/java/org/apache/dolphinscheduler/plugin/task/sql/SqlTask.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-sql/src/main/java/org/apache/dolphinscheduler/plugin/task/sql/SqlTask.java index 2322fc27ff1f..ed6527d4cefe 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-sql/src/main/java/org/apache/dolphinscheduler/plugin/task/sql/SqlTask.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-sql/src/main/java/org/apache/dolphinscheduler/plugin/task/sql/SqlTask.java @@ -20,7 +20,12 @@ import org.apache.dolphinscheduler.plugin.datasource.api.plugin.DataSourceClientProvider; import org.apache.dolphinscheduler.plugin.datasource.api.utils.CommonUtils; import org.apache.dolphinscheduler.plugin.datasource.api.utils.DataSourceUtils; -import org.apache.dolphinscheduler.plugin.task.api.*; +import org.apache.dolphinscheduler.plugin.task.api.AbstractTask; +import org.apache.dolphinscheduler.plugin.task.api.SQLTaskExecutionContext; +import org.apache.dolphinscheduler.plugin.task.api.TaskCallBack; +import org.apache.dolphinscheduler.plugin.task.api.TaskConstants; +import org.apache.dolphinscheduler.plugin.task.api.TaskException; +import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext; import org.apache.dolphinscheduler.plugin.task.api.enums.Direct; import org.apache.dolphinscheduler.plugin.task.api.enums.SqlType; import org.apache.dolphinscheduler.plugin.task.api.enums.TaskTimeoutStrategy; @@ -82,7 +87,8 @@ public class SqlTask extends AbstractTask { * create function format * include replace here which can be compatible with more cases, for example a long-running Spark session in Kyuubi will keep its own temp functions instead of destroying them right away */ - private static final String CREATE_OR_REPLACE_FUNCTION_FORMAT = "create or replace temporary function {0} as ''{1}''"; + private static final String CREATE_OR_REPLACE_FUNCTION_FORMAT = + "create or replace temporary function {0} as ''{1}''"; /** * default query sql limit @@ -113,7 +119,8 @@ public SqlTask(TaskExecutionContext taskRequest) { throw new RuntimeException("sql task params is not valid"); } - sqlTaskExecutionContext = sqlParameters.generateExtendedContext(taskExecutionContext.getResourceParametersHelper()); + sqlTaskExecutionContext = + sqlParameters.generateExtendedContext(taskExecutionContext.getResourceParametersHelper()); } @Override @@ -124,7 +131,8 @@ public AbstractParameters getParameters() { @Override public void handle(TaskCallBack taskCallBack) throws TaskException { logger.info("Full sql parameters: {}", sqlParameters); - logger.info("sql type : {}, datasource : {}, sql : {} , localParams : {},udfs : {},showType : {},connParams : {},varPool : {} ,query max result limit {}", + logger.info( + "sql type : {}, datasource : {}, sql : {} , localParams : {},udfs : {},showType : {},connParams : {},varPool : {} ,query max result limit {}", sqlParameters.getType(), sqlParameters.getDatasource(), sqlParameters.getSql(), @@ -221,7 +229,8 @@ public void executeFuncAndSql(List mainStatementsBinds, try { // create connection - connection = DataSourceClientProvider.getInstance().getConnection(DbType.valueOf(sqlParameters.getType()), baseConnectionParam); + connection = DataSourceClientProvider.getInstance().getConnection(DbType.valueOf(sqlParameters.getType()), + baseConnectionParam); // create temp function if (CollectionUtils.isNotEmpty(createFuncs)) { createTempFunction(connection, createFuncs); @@ -241,7 +250,7 @@ public void executeFuncAndSql(List mainStatementsBinds, String updateResult = executeUpdate(connection, mainStatementsBinds, "main"); result = setNonQuerySqlReturn(updateResult, sqlParameters.getLocalParams()); } - //deal out params + // deal out params sqlParameters.dealOutParam(result); // post execute @@ -297,7 +306,8 @@ private String resultProcess(ResultSet resultSet) throws Exception { rowCount++; } - int displayRows = sqlParameters.getDisplayRows() > 0 ? sqlParameters.getDisplayRows() : TaskConstants.DEFAULT_DISPLAY_ROWS; + int displayRows = sqlParameters.getDisplayRows() > 0 ? sqlParameters.getDisplayRows() + : TaskConstants.DEFAULT_DISPLAY_ROWS; displayRows = Math.min(displayRows, rowCount); logger.info("display sql result {} rows as follows:", displayRows); for (int i = 0; i < displayRows; i++) { @@ -306,8 +316,8 @@ private String resultProcess(ResultSet resultSet) throws Exception { } } - String result = resultJSONArray.isEmpty() ? - JSONUtils.toJsonString(generateEmptyRow(resultSet)) : JSONUtils.toJsonString(resultJSONArray); + String result = resultJSONArray.isEmpty() ? JSONUtils.toJsonString(generateEmptyRow(resultSet)) + : JSONUtils.toJsonString(resultJSONArray); if (sqlParameters.getSendEmail() == null || sqlParameters.getSendEmail()) { sendAttachment(sqlParameters.getGroupId(), StringUtils.isNotEmpty(sqlParameters.getTitle()) @@ -338,7 +348,6 @@ private ArrayNode generateEmptyRow(ResultSet resultSet) throws SQLException { return resultJSONArray; } - /** * send alert as an attachment * @@ -362,12 +371,14 @@ private String executeQuery(Connection connection, SqlBinds sqlBinds, String han } } - private String executeUpdate(Connection connection, List statementsBinds, String handlerType) throws Exception { + private String executeUpdate(Connection connection, List statementsBinds, + String handlerType) throws Exception { int result = 0; for (SqlBinds sqlBind : statementsBinds) { try (PreparedStatement statement = prepareStatementAndBind(connection, sqlBind)) { result = statement.executeUpdate(); - logger.info("{} statement execute update result: {}, for sql: {}", handlerType, result, sqlBind.getSql()); + logger.info("{} statement execute update result: {}, for sql: {}", handlerType, result, + sqlBind.getSql()); } } return String.valueOf(result); @@ -428,7 +439,8 @@ private PreparedStatement prepareStatementAndBind(Connection connection, SqlBind ParameterUtils.setInParameter(entry.getKey(), stmt, prop.getType(), prop.getValue()); } } - logger.info("prepare statement replace sql : {}, sql parameters : {}", sqlBinds.getSql(), sqlBinds.getParamsMap()); + logger.info("prepare statement replace sql : {}, sql parameters : {}", sqlBinds.getSql(), + sqlBinds.getParamsMap()); return stmt; } catch (Exception exception) { throw new TaskException("SQL task prepareStatementAndBind error", exception); @@ -444,14 +456,15 @@ private PreparedStatement prepareStatementAndBind(Connection connection, SqlBind * @param sqlParamsMap sql params map */ private void printReplacedSql(String content, String formatSql, String rgex, Map sqlParamsMap) { - //parameter print style + // parameter print style logger.info("after replace sql , preparing : {}", formatSql); StringBuilder logPrint = new StringBuilder("replaced sql , parameters:"); if (sqlParamsMap == null) { logger.info("printReplacedSql: sqlParamsMap is null."); } else { for (int i = 1; i <= sqlParamsMap.size(); i++) { - logPrint.append(sqlParamsMap.get(i).getValue()).append("(").append(sqlParamsMap.get(i).getType()).append(")"); + logPrint.append(sqlParamsMap.get(i).getValue()).append("(").append(sqlParamsMap.get(i).getType()) + .append(")"); } } logger.info("Sql Params are {}", logPrint); @@ -482,12 +495,13 @@ private SqlBinds getSqlAndSqlParamsMap(String sql) { sqlParameters.setTitle(title); } - //new - //replace variable TIME with $[YYYYmmddd...] in sql when history run job and batch complement job - sql = ParameterUtils.replaceScheduleTime(sql, DateUtils.timeStampToDate(taskExecutionContext.getScheduleTime())); + // new + // replace variable TIME with $[YYYYmmddd...] in sql when history run job and batch complement job + sql = ParameterUtils.replaceScheduleTime(sql, + DateUtils.timeStampToDate(taskExecutionContext.getScheduleTime())); // special characters need to be escaped, ${} needs to be escaped - setSqlParamsMap(sql, rgex, sqlParamsMap, paramsMap,taskExecutionContext.getTaskInstanceId()); - //Replace the original value in sql ！{...} ，Does not participate in precompilation + setSqlParamsMap(sql, rgex, sqlParamsMap, paramsMap, taskExecutionContext.getTaskInstanceId()); + // Replace the original value in sql ！{...} ，Does not participate in precompilation String rgexo = "['\"]*\\!\\{(.*?)\\}['\"]*"; sql = replaceOriginalValue(sql, rgexo, paramsMap); // replace the ${} of the SQL statement with the Placeholder @@ -543,7 +557,8 @@ private List createFuncs(List udfFuncParameters, Logg */ private List buildTempFuncSql(List udfFuncParameters) { return udfFuncParameters.stream().map(value -> MessageFormat - .format(CREATE_OR_REPLACE_FUNCTION_FORMAT, value.getFuncName(), value.getClassName())).collect(Collectors.toList()); + .format(CREATE_OR_REPLACE_FUNCTION_FORMAT, value.getFuncName(), value.getClassName())) + .collect(Collectors.toList()); } /** @@ -557,7 +572,8 @@ private List buildJarSql(List udfFuncParameters) { String prefixPath = defaultFS.startsWith("file://") ? "file://" : defaultFS; String uploadPath = CommonUtils.getHdfsUdfDir(value.getTenantCode()); String resourceFullName = value.getResourceName(); - resourceFullName = resourceFullName.startsWith("/") ? resourceFullName : String.format("/%s", resourceFullName); + resourceFullName = + resourceFullName.startsWith("/") ? resourceFullName : String.format("/%s", resourceFullName); return String.format("add jar %s%s%s", prefixPath, uploadPath, resourceFullName); }).collect(Collectors.toList()); } diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-sql/src/main/java/org/apache/dolphinscheduler/plugin/task/sql/SqlTaskChannel.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-sql/src/main/java/org/apache/dolphinscheduler/plugin/task/sql/SqlTaskChannel.java index 390ef863fbd7..4d9b64076cff 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-sql/src/main/java/org/apache/dolphinscheduler/plugin/task/sql/SqlTaskChannel.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-sql/src/main/java/org/apache/dolphinscheduler/plugin/task/sql/SqlTaskChannel.java @@ -27,6 +27,7 @@ import org.apache.dolphinscheduler.spi.utils.JSONUtils; public class SqlTaskChannel implements TaskChannel { + @Override public void cancelApplication(boolean status) { diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-sql/src/main/java/org/apache/dolphinscheduler/plugin/task/sql/SqlTaskChannelFactory.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-sql/src/main/java/org/apache/dolphinscheduler/plugin/task/sql/SqlTaskChannelFactory.java index 7389dcad0b96..909741edfce1 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-sql/src/main/java/org/apache/dolphinscheduler/plugin/task/sql/SqlTaskChannelFactory.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-sql/src/main/java/org/apache/dolphinscheduler/plugin/task/sql/SqlTaskChannelFactory.java @@ -27,6 +27,7 @@ @AutoService(TaskChannelFactory.class) public class SqlTaskChannelFactory implements TaskChannelFactory { + @Override public String getName() { return "SQL"; diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-sqoop/pom.xml b/dolphinscheduler-task-plugin/dolphinscheduler-task-sqoop/pom.xml index d285977a446f..c44508e7745f 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-sqoop/pom.xml +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-sqoop/pom.xml @@ -15,15 +15,14 @@ ~ See the License for the specific language governing permissions and ~ limitations under the License. --> - + 4.0.0 - dolphinscheduler-task-plugin org.apache.dolphinscheduler + dolphinscheduler-task-plugin dev-SNAPSHOT - 4.0.0 dolphinscheduler-task-sqoop jar diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-sqoop/src/main/java/org/apache/dolphinscheduler/plugin/task/sqoop/SqoopConstants.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-sqoop/src/main/java/org/apache/dolphinscheduler/plugin/task/sqoop/SqoopConstants.java index 002f06bf9c81..f4861eb4fb37 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-sqoop/src/main/java/org/apache/dolphinscheduler/plugin/task/sqoop/SqoopConstants.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-sqoop/src/main/java/org/apache/dolphinscheduler/plugin/task/sqoop/SqoopConstants.java @@ -22,7 +22,7 @@ public final class SqoopConstants { private SqoopConstants() { } - //sqoop general param + // sqoop general param public static final String SQOOP = "sqoop"; public static final String SQOOP_MR_JOB_NAME = "mapred.job.name"; public static final String SQOOP_PARALLELISM = "-m"; @@ -32,7 +32,7 @@ private SqoopConstants() { public static final String SPLIT_BY = "--split-by"; - //sqoop db + // sqoop db public static final String DB_CONNECT = "--connect"; public static final String DB_USERNAME = "--username"; public static final String DB_PWD = "--password"; @@ -45,19 +45,18 @@ private SqoopConstants() { public static final String MAP_COLUMN_HIVE = "--map-column-hive"; public static final String MAP_COLUMN_JAVA = "--map-column-java"; - - //sqoop hive source + // sqoop hive source public static final String HCATALOG_DATABASE = "--hcatalog-database"; public static final String HCATALOG_TABLE = "--hcatalog-table"; public static final String HCATALOG_PARTITION_KEYS = "--hcatalog-partition-keys"; public static final String HCATALOG_PARTITION_VALUES = "--hcatalog-partition-values"; - //sqoop hdfs + // sqoop hdfs public static final String HDFS_EXPORT_DIR = "--export-dir"; public static final String TARGET_DIR = "--target-dir"; public static final String COMPRESSION_CODEC = "--compression-codec"; - //sqoop hive + // sqoop hive public static final String HIVE_IMPORT = "--hive-import"; public static final String HIVE_DATABASE = "--hive-database"; public static final String HIVE_TABLE = "--hive-table"; @@ -69,9 +68,8 @@ private SqoopConstants() { public static final String HIVE_PARTITION_KEY = "--hive-partition-key"; public static final String HIVE_PARTITION_VALUE = "--hive-partition-value"; - //sqoop update model + // sqoop update model public static final String UPDATE_KEY = "--update-key"; public static final String UPDATE_MODE = "--update-mode"; - } diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-sqoop/src/main/java/org/apache/dolphinscheduler/plugin/task/sqoop/SqoopJobType.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-sqoop/src/main/java/org/apache/dolphinscheduler/plugin/task/sqoop/SqoopJobType.java index 407390ef2198..1b5dfc60749b 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-sqoop/src/main/java/org/apache/dolphinscheduler/plugin/task/sqoop/SqoopJobType.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-sqoop/src/main/java/org/apache/dolphinscheduler/plugin/task/sqoop/SqoopJobType.java @@ -17,7 +17,7 @@ package org.apache.dolphinscheduler.plugin.task.sqoop; -public enum SqoopJobType { +public enum SqoopJobType { CUSTOM(0, "CUSTOM"), TEMPLATE(1, "TEMPLATE"); diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-sqoop/src/main/java/org/apache/dolphinscheduler/plugin/task/sqoop/SqoopTask.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-sqoop/src/main/java/org/apache/dolphinscheduler/plugin/task/sqoop/SqoopTask.java index fe4c42857646..f61256797d66 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-sqoop/src/main/java/org/apache/dolphinscheduler/plugin/task/sqoop/SqoopTask.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-sqoop/src/main/java/org/apache/dolphinscheduler/plugin/task/sqoop/SqoopTask.java @@ -23,12 +23,10 @@ import org.apache.dolphinscheduler.plugin.task.api.parameters.AbstractParameters; import org.apache.dolphinscheduler.plugin.task.api.parser.ParamUtils; import org.apache.dolphinscheduler.plugin.task.api.parser.ParameterUtils; -import org.apache.dolphinscheduler.plugin.task.api.utils.MapUtils; import org.apache.dolphinscheduler.plugin.task.sqoop.generator.SqoopJobGenerator; import org.apache.dolphinscheduler.plugin.task.sqoop.parameter.SqoopParameters; import org.apache.dolphinscheduler.spi.utils.JSONUtils; -import java.util.HashMap; import java.util.Map; /** @@ -57,8 +55,8 @@ public SqoopTask(TaskExecutionContext taskExecutionContext) { public void init() { logger.info("sqoop task params {}", taskExecutionContext.getTaskParams()); sqoopParameters = - JSONUtils.parseObject(taskExecutionContext.getTaskParams(), SqoopParameters.class); - //check sqoop task params + JSONUtils.parseObject(taskExecutionContext.getTaskParams(), SqoopParameters.class); + // check sqoop task params if (null == sqoopParameters) { throw new IllegalArgumentException("Sqoop Task params is null"); } @@ -67,12 +65,13 @@ public void init() { throw new IllegalArgumentException("Sqoop Task params check fail"); } - sqoopTaskExecutionContext = sqoopParameters.generateExtendedContext(taskExecutionContext.getResourceParametersHelper()); + sqoopTaskExecutionContext = + sqoopParameters.generateExtendedContext(taskExecutionContext.getResourceParametersHelper()); } @Override protected String buildCommand() { - //get sqoop scripts + // get sqoop scripts SqoopJobGenerator generator = new SqoopJobGenerator(); String script = generator.generateSqoopJob(sqoopParameters, sqoopTaskExecutionContext); diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-sqoop/src/main/java/org/apache/dolphinscheduler/plugin/task/sqoop/generator/CommonGenerator.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-sqoop/src/main/java/org/apache/dolphinscheduler/plugin/task/sqoop/generator/CommonGenerator.java index 5cae89e7a40c..0426906361e3 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-sqoop/src/main/java/org/apache/dolphinscheduler/plugin/task/sqoop/generator/CommonGenerator.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-sqoop/src/main/java/org/apache/dolphinscheduler/plugin/task/sqoop/generator/CommonGenerator.java @@ -44,44 +44,44 @@ public String generate(SqoopParameters sqoopParameters) { StringBuilder commonSb = new StringBuilder(); try { - //sqoop task model + // sqoop task model commonSb.append(SqoopConstants.SQOOP) - .append(SPACE) - .append(sqoopParameters.getModelType()); + .append(SPACE) + .append(sqoopParameters.getModelType()); - //sqoop map-reduce job name + // sqoop map-reduce job name commonSb.append(SPACE).append(D).append(SPACE) - .append(String.format("%s%s%s", SqoopConstants.SQOOP_MR_JOB_NAME, - EQUAL_SIGN, sqoopParameters.getJobName())); + .append(String.format("%s%s%s", SqoopConstants.SQOOP_MR_JOB_NAME, + EQUAL_SIGN, sqoopParameters.getJobName())); - //hadoop custom param + // hadoop custom param List hadoopCustomParams = sqoopParameters.getHadoopCustomParams(); if (CollectionUtils.isNotEmpty(hadoopCustomParams)) { for (Property hadoopCustomParam : hadoopCustomParams) { String hadoopCustomParamStr = String.format("%s%s%s", hadoopCustomParam.getProp(), - EQUAL_SIGN, hadoopCustomParam.getValue()); + EQUAL_SIGN, hadoopCustomParam.getValue()); commonSb.append(SPACE).append(D) - .append(SPACE).append(hadoopCustomParamStr); + .append(SPACE).append(hadoopCustomParamStr); } } - //sqoop custom params + // sqoop custom params List sqoopAdvancedParams = sqoopParameters.getSqoopAdvancedParams(); if (CollectionUtils.isNotEmpty(sqoopAdvancedParams)) { for (Property sqoopAdvancedParam : sqoopAdvancedParams) { commonSb.append(SPACE).append(sqoopAdvancedParam.getProp()) - .append(SPACE).append(sqoopAdvancedParam.getValue()); + .append(SPACE).append(sqoopAdvancedParam.getValue()); } } - //sqoop parallelism + // sqoop parallelism if (sqoopParameters.getConcurrency() > 0) { commonSb.append(SPACE).append(SqoopConstants.SQOOP_PARALLELISM) - .append(SPACE).append(sqoopParameters.getConcurrency()); + .append(SPACE).append(sqoopParameters.getConcurrency()); if (sqoopParameters.getConcurrency() > 1) { commonSb.append(SPACE).append(SqoopConstants.SPLIT_BY) - .append(SPACE).append(sqoopParameters.getSplitBy()); + .append(SPACE).append(sqoopParameters.getSplitBy()); } } } catch (Exception e) { diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-sqoop/src/main/java/org/apache/dolphinscheduler/plugin/task/sqoop/generator/SqoopJobGenerator.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-sqoop/src/main/java/org/apache/dolphinscheduler/plugin/task/sqoop/generator/SqoopJobGenerator.java index 67bd60880a57..48d152f70d29 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-sqoop/src/main/java/org/apache/dolphinscheduler/plugin/task/sqoop/generator/SqoopJobGenerator.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-sqoop/src/main/java/org/apache/dolphinscheduler/plugin/task/sqoop/generator/SqoopJobGenerator.java @@ -65,7 +65,8 @@ private void createSqoopJobGenerator(String sourceType, String targetType) { * @param sqoopTaskExecutionContext * @return sqoop scripts */ - public String generateSqoopJob(SqoopParameters sqoopParameters, SqoopTaskExecutionContext sqoopTaskExecutionContext) { + public String generateSqoopJob(SqoopParameters sqoopParameters, + SqoopTaskExecutionContext sqoopTaskExecutionContext) { String sqoopScripts = ""; @@ -76,8 +77,8 @@ public String generateSqoopJob(SqoopParameters sqoopParameters, SqoopTaskExecuti } sqoopScripts = String.format("%s%s%s", commonGenerator.generate(sqoopParameters), - sourceGenerator.generate(sqoopParameters, sqoopTaskExecutionContext), - targetGenerator.generate(sqoopParameters, sqoopTaskExecutionContext)); + sourceGenerator.generate(sqoopParameters, sqoopTaskExecutionContext), + targetGenerator.generate(sqoopParameters, sqoopTaskExecutionContext)); } else if (SqoopJobType.CUSTOM.getDescp().equals(sqoopParameters.getJobType())) { sqoopScripts = sqoopParameters.getCustomShell().replaceAll("\\r\\n", "\n"); } diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-sqoop/src/main/java/org/apache/dolphinscheduler/plugin/task/sqoop/generator/sources/HdfsSourceGenerator.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-sqoop/src/main/java/org/apache/dolphinscheduler/plugin/task/sqoop/generator/sources/HdfsSourceGenerator.java index 652aff530e1f..d21ab912bedb 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-sqoop/src/main/java/org/apache/dolphinscheduler/plugin/task/sqoop/generator/sources/HdfsSourceGenerator.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-sqoop/src/main/java/org/apache/dolphinscheduler/plugin/task/sqoop/generator/sources/HdfsSourceGenerator.java @@ -43,13 +43,13 @@ public String generate(SqoopParameters sqoopParameters, SqoopTaskExecutionContex StringBuilder hdfsSourceSb = new StringBuilder(); try { - SourceHdfsParameter sourceHdfsParameter - = JSONUtils.parseObject(sqoopParameters.getSourceParams(), SourceHdfsParameter.class); + SourceHdfsParameter sourceHdfsParameter = + JSONUtils.parseObject(sqoopParameters.getSourceParams(), SourceHdfsParameter.class); if (null != sourceHdfsParameter) { if (StringUtils.isNotEmpty(sourceHdfsParameter.getExportDir())) { hdfsSourceSb.append(SPACE).append(HDFS_EXPORT_DIR) - .append(SPACE).append(sourceHdfsParameter.getExportDir()); + .append(SPACE).append(sourceHdfsParameter.getExportDir()); } else { throw new IllegalArgumentException("Sqoop hdfs export dir is null"); } diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-sqoop/src/main/java/org/apache/dolphinscheduler/plugin/task/sqoop/generator/sources/HiveSourceGenerator.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-sqoop/src/main/java/org/apache/dolphinscheduler/plugin/task/sqoop/generator/sources/HiveSourceGenerator.java index de1c9597b2ba..f9a2881b5f1c 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-sqoop/src/main/java/org/apache/dolphinscheduler/plugin/task/sqoop/generator/sources/HiveSourceGenerator.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-sqoop/src/main/java/org/apache/dolphinscheduler/plugin/task/sqoop/generator/sources/HiveSourceGenerator.java @@ -46,26 +46,26 @@ public String generate(SqoopParameters sqoopParameters, SqoopTaskExecutionContex StringBuilder hiveSourceSb = new StringBuilder(); try { - SourceHiveParameter sourceHiveParameter - = JSONUtils.parseObject(sqoopParameters.getSourceParams(), SourceHiveParameter.class); + SourceHiveParameter sourceHiveParameter = + JSONUtils.parseObject(sqoopParameters.getSourceParams(), SourceHiveParameter.class); if (null != sourceHiveParameter) { if (StringUtils.isNotEmpty(sourceHiveParameter.getHiveDatabase())) { hiveSourceSb.append(SPACE).append(HCATALOG_DATABASE) - .append(SPACE).append(sourceHiveParameter.getHiveDatabase()); + .append(SPACE).append(sourceHiveParameter.getHiveDatabase()); } if (StringUtils.isNotEmpty(sourceHiveParameter.getHiveTable())) { hiveSourceSb.append(SPACE).append(HCATALOG_TABLE) - .append(SPACE).append(sourceHiveParameter.getHiveTable()); + .append(SPACE).append(sourceHiveParameter.getHiveTable()); } if (StringUtils.isNotEmpty(sourceHiveParameter.getHivePartitionKey()) - && StringUtils.isNotEmpty(sourceHiveParameter.getHivePartitionValue())) { + && StringUtils.isNotEmpty(sourceHiveParameter.getHivePartitionValue())) { hiveSourceSb.append(SPACE).append(HCATALOG_PARTITION_KEYS) - .append(SPACE).append(sourceHiveParameter.getHivePartitionKey()) - .append(SPACE).append(HCATALOG_PARTITION_VALUES) - .append(SPACE).append(sourceHiveParameter.getHivePartitionValue()); + .append(SPACE).append(sourceHiveParameter.getHivePartitionKey()) + .append(SPACE).append(HCATALOG_PARTITION_VALUES) + .append(SPACE).append(sourceHiveParameter.getHivePartitionValue()); } } } catch (Exception e) { diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-sqoop/src/main/java/org/apache/dolphinscheduler/plugin/task/sqoop/generator/sources/MySQLSourceGenerator.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-sqoop/src/main/java/org/apache/dolphinscheduler/plugin/task/sqoop/generator/sources/MySQLSourceGenerator.java index 8e8ab4cdc9fc..e434f7360b78 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-sqoop/src/main/java/org/apache/dolphinscheduler/plugin/task/sqoop/generator/sources/MySQLSourceGenerator.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-sqoop/src/main/java/org/apache/dolphinscheduler/plugin/task/sqoop/generator/sources/MySQLSourceGenerator.java @@ -64,7 +64,8 @@ public String generate(SqoopParameters sqoopParameters, SqoopTaskExecutionContex StringBuilder mysqlSourceSb = new StringBuilder(); try { - SourceMysqlParameter sourceMysqlParameter = JSONUtils.parseObject(sqoopParameters.getSourceParams(), SourceMysqlParameter.class); + SourceMysqlParameter sourceMysqlParameter = + JSONUtils.parseObject(sqoopParameters.getSourceParams(), SourceMysqlParameter.class); if (null != sourceMysqlParameter) { BaseConnectionParam baseDataSource = (BaseConnectionParam) DataSourceUtils.buildConnectionParams( @@ -76,29 +77,29 @@ public String generate(SqoopParameters sqoopParameters, SqoopTaskExecutionContex mysqlSourceSb.append(SPACE).append(DB_CONNECT) .append(SPACE).append(DOUBLE_QUOTES) .append(DataSourceUtils.getJdbcUrl(DbType.MYSQL, baseDataSource)).append(DOUBLE_QUOTES) - .append(SPACE).append(DB_USERNAME) - .append(SPACE).append(baseDataSource.getUser()) - .append(SPACE).append(DB_PWD) - .append(SPACE).append(DOUBLE_QUOTES) + .append(SPACE).append(DB_USERNAME) + .append(SPACE).append(baseDataSource.getUser()) + .append(SPACE).append(DB_PWD) + .append(SPACE).append(DOUBLE_QUOTES) .append(decodePassword(baseDataSource.getPassword())).append(DOUBLE_QUOTES); - //sqoop table & sql query + // sqoop table & sql query if (sourceMysqlParameter.getSrcQueryType() == SqoopQueryType.FORM.getCode()) { if (StringUtils.isNotEmpty(sourceMysqlParameter.getSrcTable())) { mysqlSourceSb.append(SPACE).append(TABLE) - .append(SPACE).append(sourceMysqlParameter.getSrcTable()); + .append(SPACE).append(sourceMysqlParameter.getSrcTable()); } if (StringUtils.isNotEmpty(sourceMysqlParameter.getSrcColumns())) { mysqlSourceSb.append(SPACE).append(COLUMNS) - .append(SPACE).append(sourceMysqlParameter.getSrcColumns()); + .append(SPACE).append(sourceMysqlParameter.getSrcColumns()); } } else if (sourceMysqlParameter.getSrcQueryType() == SqoopQueryType.SQL.getCode() - && StringUtils.isNotEmpty(sourceMysqlParameter.getSrcQuerySql())) { + && StringUtils.isNotEmpty(sourceMysqlParameter.getSrcQuerySql())) { String srcQuery = sourceMysqlParameter.getSrcQuerySql(); mysqlSourceSb.append(SPACE).append(QUERY) - .append(SPACE).append(DOUBLE_QUOTES).append(srcQuery); + .append(SPACE).append(DOUBLE_QUOTES).append(srcQuery); if (srcQuery.toLowerCase().contains(QUERY_WHERE)) { mysqlSourceSb.append(SPACE).append(QUERY_CONDITION).append(DOUBLE_QUOTES); @@ -107,7 +108,7 @@ public String generate(SqoopParameters sqoopParameters, SqoopTaskExecutionContex } } - //sqoop hive map column + // sqoop hive map column List mapColumnHive = sourceMysqlParameter.getMapColumnHive(); if (null != mapColumnHive && !mapColumnHive.isEmpty()) { @@ -118,11 +119,11 @@ public String generate(SqoopParameters sqoopParameters, SqoopTaskExecutionContex if (StringUtils.isNotEmpty(columnMap.toString())) { mysqlSourceSb.append(SPACE).append(MAP_COLUMN_HIVE) - .append(SPACE).append(columnMap.substring(0, columnMap.length() - 1)); + .append(SPACE).append(columnMap.substring(0, columnMap.length() - 1)); } } - //sqoop map column java + // sqoop map column java List mapColumnJava = sourceMysqlParameter.getMapColumnJava(); if (null != mapColumnJava && !mapColumnJava.isEmpty()) { @@ -133,7 +134,7 @@ public String generate(SqoopParameters sqoopParameters, SqoopTaskExecutionContex if (StringUtils.isNotEmpty(columnMap.toString())) { mysqlSourceSb.append(SPACE).append(MAP_COLUMN_JAVA) - .append(SPACE).append(columnMap.substring(0, columnMap.length() - 1)); + .append(SPACE).append(columnMap.substring(0, columnMap.length() - 1)); } } } diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-sqoop/src/main/java/org/apache/dolphinscheduler/plugin/task/sqoop/generator/targets/HdfsTargetGenerator.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-sqoop/src/main/java/org/apache/dolphinscheduler/plugin/task/sqoop/generator/targets/HdfsTargetGenerator.java index 4a619ca1f135..d2c1b2ed4021 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-sqoop/src/main/java/org/apache/dolphinscheduler/plugin/task/sqoop/generator/targets/HdfsTargetGenerator.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-sqoop/src/main/java/org/apache/dolphinscheduler/plugin/task/sqoop/generator/targets/HdfsTargetGenerator.java @@ -50,18 +50,18 @@ public String generate(SqoopParameters sqoopParameters, SqoopTaskExecutionContex try { TargetHdfsParameter targetHdfsParameter = - JSONUtils.parseObject(sqoopParameters.getTargetParams(), TargetHdfsParameter.class); + JSONUtils.parseObject(sqoopParameters.getTargetParams(), TargetHdfsParameter.class); if (null != targetHdfsParameter) { if (StringUtils.isNotEmpty(targetHdfsParameter.getTargetPath())) { hdfsTargetSb.append(SPACE).append(TARGET_DIR) - .append(SPACE).append(targetHdfsParameter.getTargetPath()); + .append(SPACE).append(targetHdfsParameter.getTargetPath()); } if (StringUtils.isNotEmpty(targetHdfsParameter.getCompressionCodec())) { hdfsTargetSb.append(SPACE).append(COMPRESSION_CODEC) - .append(SPACE).append(targetHdfsParameter.getCompressionCodec()); + .append(SPACE).append(targetHdfsParameter.getCompressionCodec()); } if (StringUtils.isNotEmpty(targetHdfsParameter.getFileType())) { @@ -74,12 +74,14 @@ public String generate(SqoopParameters sqoopParameters, SqoopTaskExecutionContex if (StringUtils.isNotEmpty(targetHdfsParameter.getFieldsTerminated())) { hdfsTargetSb.append(SPACE).append(FIELDS_TERMINATED_BY) - .append(SPACE).append(SINGLE_QUOTES).append(targetHdfsParameter.getFieldsTerminated()).append(SINGLE_QUOTES); + .append(SPACE).append(SINGLE_QUOTES).append(targetHdfsParameter.getFieldsTerminated()) + .append(SINGLE_QUOTES); } if (StringUtils.isNotEmpty(targetHdfsParameter.getLinesTerminated())) { hdfsTargetSb.append(SPACE).append(LINES_TERMINATED_BY) - .append(SPACE).append(SINGLE_QUOTES).append(targetHdfsParameter.getLinesTerminated()).append(SINGLE_QUOTES); + .append(SPACE).append(SINGLE_QUOTES).append(targetHdfsParameter.getLinesTerminated()) + .append(SINGLE_QUOTES); } hdfsTargetSb.append(SPACE).append(FIELD_NULL_PLACEHOLDER); diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-sqoop/src/main/java/org/apache/dolphinscheduler/plugin/task/sqoop/generator/targets/HiveTargetGenerator.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-sqoop/src/main/java/org/apache/dolphinscheduler/plugin/task/sqoop/generator/targets/HiveTargetGenerator.java index 0fba0fed4036..c7b5071276aa 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-sqoop/src/main/java/org/apache/dolphinscheduler/plugin/task/sqoop/generator/targets/HiveTargetGenerator.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-sqoop/src/main/java/org/apache/dolphinscheduler/plugin/task/sqoop/generator/targets/HiveTargetGenerator.java @@ -54,16 +54,16 @@ public String generate(SqoopParameters sqoopParameters, SqoopTaskExecutionContex try { TargetHiveParameter targetHiveParameter = - JSONUtils.parseObject(sqoopParameters.getTargetParams(), TargetHiveParameter.class); + JSONUtils.parseObject(sqoopParameters.getTargetParams(), TargetHiveParameter.class); if (null != targetHiveParameter) { hiveTargetSb.append(SPACE).append(HIVE_IMPORT); if (StringUtils.isNotEmpty(targetHiveParameter.getHiveDatabase()) - && StringUtils.isNotEmpty(targetHiveParameter.getHiveTable())) { + && StringUtils.isNotEmpty(targetHiveParameter.getHiveTable())) { hiveTargetSb.append(SPACE).append(HIVE_DATABASE) - .append(SPACE).append(targetHiveParameter.getHiveDatabase()) - .append(SPACE).append(HIVE_TABLE) - .append(SPACE).append(targetHiveParameter.getHiveTable()); + .append(SPACE).append(targetHiveParameter.getHiveDatabase()) + .append(SPACE).append(HIVE_TABLE) + .append(SPACE).append(targetHiveParameter.getHiveTable()); } if (targetHiveParameter.isCreateHiveTable()) { @@ -76,20 +76,20 @@ public String generate(SqoopParameters sqoopParameters, SqoopTaskExecutionContex if (targetHiveParameter.isHiveOverWrite()) { hiveTargetSb.append(SPACE).append(HIVE_OVERWRITE) - .append(SPACE).append(DELETE_TARGET_DIR); + .append(SPACE).append(DELETE_TARGET_DIR); } if (StringUtils.isNotEmpty(targetHiveParameter.getReplaceDelimiter())) { hiveTargetSb.append(SPACE).append(HIVE_DELIMS_REPLACEMENT) - .append(SPACE).append(targetHiveParameter.getReplaceDelimiter()); + .append(SPACE).append(targetHiveParameter.getReplaceDelimiter()); } if (StringUtils.isNotEmpty(targetHiveParameter.getHivePartitionKey()) - && StringUtils.isNotEmpty(targetHiveParameter.getHivePartitionValue())) { + && StringUtils.isNotEmpty(targetHiveParameter.getHivePartitionValue())) { hiveTargetSb.append(SPACE).append(HIVE_PARTITION_KEY) - .append(SPACE).append(targetHiveParameter.getHivePartitionKey()) - .append(SPACE).append(HIVE_PARTITION_VALUE) - .append(SPACE).append(targetHiveParameter.getHivePartitionValue()); + .append(SPACE).append(targetHiveParameter.getHivePartitionKey()) + .append(SPACE).append(HIVE_PARTITION_VALUE) + .append(SPACE).append(targetHiveParameter.getHivePartitionValue()); } if (StringUtils.isNotEmpty(targetHiveParameter.getHiveTargetDir())) { diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-sqoop/src/main/java/org/apache/dolphinscheduler/plugin/task/sqoop/generator/targets/MySQLTargetGenerator.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-sqoop/src/main/java/org/apache/dolphinscheduler/plugin/task/sqoop/generator/targets/MySQLTargetGenerator.java index e8f7e85a5f42..94f912d97e40 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-sqoop/src/main/java/org/apache/dolphinscheduler/plugin/task/sqoop/generator/targets/MySQLTargetGenerator.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-sqoop/src/main/java/org/apache/dolphinscheduler/plugin/task/sqoop/generator/targets/MySQLTargetGenerator.java @@ -58,7 +58,7 @@ public String generate(SqoopParameters sqoopParameters, SqoopTaskExecutionContex try { TargetMysqlParameter targetMysqlParameter = - JSONUtils.parseObject(sqoopParameters.getTargetParams(), TargetMysqlParameter.class); + JSONUtils.parseObject(sqoopParameters.getTargetParams(), TargetMysqlParameter.class); if (null != targetMysqlParameter && targetMysqlParameter.getTargetDatasource() != 0) { @@ -77,12 +77,12 @@ public String generate(SqoopParameters sqoopParameters, SqoopTaskExecutionContex .append(SPACE).append(DB_PWD) .append(SPACE).append(DOUBLE_QUOTES) .append(decodePassword(baseDataSource.getPassword())).append(DOUBLE_QUOTES) - .append(SPACE).append(TABLE) - .append(SPACE).append(targetMysqlParameter.getTargetTable()); + .append(SPACE).append(TABLE) + .append(SPACE).append(targetMysqlParameter.getTargetTable()); if (StringUtils.isNotEmpty(targetMysqlParameter.getTargetColumns())) { mysqlTargetSb.append(SPACE).append(COLUMNS) - .append(SPACE).append(targetMysqlParameter.getTargetColumns()); + .append(SPACE).append(targetMysqlParameter.getTargetColumns()); } if (StringUtils.isNotEmpty(targetMysqlParameter.getFieldsTerminated())) { @@ -91,7 +91,8 @@ public String generate(SqoopParameters sqoopParameters, SqoopTaskExecutionContex mysqlTargetSb.append(SPACE).append(targetMysqlParameter.getFieldsTerminated()); } else { - mysqlTargetSb.append(SPACE).append(SINGLE_QUOTES).append(targetMysqlParameter.getFieldsTerminated()).append(SINGLE_QUOTES); + mysqlTargetSb.append(SPACE).append(SINGLE_QUOTES) + .append(targetMysqlParameter.getFieldsTerminated()).append(SINGLE_QUOTES); } } @@ -100,17 +101,18 @@ public String generate(SqoopParameters sqoopParameters, SqoopTaskExecutionContex if (targetMysqlParameter.getLinesTerminated().contains(SINGLE_QUOTES)) { mysqlTargetSb.append(SPACE).append(targetMysqlParameter.getLinesTerminated()); } else { - mysqlTargetSb.append(SPACE).append(SINGLE_QUOTES).append(targetMysqlParameter.getLinesTerminated()).append(SINGLE_QUOTES); + mysqlTargetSb.append(SPACE).append(SINGLE_QUOTES) + .append(targetMysqlParameter.getLinesTerminated()).append(SINGLE_QUOTES); } } if (targetMysqlParameter.getIsUpdate() - && StringUtils.isNotEmpty(targetMysqlParameter.getTargetUpdateKey()) - && StringUtils.isNotEmpty(targetMysqlParameter.getTargetUpdateMode())) { + && StringUtils.isNotEmpty(targetMysqlParameter.getTargetUpdateKey()) + && StringUtils.isNotEmpty(targetMysqlParameter.getTargetUpdateMode())) { mysqlTargetSb.append(SPACE).append(UPDATE_KEY) - .append(SPACE).append(targetMysqlParameter.getTargetUpdateKey()) - .append(SPACE).append(UPDATE_MODE) - .append(SPACE).append(targetMysqlParameter.getTargetUpdateMode()); + .append(SPACE).append(targetMysqlParameter.getTargetUpdateKey()) + .append(SPACE).append(UPDATE_MODE) + .append(SPACE).append(targetMysqlParameter.getTargetUpdateMode()); } } } diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-sqoop/src/main/java/org/apache/dolphinscheduler/plugin/task/sqoop/parameter/SqoopParameters.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-sqoop/src/main/java/org/apache/dolphinscheduler/plugin/task/sqoop/parameter/SqoopParameters.java index 3d88b288f0bd..d5d88506b542 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-sqoop/src/main/java/org/apache/dolphinscheduler/plugin/task/sqoop/parameter/SqoopParameters.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-sqoop/src/main/java/org/apache/dolphinscheduler/plugin/task/sqoop/parameter/SqoopParameters.java @@ -200,16 +200,16 @@ public boolean checkParameters() { if (SqoopJobType.TEMPLATE.getDescp().equals(jobType)) { sqoopParamsCheck = StringUtils.isEmpty(customShell) - && StringUtils.isNotEmpty(modelType) - && StringUtils.isNotEmpty(jobName) - && concurrency != 0 - && StringUtils.isNotEmpty(sourceType) - && StringUtils.isNotEmpty(targetType) - && StringUtils.isNotEmpty(sourceParams) - && StringUtils.isNotEmpty(targetParams); + && StringUtils.isNotEmpty(modelType) + && StringUtils.isNotEmpty(jobName) + && concurrency != 0 + && StringUtils.isNotEmpty(sourceType) + && StringUtils.isNotEmpty(targetType) + && StringUtils.isNotEmpty(sourceParams) + && StringUtils.isNotEmpty(targetParams); } else if (SqoopJobType.CUSTOM.getDescp().equals(jobType)) { sqoopParamsCheck = StringUtils.isNotEmpty(customShell) - && StringUtils.isEmpty(jobName); + && StringUtils.isEmpty(jobName); } return sqoopParamsCheck; @@ -222,12 +222,14 @@ public ResourceParametersHelper getResources() { return resources; } - SourceMysqlParameter sourceMysqlParameter = JSONUtils.parseObject(this.getSourceParams(), SourceMysqlParameter.class); + SourceMysqlParameter sourceMysqlParameter = + JSONUtils.parseObject(this.getSourceParams(), SourceMysqlParameter.class); if (sourceMysqlParameter.getSrcDatasource() != 0) { resources.put(ResourceType.DATASOURCE, sourceMysqlParameter.getSrcDatasource()); } - TargetMysqlParameter targetMysqlParameter = JSONUtils.parseObject(this.getTargetParams(), TargetMysqlParameter.class); + TargetMysqlParameter targetMysqlParameter = + JSONUtils.parseObject(this.getTargetParams(), TargetMysqlParameter.class); if (targetMysqlParameter.getTargetDatasource() != 0) { resources.put(ResourceType.DATASOURCE, targetMysqlParameter.getTargetDatasource()); } @@ -242,11 +244,15 @@ public SqoopTaskExecutionContext generateExtendedContext(ResourceParametersHelpe return sqoopTaskExecutionContext; } - SourceMysqlParameter sourceMysqlParameter = JSONUtils.parseObject(this.getSourceParams(), SourceMysqlParameter.class); - TargetMysqlParameter targetMysqlParameter = JSONUtils.parseObject(this.getTargetParams(), TargetMysqlParameter.class); + SourceMysqlParameter sourceMysqlParameter = + JSONUtils.parseObject(this.getSourceParams(), SourceMysqlParameter.class); + TargetMysqlParameter targetMysqlParameter = + JSONUtils.parseObject(this.getTargetParams(), TargetMysqlParameter.class); - DataSourceParameters dataSource = (DataSourceParameters) parametersHelper.getResourceParameters(ResourceType.DATASOURCE, sourceMysqlParameter.getSrcDatasource()); - DataSourceParameters dataTarget = (DataSourceParameters) parametersHelper.getResourceParameters(ResourceType.DATASOURCE, targetMysqlParameter.getTargetDatasource()); + DataSourceParameters dataSource = (DataSourceParameters) parametersHelper + .getResourceParameters(ResourceType.DATASOURCE, sourceMysqlParameter.getSrcDatasource()); + DataSourceParameters dataTarget = (DataSourceParameters) parametersHelper + .getResourceParameters(ResourceType.DATASOURCE, targetMysqlParameter.getTargetDatasource()); if (Objects.nonNull(dataSource)) { sqoopTaskExecutionContext.setDataSourceId(sourceMysqlParameter.getSrcDatasource()); diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-sqoop/src/test/java/org/apache/dolphinscheduler/plugin/task/sqoop/EntityTestUtils.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-sqoop/src/test/java/org/apache/dolphinscheduler/plugin/task/sqoop/EntityTestUtils.java index 37388dbf1d1c..e1d10d1b7dfc 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-sqoop/src/test/java/org/apache/dolphinscheduler/plugin/task/sqoop/EntityTestUtils.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-sqoop/src/test/java/org/apache/dolphinscheduler/plugin/task/sqoop/EntityTestUtils.java @@ -31,12 +31,13 @@ public class EntityTestUtils { private static final Map OBJECT_MAP = new HashMap<>(); - + private static final String SKIP_METHOD = "getClass,notify,notifyAll,wait,equals,hashCode,clone"; static { OBJECT_MAP.put("java.lang.Long", 1L); - OBJECT_MAP.put("java.lang.String", "[{\"direct\":\"OUT\",\"prop\":\"percentage5\",\"type\":\"VARCHAR\",\"value\":\"qwe\"}]"); + OBJECT_MAP.put("java.lang.String", + "[{\"direct\":\"OUT\",\"prop\":\"percentage5\",\"type\":\"VARCHAR\",\"value\":\"qwe\"}]"); OBJECT_MAP.put("java.lang.Integer", 1); OBJECT_MAP.put("int", 1); OBJECT_MAP.put("long", 1L); @@ -45,9 +46,8 @@ public class EntityTestUtils { OBJECT_MAP.put("java.util.Map", new HashMap()); OBJECT_MAP.put("boolean", true); } - - public static void run(List classList) - throws IllegalAccessException, InvocationTargetException, InstantiationException { + + public static void run(List classList) throws IllegalAccessException, InvocationTargetException, InstantiationException { for (Class temp : classList) { Object tempInstance = new Object(); Constructor[] constructors = temp.getConstructors(); @@ -82,4 +82,4 @@ public static void run(List classList) } } } -} \ No newline at end of file +} diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-subprocess/pom.xml b/dolphinscheduler-task-plugin/dolphinscheduler-task-subprocess/pom.xml index f723207cade4..383e1d94233c 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-subprocess/pom.xml +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-subprocess/pom.xml @@ -15,15 +15,14 @@ ~ See the License for the specific language governing permissions and ~ limitations under the License. --> - + 4.0.0 - dolphinscheduler-task-plugin org.apache.dolphinscheduler + dolphinscheduler-task-plugin dev-SNAPSHOT - 4.0.0 dolphinscheduler-task-subprocess jar @@ -41,4 +40,4 @@ - \ No newline at end of file + diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-subprocess/src/main/java/org/apache/dolphinscheduler/plugin/task/subprocess/SubProcessTaskChannelFactory.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-subprocess/src/main/java/org/apache/dolphinscheduler/plugin/task/subprocess/SubProcessTaskChannelFactory.java index 18f356b944dc..c440e265d329 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-subprocess/src/main/java/org/apache/dolphinscheduler/plugin/task/subprocess/SubProcessTaskChannelFactory.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-subprocess/src/main/java/org/apache/dolphinscheduler/plugin/task/subprocess/SubProcessTaskChannelFactory.java @@ -29,6 +29,7 @@ @AutoService(TaskChannelFactory.class) public class SubProcessTaskChannelFactory implements TaskChannelFactory { + @Override public TaskChannel create() { return new SubProcessTaskChannel(); diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-switch/pom.xml b/dolphinscheduler-task-plugin/dolphinscheduler-task-switch/pom.xml index 41159491f75d..a8a9892b59f2 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-switch/pom.xml +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-switch/pom.xml @@ -15,15 +15,14 @@ ~ See the License for the specific language governing permissions and ~ limitations under the License. --> - + 4.0.0 - dolphinscheduler-task-plugin org.apache.dolphinscheduler + dolphinscheduler-task-plugin dev-SNAPSHOT - 4.0.0 dolphinscheduler-task-switch jar @@ -41,4 +40,4 @@ - \ No newline at end of file + diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-switch/src/main/java/org/apache/dolphinscheduler/plugin/task/switchtask/SwitchParameters.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-switch/src/main/java/org/apache/dolphinscheduler/plugin/task/switchtask/SwitchParameters.java index d92fd4599369..37287aec03be 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-switch/src/main/java/org/apache/dolphinscheduler/plugin/task/switchtask/SwitchParameters.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-switch/src/main/java/org/apache/dolphinscheduler/plugin/task/switchtask/SwitchParameters.java @@ -79,15 +79,16 @@ public void setSwitchResult(SwitchResult switchResult) { @Override public boolean checkParameters() { - //default next node should not be null + // default next node should not be null boolean defaultNode = switchResult != null && switchResult.getNextNode() != null; if (!defaultNode) { return false; } - //validate conditions must have next node + // validate conditions must have next node List conditions = this.switchResult.getDependTaskList(); if (conditions != null && conditions.size() != 0) { - if (conditions.stream().anyMatch(e -> (StringUtils.isNotEmpty(e.getCondition()) && e.getNextNode() == null))) { + if (conditions.stream() + .anyMatch(e -> (StringUtils.isNotEmpty(e.getCondition()) && e.getNextNode() == null))) { return false; } } diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-switch/src/main/java/org/apache/dolphinscheduler/plugin/task/switchtask/SwitchResult.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-switch/src/main/java/org/apache/dolphinscheduler/plugin/task/switchtask/SwitchResult.java index da67918a6108..f6bb53e71953 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-switch/src/main/java/org/apache/dolphinscheduler/plugin/task/switchtask/SwitchResult.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-switch/src/main/java/org/apache/dolphinscheduler/plugin/task/switchtask/SwitchResult.java @@ -20,6 +20,7 @@ import java.util.List; public class SwitchResult { + private List dependTaskList; private Long nextNode; diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-switch/src/main/java/org/apache/dolphinscheduler/plugin/task/switchtask/SwitchTaskChannelFactory.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-switch/src/main/java/org/apache/dolphinscheduler/plugin/task/switchtask/SwitchTaskChannelFactory.java index fb41a0b6770c..4a25905d7998 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-switch/src/main/java/org/apache/dolphinscheduler/plugin/task/switchtask/SwitchTaskChannelFactory.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-switch/src/main/java/org/apache/dolphinscheduler/plugin/task/switchtask/SwitchTaskChannelFactory.java @@ -17,7 +17,6 @@ package org.apache.dolphinscheduler.plugin.task.switchtask; -import com.google.auto.service.AutoService; import org.apache.dolphinscheduler.plugin.task.api.TaskChannel; import org.apache.dolphinscheduler.plugin.task.api.TaskChannelFactory; import org.apache.dolphinscheduler.plugin.task.api.TaskConstants; @@ -25,8 +24,11 @@ import java.util.List; +import com.google.auto.service.AutoService; + @AutoService(TaskChannelFactory.class) public class SwitchTaskChannelFactory implements TaskChannelFactory { + @Override public TaskChannel create() { return new SwitchTaskChannel(); diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-zeppelin/src/main/java/org/apache/dolphinscheduler/plugin/task/zeppelin/ZeppelinParameters.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-zeppelin/src/main/java/org/apache/dolphinscheduler/plugin/task/zeppelin/ZeppelinParameters.java index b95c93971f14..9f45bb952f0b 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-zeppelin/src/main/java/org/apache/dolphinscheduler/plugin/task/zeppelin/ZeppelinParameters.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-zeppelin/src/main/java/org/apache/dolphinscheduler/plugin/task/zeppelin/ZeppelinParameters.java @@ -17,9 +17,6 @@ package org.apache.dolphinscheduler.plugin.task.zeppelin; -import lombok.Getter; -import lombok.Setter; -import lombok.ToString; import org.apache.dolphinscheduler.plugin.task.api.model.ResourceInfo; import org.apache.dolphinscheduler.plugin.task.api.parameters.AbstractParameters; import org.apache.dolphinscheduler.spi.utils.StringUtils; @@ -27,6 +24,10 @@ import java.util.Collections; import java.util.List; +import lombok.Getter; +import lombok.Setter; +import lombok.ToString; + @Getter @Setter @ToString diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-zeppelin/src/main/java/org/apache/dolphinscheduler/plugin/task/zeppelin/ZeppelinTaskChannelFactory.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-zeppelin/src/main/java/org/apache/dolphinscheduler/plugin/task/zeppelin/ZeppelinTaskChannelFactory.java index aefe54129c7b..006735875ce5 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-zeppelin/src/main/java/org/apache/dolphinscheduler/plugin/task/zeppelin/ZeppelinTaskChannelFactory.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-zeppelin/src/main/java/org/apache/dolphinscheduler/plugin/task/zeppelin/ZeppelinTaskChannelFactory.java @@ -27,6 +27,7 @@ @AutoService(TaskChannelFactory.class) public class ZeppelinTaskChannelFactory implements TaskChannelFactory { + @Override public String getName() { return "ZEPPELIN"; diff --git a/dolphinscheduler-tools/pom.xml b/dolphinscheduler-tools/pom.xml index 4212bcf4bfd9..f82acdcdf80c 100644 --- a/dolphinscheduler-tools/pom.xml +++ b/dolphinscheduler-tools/pom.xml @@ -18,25 +18,17 @@ ~ under the License. ~ --> - + 4.0.0 - dolphinscheduler org.apache.dolphinscheduler + dolphinscheduler dev-SNAPSHOT - 4.0.0 dolphinscheduler-tools - - - org.apache.dolphinscheduler - dolphinscheduler-dao - - - @@ -49,6 +41,13 @@ + + + org.apache.dolphinscheduler + dolphinscheduler-dao + + + @@ -65,10 +64,10 @@ dolphinscheduler-tools - package single + package tools diff --git a/dolphinscheduler-tools/src/main/java/org/apache/dolphinscheduler/tools/datasource/DolphinSchedulerManager.java b/dolphinscheduler-tools/src/main/java/org/apache/dolphinscheduler/tools/datasource/DolphinSchedulerManager.java index a588fabcfb1f..b56d103606cc 100644 --- a/dolphinscheduler-tools/src/main/java/org/apache/dolphinscheduler/tools/datasource/DolphinSchedulerManager.java +++ b/dolphinscheduler-tools/src/main/java/org/apache/dolphinscheduler/tools/datasource/DolphinSchedulerManager.java @@ -33,6 +33,7 @@ @Service public class DolphinSchedulerManager { + private static final Logger logger = LoggerFactory.getLogger(DolphinSchedulerManager.class); private final UpgradeDao upgradeDao; @@ -40,11 +41,10 @@ public class DolphinSchedulerManager { public DolphinSchedulerManager(DataSource dataSource, List daos) throws Exception { final DbType type = getCurrentDbType(dataSource); upgradeDao = daos.stream() - .filter(it -> it.getDbType() == type) - .findFirst() - .orElseThrow(() -> new RuntimeException( - "Cannot find UpgradeDao implementation for db type: " + type - )); + .filter(it -> it.getDbType() == type) + .findFirst() + .orElseThrow(() -> new RuntimeException( + "Cannot find UpgradeDao implementation for db type: " + type)); } private DbType getCurrentDbType(DataSource dataSource) throws Exception { @@ -65,8 +65,8 @@ public void initDolphinScheduler() { public boolean schemaIsInitialized() { // Determines whether the dolphinscheduler table structure has been init if (upgradeDao.isExistsTable("t_escheduler_version") - || upgradeDao.isExistsTable("t_ds_version") - || upgradeDao.isExistsTable("t_escheduler_queue")) { + || upgradeDao.isExistsTable("t_ds_version") + || upgradeDao.isExistsTable("t_escheduler_queue")) { logger.info("The database has been initialized. Skip the initialization step"); return true; } @@ -106,7 +106,7 @@ public void upgradeDolphinScheduler() throws IOException { if (SchemaUtils.isAGreatVersion(schemaVersion, version)) { logger.info("upgrade DolphinScheduler metadata version from {} to {}", version, schemaVersion); logger.info("Begin upgrading DolphinScheduler's table structure"); - upgradeDao.upgradeDolphinScheduler(schemaDir); + upgradeDao.upgradeDolphinScheduler(schemaDir); if ("1.3.0".equals(schemaVersion)) { upgradeDao.upgradeDolphinSchedulerWorkerGroup(); } else if ("1.3.2".equals(schemaVersion)) { @@ -118,7 +118,8 @@ public void upgradeDolphinScheduler() throws IOException { } } - if (SchemaUtils.isAGreatVersion("2.0.6", currentVersion) && SchemaUtils.isAGreatVersion(SchemaUtils.getSoftVersion(), currentVersion)) { + if (SchemaUtils.isAGreatVersion("2.0.6", currentVersion) + && SchemaUtils.isAGreatVersion(SchemaUtils.getSoftVersion(), currentVersion)) { upgradeDao.upgradeDolphinSchedulerResourceFileSize(); } } diff --git a/dolphinscheduler-tools/src/main/java/org/apache/dolphinscheduler/tools/datasource/InitDolphinScheduler.java b/dolphinscheduler-tools/src/main/java/org/apache/dolphinscheduler/tools/datasource/InitDolphinScheduler.java index 6ccd73decdb9..15b1cbb3b2a1 100644 --- a/dolphinscheduler-tools/src/main/java/org/apache/dolphinscheduler/tools/datasource/InitDolphinScheduler.java +++ b/dolphinscheduler-tools/src/main/java/org/apache/dolphinscheduler/tools/datasource/InitDolphinScheduler.java @@ -27,6 +27,7 @@ @SpringBootApplication public class InitDolphinScheduler { + public static void main(String[] args) { SpringApplication.run(InitDolphinScheduler.class, args); } @@ -34,6 +35,7 @@ public static void main(String[] args) { @Component @Profile("init") static class InitRunner implements CommandLineRunner { + private static final Logger logger = LoggerFactory.getLogger(InitRunner.class); private final DolphinSchedulerManager dolphinSchedulerManager; diff --git a/dolphinscheduler-tools/src/main/java/org/apache/dolphinscheduler/tools/datasource/UpgradeDolphinScheduler.java b/dolphinscheduler-tools/src/main/java/org/apache/dolphinscheduler/tools/datasource/UpgradeDolphinScheduler.java index 22e1338431f7..bdd7fcf9977d 100644 --- a/dolphinscheduler-tools/src/main/java/org/apache/dolphinscheduler/tools/datasource/UpgradeDolphinScheduler.java +++ b/dolphinscheduler-tools/src/main/java/org/apache/dolphinscheduler/tools/datasource/UpgradeDolphinScheduler.java @@ -27,6 +27,7 @@ @SpringBootApplication public class UpgradeDolphinScheduler { + public static void main(String[] args) { SpringApplication.run(UpgradeDolphinScheduler.class, args); } @@ -34,6 +35,7 @@ public static void main(String[] args) { @Component @Profile("upgrade") static class UpgradeRunner implements CommandLineRunner { + private static final Logger logger = LoggerFactory.getLogger(UpgradeRunner.class); private final DolphinSchedulerManager dolphinSchedulerManager; diff --git a/dolphinscheduler-tools/src/main/java/org/apache/dolphinscheduler/tools/datasource/dao/MySQLUpgradeDao.java b/dolphinscheduler-tools/src/main/java/org/apache/dolphinscheduler/tools/datasource/dao/MySQLUpgradeDao.java index 14fa45cff192..73619320349f 100644 --- a/dolphinscheduler-tools/src/main/java/org/apache/dolphinscheduler/tools/datasource/dao/MySQLUpgradeDao.java +++ b/dolphinscheduler-tools/src/main/java/org/apache/dolphinscheduler/tools/datasource/dao/MySQLUpgradeDao.java @@ -32,6 +32,7 @@ @Service public class MySQLUpgradeDao extends UpgradeDao { + public static final Logger logger = LoggerFactory.getLogger(MySQLUpgradeDao.class); private MySQLUpgradeDao(DataSource dataSource) { @@ -62,8 +63,8 @@ public boolean isExistsTable(String tableName) { rs = conn.getMetaData().getTables(conn.getCatalog(), conn.getSchema(), tableName, null); return rs.next(); } catch (SQLException e) { - logger.error(e.getMessage(),e); - throw new RuntimeException(e.getMessage(),e); + logger.error(e.getMessage(), e); + throw new RuntimeException(e.getMessage(), e); } finally { ConnectionUtils.releaseResource(rs, conn); } @@ -77,16 +78,16 @@ public boolean isExistsTable(String tableName) { * @return if column name exist return true，else return false */ @Override - public boolean isExistsColumn(String tableName,String columnName) { + public boolean isExistsColumn(String tableName, String columnName) { Connection conn = null; try { conn = dataSource.getConnection(); - ResultSet rs = conn.getMetaData().getColumns(conn.getCatalog(), conn.getSchema(),tableName,columnName); + ResultSet rs = conn.getMetaData().getColumns(conn.getCatalog(), conn.getSchema(), tableName, columnName); return rs.next(); } catch (SQLException e) { - logger.error(e.getMessage(),e); - throw new RuntimeException(e.getMessage(),e); + logger.error(e.getMessage(), e); + throw new RuntimeException(e.getMessage(), e); } finally { ConnectionUtils.releaseResource(conn); } diff --git a/dolphinscheduler-tools/src/main/java/org/apache/dolphinscheduler/tools/datasource/dao/PostgreSQLUpgradeDao.java b/dolphinscheduler-tools/src/main/java/org/apache/dolphinscheduler/tools/datasource/dao/PostgreSQLUpgradeDao.java index f0ad67c1fd0a..43761ba56438 100644 --- a/dolphinscheduler-tools/src/main/java/org/apache/dolphinscheduler/tools/datasource/dao/PostgreSQLUpgradeDao.java +++ b/dolphinscheduler-tools/src/main/java/org/apache/dolphinscheduler/tools/datasource/dao/PostgreSQLUpgradeDao.java @@ -33,6 +33,7 @@ @Service public class PostgreSQLUpgradeDao extends UpgradeDao { + public static final Logger logger = LoggerFactory.getLogger(PostgreSQLUpgradeDao.class); private PostgreSQLUpgradeDao(DataSource dataSource) { @@ -71,7 +72,6 @@ public String getSchema() { return ""; } - /** * determines whether a table exists * diff --git a/dolphinscheduler-tools/src/main/java/org/apache/dolphinscheduler/tools/datasource/dao/ResourceDao.java b/dolphinscheduler-tools/src/main/java/org/apache/dolphinscheduler/tools/datasource/dao/ResourceDao.java index d39779416656..48a46249e78f 100644 --- a/dolphinscheduler-tools/src/main/java/org/apache/dolphinscheduler/tools/datasource/dao/ResourceDao.java +++ b/dolphinscheduler-tools/src/main/java/org/apache/dolphinscheduler/tools/datasource/dao/ResourceDao.java @@ -17,17 +17,17 @@ package org.apache.dolphinscheduler.tools.datasource.dao; -import java.sql.SQLException; -import java.util.Objects; import org.apache.dolphinscheduler.common.utils.ConnectionUtils; import org.apache.dolphinscheduler.spi.utils.StringUtils; import java.sql.Connection; import java.sql.PreparedStatement; import java.sql.ResultSet; +import java.sql.SQLException; import java.util.Arrays; import java.util.HashMap; import java.util.Map; +import java.util.Objects; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -39,6 +39,7 @@ * resource dao */ public class ResourceDao { + public static final Logger logger = LoggerFactory.getLogger(ResourceDao.class); /** @@ -82,7 +83,8 @@ Map listAllResources(Connection conn) { private Map listAllResourcesByFileType(Connection conn, int type) { Map resourceSizeMap = new HashMap<>(); - String sql = String.format("SELECT full_name, type, size, is_directory FROM t_ds_resources where type = %d", type); + String sql = + String.format("SELECT full_name, type, size, is_directory FROM t_ds_resources where type = %d", type); ResultSet rs = null; PreparedStatement pstmt = null; try { @@ -97,7 +99,7 @@ private Map listAllResourcesByFileType(Connection conn, int type) if (StringUtils.isNotBlank(fullName) && !isDirectory) { String[] splits = fullName.split("/"); for (int i = 1; i < splits.length; i++) { - String parentFullName = Joiner.on("/").join(Arrays.copyOfRange(splits,0, splits.length - i)); + String parentFullName = Joiner.on("/").join(Arrays.copyOfRange(splits, 0, splits.length - i)); if (!Strings.isNullOrEmpty(parentFullName)) { long size = resourceSizeMap.getOrDefault(parentFullName, 0L); resourceSizeMap.put(parentFullName, size + fileSize); diff --git a/dolphinscheduler-ui/pom.xml b/dolphinscheduler-ui/pom.xml index ed8dc6fcd95a..fb6d32fa49a9 100644 --- a/dolphinscheduler-ui/pom.xml +++ b/dolphinscheduler-ui/pom.xml @@ -15,73 +15,73 @@ ~ See the License for the specific language governing permissions and ~ limitations under the License. --> + + 4.0.0 + + org.apache.dolphinscheduler + dolphinscheduler + dev-SNAPSHOT + - - - dolphinscheduler - org.apache.dolphinscheduler - dev-SNAPSHOT - - 4.0.0 + dolphinscheduler-ui - dolphinscheduler-ui + ${project.artifactId} - ${project.artifactId} + + v16.13.1 + v6.32.6 + src + 1.12.1 + - - v16.13.1 - v6.32.6 - src - 1.12.1 - + + + release + + + + com.github.eirslett + frontend-maven-plugin + ${frontend-maven-plugin.version} + + false + + + + install node and pnpm + + install-node-and-pnpm + + + ${node.version} + ${pnpm.version} + + + + pnpm install + + pnpm + + generate-resources + + install + + + + pnpm run build:prod + + pnpm + + + run build:prod + + + + + - - - release - - - - com.github.eirslett - frontend-maven-plugin - ${frontend-maven-plugin.version} - - false - - - - install node and pnpm - - install-node-and-pnpm - - - ${node.version} - ${pnpm.version} - - - - pnpm install - - pnpm - - generate-resources - - install - - - - pnpm run build:prod - - pnpm - - - run build:prod - - - - - - - - - + + + diff --git a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/WorkerServer.java b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/WorkerServer.java index 8f8a27372b34..c3ae0884b62b 100644 --- a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/WorkerServer.java +++ b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/WorkerServer.java @@ -17,7 +17,6 @@ package org.apache.dolphinscheduler.server.worker; -import org.apache.commons.collections4.CollectionUtils; import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.IStoppable; import org.apache.dolphinscheduler.common.lifecycle.ServerLifeCycleManager; @@ -35,6 +34,13 @@ import org.apache.dolphinscheduler.service.bean.SpringApplicationContext; import org.apache.dolphinscheduler.service.task.TaskPluginManager; import org.apache.dolphinscheduler.service.utils.LoggerUtils; + +import org.apache.commons.collections4.CollectionUtils; + +import java.util.Collection; + +import javax.annotation.PostConstruct; + import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; @@ -44,9 +50,6 @@ import org.springframework.context.annotation.FilterType; import org.springframework.transaction.annotation.EnableTransactionManagement; -import javax.annotation.PostConstruct; -import java.util.Collection; - @SpringBootApplication @EnableTransactionManagement @ComponentScan(basePackages = "org.apache.dolphinscheduler", excludeFilters = { diff --git a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/message/MessageRetryRunner.java b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/message/MessageRetryRunner.java index 58e1fe7e8864..eb4cbf121004 100644 --- a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/message/MessageRetryRunner.java +++ b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/message/MessageRetryRunner.java @@ -17,25 +17,28 @@ package org.apache.dolphinscheduler.server.worker.message; -import lombok.NonNull; import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.lifecycle.ServerLifeCycleManager; import org.apache.dolphinscheduler.common.thread.BaseDaemonThread; import org.apache.dolphinscheduler.remote.command.BaseCommand; import org.apache.dolphinscheduler.remote.command.CommandType; import org.apache.dolphinscheduler.service.utils.LoggerUtils; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.context.ApplicationContext; -import org.springframework.stereotype.Component; -import javax.annotation.PostConstruct; import java.time.Duration; import java.util.HashMap; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; +import javax.annotation.PostConstruct; + +import lombok.NonNull; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.context.ApplicationContext; +import org.springframework.stereotype.Component; + @Component public class MessageRetryRunner extends BaseDaemonThread { diff --git a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/message/TaskExecuteResultMessageSender.java b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/message/TaskExecuteResultMessageSender.java index a7d05a479194..3cdabad3de91 100644 --- a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/message/TaskExecuteResultMessageSender.java +++ b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/message/TaskExecuteResultMessageSender.java @@ -44,10 +44,10 @@ public void sendMessage(TaskExecuteResultCommand message) throws RemotingExcepti public TaskExecuteResultCommand buildMessage(TaskExecutionContext taskExecutionContext, String messageReceiverAddress) { - TaskExecuteResultCommand taskExecuteResultMessage - = new TaskExecuteResultCommand(workerConfig.getWorkerAddress(), - messageReceiverAddress, - System.currentTimeMillis()); + TaskExecuteResultCommand taskExecuteResultMessage = + new TaskExecuteResultCommand(workerConfig.getWorkerAddress(), + messageReceiverAddress, + System.currentTimeMillis()); taskExecuteResultMessage.setProcessInstanceId(taskExecutionContext.getProcessInstanceId()); taskExecuteResultMessage.setTaskInstanceId(taskExecutionContext.getTaskInstanceId()); taskExecuteResultMessage.setStatus(taskExecutionContext.getCurrentExecutionStatus().getCode()); diff --git a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/message/TaskRejectMessageSender.java b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/message/TaskRejectMessageSender.java index d50c5d899711..81b811e485df 100644 --- a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/message/TaskRejectMessageSender.java +++ b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/message/TaskRejectMessageSender.java @@ -44,8 +44,8 @@ public void sendMessage(TaskRejectCommand message) throws RemotingException { public TaskRejectCommand buildMessage(TaskExecutionContext taskExecutionContext, String masterAddress) { TaskRejectCommand taskRejectMessage = new TaskRejectCommand(workerConfig.getWorkerAddress(), - masterAddress, - System.currentTimeMillis()); + masterAddress, + System.currentTimeMillis()); taskRejectMessage.setTaskInstanceId(taskExecutionContext.getTaskInstanceId()); taskRejectMessage.setProcessInstanceId(taskExecutionContext.getProcessInstanceId()); taskRejectMessage.setHost(taskExecutionContext.getHost()); diff --git a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/metrics/TaskMetrics.java b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/metrics/TaskMetrics.java index 6d3de8e146b3..32213774b4d8 100644 --- a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/metrics/TaskMetrics.java +++ b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/metrics/TaskMetrics.java @@ -17,16 +17,15 @@ package org.apache.dolphinscheduler.server.worker.metrics; - import org.apache.dolphinscheduler.plugin.task.api.TaskChannelFactory; import java.util.HashMap; import java.util.Map; import java.util.ServiceLoader; +import lombok.experimental.UtilityClass; import io.micrometer.core.instrument.Counter; import io.micrometer.core.instrument.Metrics; -import lombok.experimental.UtilityClass; @UtilityClass public class TaskMetrics { @@ -45,8 +44,7 @@ public class TaskMetrics { Counter.builder("ds.task.execution.count.by.type") .tag("task_type", taskChannelFactory.getName()) .description("task execution counter by type") - .register(Metrics.globalRegistry) - ); + .register(Metrics.globalRegistry)); } } diff --git a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/metrics/WorkerServerMetrics.java b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/metrics/WorkerServerMetrics.java index abe5f3fbb22e..ad40bc344353 100644 --- a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/metrics/WorkerServerMetrics.java +++ b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/metrics/WorkerServerMetrics.java @@ -20,25 +20,25 @@ import java.util.concurrent.TimeUnit; import java.util.function.Supplier; +import lombok.experimental.UtilityClass; import io.micrometer.core.instrument.Counter; import io.micrometer.core.instrument.DistributionSummary; import io.micrometer.core.instrument.Gauge; import io.micrometer.core.instrument.Metrics; import io.micrometer.core.instrument.Timer; -import lombok.experimental.UtilityClass; @UtilityClass public class WorkerServerMetrics { private final Counter workerOverloadCounter = - Counter.builder("ds.worker.overload.count") - .description("overloaded workers count") - .register(Metrics.globalRegistry); + Counter.builder("ds.worker.overload.count") + .description("overloaded workers count") + .register(Metrics.globalRegistry); private final Counter workerFullSubmitQueueCounter = - Counter.builder("ds.worker.full.submit.queue.count") - .description("full worker submit queues count") - .register(Metrics.globalRegistry); + Counter.builder("ds.worker.full.submit.queue.count") + .description("full worker submit queues count") + .register(Metrics.globalRegistry); private final Counter workerResourceDownloadSuccessCounter = Counter.builder("ds.worker.resource.download.count") @@ -61,11 +61,11 @@ public class WorkerServerMetrics { private final DistributionSummary workerResourceDownloadSizeDistribution = DistributionSummary.builder("ds.worker.resource.download.size") - .baseUnit("bytes") - .publishPercentiles(0.5, 0.75, 0.95, 0.99) - .publishPercentileHistogram() - .description("size of downloaded resource files on worker") - .register(Metrics.globalRegistry); + .baseUnit("bytes") + .publishPercentiles(0.5, 0.75, 0.95, 0.99) + .publishPercentileHistogram() + .description("size of downloaded resource files on worker") + .register(Metrics.globalRegistry); public void incWorkerOverloadCount() { workerOverloadCounter.increment(); @@ -93,8 +93,8 @@ public void recordWorkerResourceDownloadSize(final long size) { public void registerWorkerRunningTaskGauge(final Supplier supplier) { Gauge.builder("ds.task.running", supplier) - .description("number of running tasks on workers") - .register(Metrics.globalRegistry); + .description("number of running tasks on workers") + .register(Metrics.globalRegistry); } } diff --git a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/processor/HostUpdateProcessor.java b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/processor/HostUpdateProcessor.java index d36c80012d97..0f6a3d093bbd 100644 --- a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/processor/HostUpdateProcessor.java +++ b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/processor/HostUpdateProcessor.java @@ -30,7 +30,6 @@ import org.springframework.stereotype.Component; import com.google.common.base.Preconditions; - import io.netty.channel.Channel; /** @@ -48,7 +47,7 @@ public class HostUpdateProcessor implements NettyRequestProcessor { @Override public void process(Channel channel, Command command) { Preconditions.checkArgument(CommandType.PROCESS_HOST_UPDATE_REQUEST == command.getType(), - String.format("invalid command type : %s", command.getType())); + String.format("invalid command type : %s", command.getType())); HostUpdateCommand updateCommand = JSONUtils.parseObject(command.getBody(), HostUpdateCommand.class); if (updateCommand == null) { logger.error("host update command is null"); diff --git a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/processor/TaskDispatchProcessor.java b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/processor/TaskDispatchProcessor.java index d85181f1ed82..ec5bd2ebadde 100644 --- a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/processor/TaskDispatchProcessor.java +++ b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/processor/TaskDispatchProcessor.java @@ -37,6 +37,7 @@ import org.apache.dolphinscheduler.service.task.TaskPluginManager; import org.apache.dolphinscheduler.service.utils.LogUtils; import org.apache.dolphinscheduler.service.utils.LoggerUtils; + import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; @@ -115,8 +116,9 @@ public void process(Channel channel, Command command) { taskExecutionContext.setLogPath(LogUtils.getTaskLogPath(taskExecutionContext)); // delay task process - long remainTime = DateUtils.getRemainTime(DateUtils.timeStampToDate(taskExecutionContext.getFirstSubmitTime()), - taskExecutionContext.getDelayTime() * 60L); + long remainTime = + DateUtils.getRemainTime(DateUtils.timeStampToDate(taskExecutionContext.getFirstSubmitTime()), + taskExecutionContext.getDelayTime() * 60L); if (remainTime > 0) { logger.info("Current taskInstance is choose delay execution, delay time: {}s", remainTime); taskExecutionContext.setCurrentExecutionStatus(TaskExecutionStatus.DELAY_EXECUTION); @@ -137,10 +139,14 @@ public void process(Channel channel, Command command) { // submit task to manager boolean offer = workerManager.offer(workerTaskExecuteRunnable); if (!offer) { - logger.warn("submit task to wait queue error, queue is full, current queue size is {}, will send a task reject message to master", workerManager.getWaitSubmitQueueSize()); - workerMessageSender.sendMessageWithRetry(taskExecutionContext, workflowMasterAddress, CommandType.TASK_REJECT); + logger.warn( + "submit task to wait queue error, queue is full, current queue size is {}, will send a task reject message to master", + workerManager.getWaitSubmitQueueSize()); + workerMessageSender.sendMessageWithRetry(taskExecutionContext, workflowMasterAddress, + CommandType.TASK_REJECT); } else { - logger.info("Submit task to wait queue success, current queue size is {}", workerManager.getWaitSubmitQueueSize()); + logger.info("Submit task to wait queue success, current queue size is {}", + workerManager.getWaitSubmitQueueSize()); } } finally { LoggerUtils.removeWorkflowAndTaskInstanceIdMDC(); diff --git a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/processor/TaskExecuteResultAckProcessor.java b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/processor/TaskExecuteResultAckProcessor.java index 4f6eb9ce4796..e2ff1d005a54 100644 --- a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/processor/TaskExecuteResultAckProcessor.java +++ b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/processor/TaskExecuteResultAckProcessor.java @@ -17,8 +17,6 @@ package org.apache.dolphinscheduler.server.worker.processor; -import com.google.common.base.Preconditions; -import io.netty.channel.Channel; import org.apache.dolphinscheduler.common.utils.JSONUtils; import org.apache.dolphinscheduler.remote.command.Command; import org.apache.dolphinscheduler.remote.command.CommandType; @@ -26,11 +24,15 @@ import org.apache.dolphinscheduler.remote.processor.NettyRequestProcessor; import org.apache.dolphinscheduler.server.worker.message.MessageRetryRunner; import org.apache.dolphinscheduler.service.utils.LoggerUtils; + import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; +import com.google.common.base.Preconditions; +import io.netty.channel.Channel; + /** * task execute running ack, from master to worker */ diff --git a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/processor/TaskExecuteRunningAckProcessor.java b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/processor/TaskExecuteRunningAckProcessor.java index fc8ead96fa72..9cbde24d9e2d 100644 --- a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/processor/TaskExecuteRunningAckProcessor.java +++ b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/processor/TaskExecuteRunningAckProcessor.java @@ -24,13 +24,13 @@ import org.apache.dolphinscheduler.remote.processor.NettyRequestProcessor; import org.apache.dolphinscheduler.server.worker.message.MessageRetryRunner; import org.apache.dolphinscheduler.service.utils.LoggerUtils; + import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; import com.google.common.base.Preconditions; - import io.netty.channel.Channel; /** diff --git a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/processor/TaskRejectAckProcessor.java b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/processor/TaskRejectAckProcessor.java index 7fefb6172984..68c88332e2b8 100644 --- a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/processor/TaskRejectAckProcessor.java +++ b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/processor/TaskRejectAckProcessor.java @@ -17,8 +17,6 @@ package org.apache.dolphinscheduler.server.worker.processor; -import com.google.common.base.Preconditions; -import io.netty.channel.Channel; import org.apache.dolphinscheduler.common.utils.JSONUtils; import org.apache.dolphinscheduler.remote.command.Command; import org.apache.dolphinscheduler.remote.command.CommandType; @@ -26,11 +24,15 @@ import org.apache.dolphinscheduler.remote.processor.NettyRequestProcessor; import org.apache.dolphinscheduler.server.worker.message.MessageRetryRunner; import org.apache.dolphinscheduler.service.utils.LoggerUtils; + import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; +import com.google.common.base.Preconditions; +import io.netty.channel.Channel; + @Component public class TaskRejectAckProcessor implements NettyRequestProcessor { diff --git a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/processor/TaskSavePointProcessor.java b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/processor/TaskSavePointProcessor.java index b264eb0110d3..3190af4b4f79 100644 --- a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/processor/TaskSavePointProcessor.java +++ b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/processor/TaskSavePointProcessor.java @@ -17,10 +17,6 @@ package org.apache.dolphinscheduler.server.worker.processor; -import com.google.common.base.Preconditions; -import io.netty.channel.Channel; -import io.netty.channel.ChannelFuture; -import io.netty.channel.ChannelFutureListener; import org.apache.dolphinscheduler.common.utils.JSONUtils; import org.apache.dolphinscheduler.plugin.task.api.AbstractTask; import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext; @@ -34,11 +30,18 @@ import org.apache.dolphinscheduler.server.worker.runner.WorkerManagerThread; import org.apache.dolphinscheduler.server.worker.runner.WorkerTaskExecuteRunnable; import org.apache.dolphinscheduler.service.utils.LoggerUtils; + import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; +import com.google.common.base.Preconditions; + +import io.netty.channel.Channel; +import io.netty.channel.ChannelFuture; +import io.netty.channel.ChannelFutureListener; + /** * task save point processor */ @@ -62,8 +65,9 @@ public class TaskSavePointProcessor implements NettyRequestProcessor { @Override public void process(Channel channel, Command command) { Preconditions.checkArgument(CommandType.TASK_SAVEPOINT_REQUEST == command.getType(), - String.format("invalid command type : %s", command.getType())); - TaskSavePointRequestCommand taskSavePointRequestCommand = JSONUtils.parseObject(command.getBody(), TaskSavePointRequestCommand.class); + String.format("invalid command type : %s", command.getType())); + TaskSavePointRequestCommand taskSavePointRequestCommand = + JSONUtils.parseObject(command.getBody(), TaskSavePointRequestCommand.class); if (taskSavePointRequestCommand == null) { logger.error("task savepoint request command is null"); return; @@ -71,9 +75,11 @@ public void process(Channel channel, Command command) { logger.info("Receive task savepoint command : {}", taskSavePointRequestCommand); int taskInstanceId = taskSavePointRequestCommand.getTaskInstanceId(); - TaskExecutionContext taskExecutionContext = TaskExecutionContextCacheManager.getByTaskInstanceId(taskInstanceId); + TaskExecutionContext taskExecutionContext = + TaskExecutionContextCacheManager.getByTaskInstanceId(taskInstanceId); if (taskExecutionContext == null) { - logger.error("taskRequest cache is null, taskInstanceId: {}", taskSavePointRequestCommand.getTaskInstanceId()); + logger.error("taskRequest cache is null, taskInstanceId: {}", + taskSavePointRequestCommand.getTaskInstanceId()); return; } @@ -91,12 +97,15 @@ private void sendTaskSavePointResponseCommand(Channel channel, TaskExecutionCont TaskSavePointResponseCommand taskSavePointResponseCommand = new TaskSavePointResponseCommand(); taskSavePointResponseCommand.setTaskInstanceId(taskExecutionContext.getTaskInstanceId()); channel.writeAndFlush(taskSavePointResponseCommand.convert2Command()).addListener(new ChannelFutureListener() { + @Override public void operationComplete(ChannelFuture future) throws Exception { if (!future.isSuccess()) { - logger.error("Submit kill response to master error, kill command: {}", taskSavePointResponseCommand); + logger.error("Submit kill response to master error, kill command: {}", + taskSavePointResponseCommand); } else - logger.info("Submit kill response to master success, kill command: {}", taskSavePointResponseCommand); + logger.info("Submit kill response to master success, kill command: {}", + taskSavePointResponseCommand); } }); } @@ -117,7 +126,7 @@ protected void doSavePoint(int taskInstanceId) { return; } try { - ((StreamTask)task).savePoint(); + ((StreamTask) task).savePoint(); } catch (Exception e) { logger.error("task save point error", e); } diff --git a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/registry/WorkerConnectionStateListener.java b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/registry/WorkerConnectionStateListener.java index 36b95e1b629c..a8f8cad6bc02 100644 --- a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/registry/WorkerConnectionStateListener.java +++ b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/registry/WorkerConnectionStateListener.java @@ -17,12 +17,14 @@ package org.apache.dolphinscheduler.server.worker.registry; -import lombok.NonNull; import org.apache.dolphinscheduler.common.lifecycle.ServerLifeCycleManager; import org.apache.dolphinscheduler.registry.api.ConnectionListener; import org.apache.dolphinscheduler.registry.api.ConnectionState; import org.apache.dolphinscheduler.server.worker.config.WorkerConfig; import org.apache.dolphinscheduler.service.registry.RegistryClient; + +import lombok.NonNull; + import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/registry/WorkerStopStrategy.java b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/registry/WorkerStopStrategy.java index 08d434ea2d39..7e7e294ecccf 100644 --- a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/registry/WorkerStopStrategy.java +++ b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/registry/WorkerStopStrategy.java @@ -20,6 +20,7 @@ import org.apache.dolphinscheduler.registry.api.StrategyType; import org.apache.dolphinscheduler.server.worker.config.WorkerConfig; import org.apache.dolphinscheduler.service.registry.RegistryClient; + import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; diff --git a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/registry/WorkerWaitingStrategy.java b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/registry/WorkerWaitingStrategy.java index 203e5711f187..2d1c3359b245 100644 --- a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/registry/WorkerWaitingStrategy.java +++ b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/registry/WorkerWaitingStrategy.java @@ -27,14 +27,15 @@ import org.apache.dolphinscheduler.server.worker.rpc.WorkerRpcServer; import org.apache.dolphinscheduler.server.worker.runner.WorkerManagerThread; import org.apache.dolphinscheduler.service.registry.RegistryClient; + +import java.time.Duration; + import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.stereotype.Service; -import java.time.Duration; - @Service @ConditionalOnProperty(prefix = "worker.registry-disconnect-strategy", name = "strategy", havingValue = "waiting") public class WorkerWaitingStrategy implements WorkerConnectStrategy { diff --git a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/rpc/WorkerMessageSender.java b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/rpc/WorkerMessageSender.java index 5a2752ca3455..ae2fba6c7be8 100644 --- a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/rpc/WorkerMessageSender.java +++ b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/rpc/WorkerMessageSender.java @@ -29,14 +29,14 @@ import javax.annotation.PostConstruct; +import lombok.NonNull; + import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.context.ApplicationContext; import org.springframework.stereotype.Component; -import lombok.NonNull; - @Component public class WorkerMessageSender { @@ -54,7 +54,7 @@ public class WorkerMessageSender { public void init() { Map messageSenders = applicationContext.getBeansOfType(MessageSender.class); messageSenders.values().forEach(messageSender -> messageSenderMap.put(messageSender.getMessageType(), - messageSender)); + messageSender)); } // todo: use message rather than context diff --git a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/rpc/WorkerRpcClient.java b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/rpc/WorkerRpcClient.java index 4be20873f973..a01cbaabdf82 100644 --- a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/rpc/WorkerRpcClient.java +++ b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/rpc/WorkerRpcClient.java @@ -57,7 +57,7 @@ public void start() { this.nettyRemotingClient = new NettyRemotingClient(nettyClientConfig); // we only use the client to handle the ack message, we can optimize this, send ack to the nettyServer. this.nettyRemotingClient.registerProcessor(CommandType.TASK_EXECUTE_RUNNING_ACK, - taskExecuteRunningAckProcessor); + taskExecuteRunningAckProcessor); this.nettyRemotingClient.registerProcessor(CommandType.TASK_EXECUTE_RESULT_ACK, taskExecuteResultAckProcessor); this.nettyRemotingClient.registerProcessor(CommandType.TASK_REJECT_ACK, taskRejectAckProcessor); logger.info("Worker rpc client started"); diff --git a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/runner/DefaultWorkerDelayTaskExecuteRunnable.java b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/runner/DefaultWorkerDelayTaskExecuteRunnable.java index c00fb4f99aa1..e926b8d9bbe6 100644 --- a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/runner/DefaultWorkerDelayTaskExecuteRunnable.java +++ b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/runner/DefaultWorkerDelayTaskExecuteRunnable.java @@ -17,7 +17,6 @@ package org.apache.dolphinscheduler.server.worker.runner; -import lombok.NonNull; import org.apache.dolphinscheduler.plugin.task.api.TaskCallBack; import org.apache.dolphinscheduler.plugin.task.api.TaskException; import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext; @@ -29,6 +28,8 @@ import javax.annotation.Nullable; +import lombok.NonNull; + public class DefaultWorkerDelayTaskExecuteRunnable extends WorkerDelayTaskExecuteRunnable { public DefaultWorkerDelayTaskExecuteRunnable(@NonNull TaskExecutionContext taskExecutionContext, @@ -38,7 +39,8 @@ public DefaultWorkerDelayTaskExecuteRunnable(@NonNull TaskExecutionContext taskE @NonNull AlertClientService alertClientService, @NonNull TaskPluginManager taskPluginManager, @Nullable StorageOperate storageOperate) { - super(taskExecutionContext, workerConfig, workflowMaster, workerMessageSender, alertClientService, taskPluginManager, storageOperate); + super(taskExecutionContext, workerConfig, workflowMaster, workerMessageSender, alertClientService, + taskPluginManager, storageOperate); } @Override diff --git a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/runner/DefaultWorkerDelayTaskExecuteRunnableFactory.java b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/runner/DefaultWorkerDelayTaskExecuteRunnableFactory.java index 49262b3900d5..dad1421675b1 100644 --- a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/runner/DefaultWorkerDelayTaskExecuteRunnableFactory.java +++ b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/runner/DefaultWorkerDelayTaskExecuteRunnableFactory.java @@ -17,7 +17,6 @@ package org.apache.dolphinscheduler.server.worker.runner; -import lombok.NonNull; import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext; import org.apache.dolphinscheduler.server.worker.config.WorkerConfig; import org.apache.dolphinscheduler.server.worker.rpc.WorkerMessageSender; @@ -27,7 +26,11 @@ import javax.annotation.Nullable; -public class DefaultWorkerDelayTaskExecuteRunnableFactory extends WorkerDelayTaskExecuteRunnableFactory { +import lombok.NonNull; + +public class DefaultWorkerDelayTaskExecuteRunnableFactory + extends + WorkerDelayTaskExecuteRunnableFactory { protected DefaultWorkerDelayTaskExecuteRunnableFactory(@NonNull TaskExecutionContext taskExecutionContext, @NonNull WorkerConfig workerConfig, @@ -36,7 +39,8 @@ protected DefaultWorkerDelayTaskExecuteRunnableFactory(@NonNull TaskExecutionCon @NonNull AlertClientService alertClientService, @NonNull TaskPluginManager taskPluginManager, @Nullable StorageOperate storageOperate) { - super(taskExecutionContext, workerConfig, workflowMasterAddress, workerMessageSender, alertClientService, taskPluginManager, storageOperate); + super(taskExecutionContext, workerConfig, workflowMasterAddress, workerMessageSender, alertClientService, + taskPluginManager, storageOperate); } @Override diff --git a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/runner/WorkerDelayTaskExecuteRunnable.java b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/runner/WorkerDelayTaskExecuteRunnable.java index 19351ee7645f..e322aa9cf08c 100644 --- a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/runner/WorkerDelayTaskExecuteRunnable.java +++ b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/runner/WorkerDelayTaskExecuteRunnable.java @@ -17,7 +17,6 @@ package org.apache.dolphinscheduler.server.worker.runner; -import lombok.NonNull; import org.apache.dolphinscheduler.common.utils.DateUtils; import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext; import org.apache.dolphinscheduler.server.worker.config.WorkerConfig; @@ -26,10 +25,13 @@ import org.apache.dolphinscheduler.service.storage.StorageOperate; import org.apache.dolphinscheduler.service.task.TaskPluginManager; -import javax.annotation.Nullable; import java.util.concurrent.Delayed; import java.util.concurrent.TimeUnit; +import javax.annotation.Nullable; + +import lombok.NonNull; + public abstract class WorkerDelayTaskExecuteRunnable extends WorkerTaskExecuteRunnable implements Delayed { protected WorkerDelayTaskExecuteRunnable(@NonNull TaskExecutionContext taskExecutionContext, @@ -39,7 +41,8 @@ protected WorkerDelayTaskExecuteRunnable(@NonNull TaskExecutionContext taskExecu @NonNull AlertClientService alertClientService, @NonNull TaskPluginManager taskPluginManager, @Nullable StorageOperate storageOperate) { - super(taskExecutionContext, workerConfig, masterAddress, workerMessageSender, alertClientService, taskPluginManager, storageOperate); + super(taskExecutionContext, workerConfig, masterAddress, workerMessageSender, alertClientService, + taskPluginManager, storageOperate); } @Override @@ -48,7 +51,8 @@ public long getDelay(TimeUnit unit) { return unit.convert( DateUtils.getRemainTime( DateUtils.timeStampToDate(taskExecutionContext.getFirstSubmitTime()), - taskExecutionContext.getDelayTime() * 60L), TimeUnit.SECONDS); + taskExecutionContext.getDelayTime() * 60L), + TimeUnit.SECONDS); } @Override diff --git a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/runner/WorkerDelayTaskExecuteRunnableFactory.java b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/runner/WorkerDelayTaskExecuteRunnableFactory.java index 7db96fc3808b..d8677b46807c 100644 --- a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/runner/WorkerDelayTaskExecuteRunnableFactory.java +++ b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/runner/WorkerDelayTaskExecuteRunnableFactory.java @@ -17,7 +17,6 @@ package org.apache.dolphinscheduler.server.worker.runner; -import lombok.NonNull; import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext; import org.apache.dolphinscheduler.server.worker.config.WorkerConfig; import org.apache.dolphinscheduler.server.worker.rpc.WorkerMessageSender; @@ -27,7 +26,11 @@ import javax.annotation.Nullable; -public abstract class WorkerDelayTaskExecuteRunnableFactory implements WorkerTaskExecuteRunnableFactory { +import lombok.NonNull; + +public abstract class WorkerDelayTaskExecuteRunnableFactory + implements + WorkerTaskExecuteRunnableFactory { protected final @NonNull TaskExecutionContext taskExecutionContext; protected final @NonNull WorkerConfig workerConfig; @@ -38,13 +41,13 @@ public abstract class WorkerDelayTaskExecuteRunnableFactory getTaskExecuteThreadMap() { return taskExecuteThreadMap; } -} \ No newline at end of file +} diff --git a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/runner/WorkerManagerThread.java b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/runner/WorkerManagerThread.java index 1e6a0ba6b723..7ccaee2a5e9b 100644 --- a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/runner/WorkerManagerThread.java +++ b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/runner/WorkerManagerThread.java @@ -23,14 +23,16 @@ import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContextCacheManager; import org.apache.dolphinscheduler.server.worker.config.WorkerConfig; import org.apache.dolphinscheduler.server.worker.metrics.WorkerServerMetrics; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.springframework.stereotype.Component; -import javax.annotation.Nullable; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.DelayQueue; +import javax.annotation.Nullable; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.stereotype.Component; + /** * Manage tasks */ @@ -48,7 +50,8 @@ public class WorkerManagerThread implements Runnable { /** * running task */ - private final ConcurrentHashMap taskExecuteThreadMap = new ConcurrentHashMap<>(); + private final ConcurrentHashMap taskExecuteThreadMap = + new ConcurrentHashMap<>(); public WorkerManagerThread(WorkerConfig workerConfig) { workerExecThreads = workerConfig.getExecThreads(); diff --git a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/runner/WorkerTaskExecuteRunnable.java b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/runner/WorkerTaskExecuteRunnable.java index ebc358b026a7..d0df1dcb31e7 100644 --- a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/runner/WorkerTaskExecuteRunnable.java +++ b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/runner/WorkerTaskExecuteRunnable.java @@ -45,11 +45,9 @@ import org.apache.dolphinscheduler.service.utils.LoggerUtils; import org.apache.dolphinscheduler.service.utils.ProcessUtils; -import javax.annotation.Nullable; import java.io.File; import java.io.IOException; import java.nio.file.NoSuchFileException; -import java.util.Date; import java.util.List; import javax.annotation.Nullable; @@ -92,11 +90,12 @@ protected WorkerTaskExecuteRunnable( this.alertClientService = alertClientService; this.taskPluginManager = taskPluginManager; this.storageOperate = storageOperate; - String taskLogName = LoggerUtils.buildTaskId(DateUtils.timeStampToDate(taskExecutionContext.getFirstSubmitTime()), - taskExecutionContext.getProcessDefineCode(), - taskExecutionContext.getProcessDefineVersion(), - taskExecutionContext.getProcessInstanceId(), - taskExecutionContext.getTaskInstanceId()); + String taskLogName = + LoggerUtils.buildTaskId(DateUtils.timeStampToDate(taskExecutionContext.getFirstSubmitTime()), + taskExecutionContext.getProcessDefineCode(), + taskExecutionContext.getProcessDefineVersion(), + taskExecutionContext.getProcessInstanceId(), + taskExecutionContext.getTaskInstanceId()); taskExecutionContext.setTaskLogName(taskLogName); logger.info("Set task logger name: {}", taskLogName); } diff --git a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/runner/WorkerTaskExecuteRunnableFactoryBuilder.java b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/runner/WorkerTaskExecuteRunnableFactoryBuilder.java index fee1635be00e..8ec586397fb9 100644 --- a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/runner/WorkerTaskExecuteRunnableFactoryBuilder.java +++ b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/runner/WorkerTaskExecuteRunnableFactoryBuilder.java @@ -17,8 +17,6 @@ package org.apache.dolphinscheduler.server.worker.runner; -import lombok.NonNull; -import lombok.experimental.UtilityClass; import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext; import org.apache.dolphinscheduler.server.worker.config.WorkerConfig; import org.apache.dolphinscheduler.server.worker.rpc.WorkerMessageSender; @@ -28,6 +26,9 @@ import javax.annotation.Nullable; +import lombok.NonNull; +import lombok.experimental.UtilityClass; + @UtilityClass public class WorkerTaskExecuteRunnableFactoryBuilder { diff --git a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/utils/TaskExecutionCheckerUtils.java b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/utils/TaskExecutionCheckerUtils.java index d0467a25dcc9..1aea96d9d83c 100644 --- a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/utils/TaskExecutionCheckerUtils.java +++ b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/utils/TaskExecutionCheckerUtils.java @@ -26,7 +26,7 @@ import org.apache.dolphinscheduler.server.worker.config.WorkerConfig; import org.apache.dolphinscheduler.server.worker.metrics.WorkerServerMetrics; import org.apache.dolphinscheduler.service.storage.StorageOperate; -import org.apache.dolphinscheduler.service.utils.CommonUtils; + import org.apache.commons.collections.CollectionUtils; import org.apache.commons.collections.MapUtils; import org.apache.commons.lang3.SystemUtils; diff --git a/dolphinscheduler-worker/src/test/java/org/apache/dolphinscheduler/server/worker/config/BeanConfigTest.java b/dolphinscheduler-worker/src/test/java/org/apache/dolphinscheduler/server/worker/config/BeanConfigTest.java index 93dcc9fee470..2323e591bfb9 100644 --- a/dolphinscheduler-worker/src/test/java/org/apache/dolphinscheduler/server/worker/config/BeanConfigTest.java +++ b/dolphinscheduler-worker/src/test/java/org/apache/dolphinscheduler/server/worker/config/BeanConfigTest.java @@ -18,6 +18,7 @@ package org.apache.dolphinscheduler.server.worker.config; import org.apache.dolphinscheduler.service.alert.AlertClientService; + import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; @@ -36,4 +37,4 @@ public class BeanConfigTest { public void alertClientService() { Assertions.assertNotNull(alertClientService); } -} \ No newline at end of file +} diff --git a/dolphinscheduler-worker/src/test/java/org/apache/dolphinscheduler/server/worker/processor/TaskDispatchProcessorTest.java b/dolphinscheduler-worker/src/test/java/org/apache/dolphinscheduler/server/worker/processor/TaskDispatchProcessorTest.java index 049d01ac29ee..c2e0ce4d8a33 100644 --- a/dolphinscheduler-worker/src/test/java/org/apache/dolphinscheduler/server/worker/processor/TaskDispatchProcessorTest.java +++ b/dolphinscheduler-worker/src/test/java/org/apache/dolphinscheduler/server/worker/processor/TaskDispatchProcessorTest.java @@ -17,7 +17,6 @@ package org.apache.dolphinscheduler.server.worker.processor; -import io.netty.channel.Channel; import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext; import org.apache.dolphinscheduler.remote.command.Command; import org.apache.dolphinscheduler.remote.command.CommandType; @@ -28,6 +27,7 @@ import org.apache.dolphinscheduler.service.alert.AlertClientService; import org.apache.dolphinscheduler.service.storage.StorageOperate; import org.apache.dolphinscheduler.service.task.TaskPluginManager; + import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; import org.mockito.InjectMocks; @@ -35,6 +35,8 @@ import org.mockito.Mockito; import org.mockito.junit.jupiter.MockitoExtension; +import io.netty.channel.Channel; + /** * test task execute processor */ diff --git a/dolphinscheduler-worker/src/test/java/org/apache/dolphinscheduler/server/worker/runner/DefaultWorkerDelayTaskExecuteRunnableTest.java b/dolphinscheduler-worker/src/test/java/org/apache/dolphinscheduler/server/worker/runner/DefaultWorkerDelayTaskExecuteRunnableTest.java index f0189df876a5..929c80a62ca9 100644 --- a/dolphinscheduler-worker/src/test/java/org/apache/dolphinscheduler/server/worker/runner/DefaultWorkerDelayTaskExecuteRunnableTest.java +++ b/dolphinscheduler-worker/src/test/java/org/apache/dolphinscheduler/server/worker/runner/DefaultWorkerDelayTaskExecuteRunnableTest.java @@ -25,8 +25,9 @@ import org.apache.dolphinscheduler.service.alert.AlertClientService; import org.apache.dolphinscheduler.service.storage.StorageOperate; import org.apache.dolphinscheduler.service.task.TaskPluginManager; -import org.junit.jupiter.api.Test; + import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; import org.mockito.Mockito; public class DefaultWorkerDelayTaskExecuteRunnableTest { @@ -61,8 +62,7 @@ public void testDryRun() { workerMessageSender, alertClientService, taskPluginManager, - storageOperate - ); + storageOperate); Assertions.assertAll(workerTaskExecuteRunnable::run); Assertions.assertEquals(TaskExecutionStatus.SUCCESS, taskExecutionContext.getCurrentExecutionStatus()); @@ -78,7 +78,8 @@ public void testErrorboundTestDataSource() { .firstSubmitTime(System.currentTimeMillis()) .taskLogName("TestLogName") .taskType("SQL") - .taskParams("{\"localParams\":[],\"resourceList\":[],\"type\":\"POSTGRESQL\",\"datasource\":null,\"sql\":\"select * from t_ds_user\",\"sqlType\":\"0\",\"preStatements\":[],\"postStatements\":[],\"segmentSeparator\":\"\",\"displayRows\":10,\"conditionResult\":\"null\",\"dependence\":\"null\",\"switchResult\":\"null\",\"waitStartTimeout\":null}") + .taskParams( + "{\"localParams\":[],\"resourceList\":[],\"type\":\"POSTGRESQL\",\"datasource\":null,\"sql\":\"select * from t_ds_user\",\"sqlType\":\"0\",\"preStatements\":[],\"postStatements\":[],\"segmentSeparator\":\"\",\"displayRows\":10,\"conditionResult\":\"null\",\"dependence\":\"null\",\"switchResult\":\"null\",\"waitStartTimeout\":null}") .build(); WorkerTaskExecuteRunnable workerTaskExecuteRunnable = new DefaultWorkerDelayTaskExecuteRunnable( taskExecutionContext, @@ -87,8 +88,7 @@ public void testErrorboundTestDataSource() { workerMessageSender, alertClientService, taskPluginManager, - storageOperate - ); + storageOperate); Assertions.assertAll(workerTaskExecuteRunnable::run); Assertions.assertEquals(TaskExecutionStatus.FAILURE, taskExecutionContext.getCurrentExecutionStatus()); diff --git a/pom.xml b/pom.xml index e368b4f5a232..87f5b5e723fe 100644 --- a/pom.xml +++ b/pom.xml @@ -74,7 +74,7 @@ 3.1.1 3.2.1 2.2.0 - 2.23.0 + 2.27.2 0.8.8 false 2.7 @@ -639,9 +639,12 @@ spotless-maven-plugin ${spotless.version} - - HEAD + + + **/e2e/**/*.java + **/microbench/**/*.java + style/spotless_dolphinscheduler_formatter.xml @@ -687,8 +690,12 @@ - **/*.md + docs/**/*.md + + + **/.github/**/*.md + diff --git a/tools/release/README.md b/tools/release/README.md index b4596f0f8885..d2bd5ca61ac0 100644 --- a/tools/release/README.md +++ b/tools/release/README.md @@ -36,3 +36,4 @@ python release.py -h * Auto cherry-pick: `python release.py cherry-pick`, will cause error when your default branch is not up-to-date, or cherry-pick with conflict. But if you fix you can directly re-run this command, it will continue the pick * Generate changelog: `python release.py changelog` * Generate contributor: `python release.py contributor` +