From 2bb569cbb1036512b9f8aefa7b4f96fc7c09f45b Mon Sep 17 00:00:00 2001 From: zhanqian <360400325@qq.com> Date: Thu, 10 Mar 2022 20:01:44 +0800 Subject: [PATCH 1/7] issue 8645 (#8811) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: 北笙 <“zhanqian@cai-inc.com”> Co-authored-by: zhanqian --- .../api/service/impl/ProcessDefinitionServiceImpl.java | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ProcessDefinitionServiceImpl.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ProcessDefinitionServiceImpl.java index b5fd941c8780..cc7a437effe3 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ProcessDefinitionServiceImpl.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ProcessDefinitionServiceImpl.java @@ -925,16 +925,20 @@ private boolean checkAndImport(User loginUser, long projectCode, Map checkResult = verifyProcessDefinitionName(loginUser, projectCode, processDefinition.getName()); + Map checkResult = verifyProcessDefinitionName(loginUser, projectCode, importProcessDefinitionName); if (Status.SUCCESS.equals(checkResult.get(Constants.STATUS))) { putMsg(result, Status.SUCCESS); } else { result.putAll(checkResult); return false; } - String processDefinitionName = recursionProcessDefinitionName(projectCode, processDefinition.getName(), 1); - processDefinition.setName(processDefinitionName + "_import_" + DateUtils.getCurrentTimeStamp()); + processDefinition.setName(importProcessDefinitionName); processDefinition.setId(0); processDefinition.setProjectCode(projectCode); processDefinition.setUserId(loginUser.getId()); From 12b9d582ba0529fef1041d56fecdf59199b72cf8 Mon Sep 17 00:00:00 2001 From: shuai hou <120306274@qq.com> Date: Fri, 25 Mar 2022 13:40:20 +0800 Subject: [PATCH 2/7] [fix-8929][DAO]Description Failed to upload the file because the full name was too long (#9020) * [fix-8929][DAO]Description Failed to upload the file because the full name was too long * [DS-#8929][fix]Description Failed to upload the file because the full name was too long * [DS-#8929][fix]Description Failed to upload the file because the full name was too long * [DS-#8929][fix]Description Failed to upload the file because the full name was too long Co-authored-by: houshuai --- .../resources/sql/dolphinscheduler_h2.sql | 2 +- .../resources/sql/dolphinscheduler_mysql.sql | 2 +- .../sql/dolphinscheduler_postgresql.sql | 2 +- .../mysql/dolphinscheduler_ddl.sql | 38 ++++++++++++++++ .../mysql/dolphinscheduler_dml.sql | 16 +++++++ .../postgresql/dolphinscheduler_ddl.sql | 44 +++++++++++++++++++ .../postgresql/dolphinscheduler_dml.sql | 17 +++++++ 7 files changed, 118 insertions(+), 3 deletions(-) create mode 100644 dolphinscheduler-dao/src/main/resources/sql/upgrade/2.0.6_schema/mysql/dolphinscheduler_ddl.sql create mode 100644 dolphinscheduler-dao/src/main/resources/sql/upgrade/2.0.6_schema/mysql/dolphinscheduler_dml.sql create mode 100644 dolphinscheduler-dao/src/main/resources/sql/upgrade/2.0.6_schema/postgresql/dolphinscheduler_ddl.sql create mode 100644 dolphinscheduler-dao/src/main/resources/sql/upgrade/2.0.6_schema/postgresql/dolphinscheduler_dml.sql diff --git a/dolphinscheduler-dao/src/main/resources/sql/dolphinscheduler_h2.sql b/dolphinscheduler-dao/src/main/resources/sql/dolphinscheduler_h2.sql index 66862e11d9f0..c75ad0313b19 100644 --- a/dolphinscheduler-dao/src/main/resources/sql/dolphinscheduler_h2.sql +++ b/dolphinscheduler-dao/src/main/resources/sql/dolphinscheduler_h2.sql @@ -755,7 +755,7 @@ CREATE TABLE t_ds_resources create_time datetime DEFAULT NULL, update_time datetime DEFAULT NULL, pid int(11) DEFAULT NULL, - full_name varchar(64) DEFAULT NULL, + full_name varchar(128) DEFAULT NULL, is_directory tinyint(4) DEFAULT NULL, PRIMARY KEY (id), UNIQUE KEY t_ds_resources_un (full_name, type) diff --git a/dolphinscheduler-dao/src/main/resources/sql/dolphinscheduler_mysql.sql b/dolphinscheduler-dao/src/main/resources/sql/dolphinscheduler_mysql.sql index 5f47551fc53b..a58ac66dd60c 100644 --- a/dolphinscheduler-dao/src/main/resources/sql/dolphinscheduler_mysql.sql +++ b/dolphinscheduler-dao/src/main/resources/sql/dolphinscheduler_mysql.sql @@ -749,7 +749,7 @@ CREATE TABLE `t_ds_resources` ( `create_time` datetime DEFAULT NULL COMMENT 'create time', `update_time` datetime DEFAULT NULL COMMENT 'update time', `pid` int(11) DEFAULT NULL, - `full_name` varchar(64) DEFAULT NULL, + `full_name` varchar(128) DEFAULT NULL, `is_directory` tinyint(4) DEFAULT NULL, PRIMARY KEY (`id`), UNIQUE KEY `t_ds_resources_un` (`full_name`,`type`) diff --git a/dolphinscheduler-dao/src/main/resources/sql/dolphinscheduler_postgresql.sql b/dolphinscheduler-dao/src/main/resources/sql/dolphinscheduler_postgresql.sql index 82bb412a6b58..93c552f95efa 100644 --- a/dolphinscheduler-dao/src/main/resources/sql/dolphinscheduler_postgresql.sql +++ b/dolphinscheduler-dao/src/main/resources/sql/dolphinscheduler_postgresql.sql @@ -640,7 +640,7 @@ CREATE TABLE t_ds_resources ( create_time timestamp DEFAULT NULL , update_time timestamp DEFAULT NULL , pid int, - full_name varchar(64), + full_name varchar(128), is_directory boolean DEFAULT FALSE, PRIMARY KEY (id), CONSTRAINT t_ds_resources_un UNIQUE (full_name, type) diff --git a/dolphinscheduler-dao/src/main/resources/sql/upgrade/2.0.6_schema/mysql/dolphinscheduler_ddl.sql b/dolphinscheduler-dao/src/main/resources/sql/upgrade/2.0.6_schema/mysql/dolphinscheduler_ddl.sql new file mode 100644 index 000000000000..45f8acd4da5f --- /dev/null +++ b/dolphinscheduler-dao/src/main/resources/sql/upgrade/2.0.6_schema/mysql/dolphinscheduler_ddl.sql @@ -0,0 +1,38 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +SET sql_mode=(SELECT REPLACE(@@sql_mode,'ONLY_FULL_GROUP_BY','')); + +-- uc_dolphin_T_t_ds_resources_R_full_name +drop PROCEDURE if EXISTS uc_dolphin_T_t_ds_resources_R_full_name; +delimiter d// +CREATE PROCEDURE uc_dolphin_T_t_ds_resources_R_full_name() +BEGIN + IF EXISTS (SELECT 1 FROM information_schema.COLUMNS + WHERE TABLE_NAME='t_ds_resources' + AND TABLE_SCHEMA=(SELECT DATABASE()) + AND COLUMN_NAME ='full_name') + THEN +ALTER TABLE t_ds_resources MODIFY COLUMN `full_name` varchar(128); +END IF; +END; + +d// + +delimiter ; +CALL uc_dolphin_T_t_ds_resources_R_full_name; +DROP PROCEDURE uc_dolphin_T_t_ds_resources_R_full_name; diff --git a/dolphinscheduler-dao/src/main/resources/sql/upgrade/2.0.6_schema/mysql/dolphinscheduler_dml.sql b/dolphinscheduler-dao/src/main/resources/sql/upgrade/2.0.6_schema/mysql/dolphinscheduler_dml.sql new file mode 100644 index 000000000000..4a14f326b985 --- /dev/null +++ b/dolphinscheduler-dao/src/main/resources/sql/upgrade/2.0.6_schema/mysql/dolphinscheduler_dml.sql @@ -0,0 +1,16 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ diff --git a/dolphinscheduler-dao/src/main/resources/sql/upgrade/2.0.6_schema/postgresql/dolphinscheduler_ddl.sql b/dolphinscheduler-dao/src/main/resources/sql/upgrade/2.0.6_schema/postgresql/dolphinscheduler_ddl.sql new file mode 100644 index 000000000000..14a20fcd8e7b --- /dev/null +++ b/dolphinscheduler-dao/src/main/resources/sql/upgrade/2.0.6_schema/postgresql/dolphinscheduler_ddl.sql @@ -0,0 +1,44 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +delimiter d// +CREATE OR REPLACE FUNCTION public.dolphin_update_metadata( + ) + RETURNS character varying + LANGUAGE 'plpgsql' + COST 100 + VOLATILE PARALLEL UNSAFE +AS $BODY$ +DECLARE + v_schema varchar; +BEGIN + ---get schema name + v_schema =current_schema(); + + --- alter column + EXECUTE 'ALTER TABLE ' || quote_ident(v_schema) ||'.t_ds_resources ALTER COLUMN full_name Type varchar(128)'; + + return 'Success!'; + exception when others then + ---Raise EXCEPTION '(%)',SQLERRM; + return SQLERRM; +END; +$BODY$; + +select dolphin_update_metadata(); + +d// \ No newline at end of file diff --git a/dolphinscheduler-dao/src/main/resources/sql/upgrade/2.0.6_schema/postgresql/dolphinscheduler_dml.sql b/dolphinscheduler-dao/src/main/resources/sql/upgrade/2.0.6_schema/postgresql/dolphinscheduler_dml.sql new file mode 100644 index 000000000000..5f26e3515d67 --- /dev/null +++ b/dolphinscheduler-dao/src/main/resources/sql/upgrade/2.0.6_schema/postgresql/dolphinscheduler_dml.sql @@ -0,0 +1,17 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + From b84815c8e48333c38b6afe4d278fe1eb63356534 Mon Sep 17 00:00:00 2001 From: Tq Date: Thu, 28 Apr 2022 18:16:37 +0800 Subject: [PATCH 3/7] [BUG][ALERT-SERVER]validate script before alert script (#9834) * validate script before alert script * fix validate script before alert script --- .../plugin/alert/script/ScriptSender.java | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-script/src/main/java/org/apache/dolphinscheduler/plugin/alert/script/ScriptSender.java b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-script/src/main/java/org/apache/dolphinscheduler/plugin/alert/script/ScriptSender.java index a4dbe1eabc29..0d422f279b6b 100644 --- a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-script/src/main/java/org/apache/dolphinscheduler/plugin/alert/script/ScriptSender.java +++ b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-script/src/main/java/org/apache/dolphinscheduler/plugin/alert/script/ScriptSender.java @@ -18,11 +18,11 @@ package org.apache.dolphinscheduler.plugin.alert.script; import org.apache.dolphinscheduler.alert.api.AlertResult; +import org.slf4j.Logger; +import java.io.File; import java.util.Map; -import org.slf4j.Logger; - public final class ScriptSender { private static final Logger log = org.slf4j.LoggerFactory.getLogger(ScriptSender.class); private static final String ALERT_TITLE_OPTION = " -t "; @@ -53,6 +53,21 @@ private AlertResult executeShellScript(String title, String content) { alertResult.setMessage("shell script not support windows os"); return alertResult; } + //validate script path in case of injections + File shellScriptFile = new File(scriptPath); + //validate existence + if (!shellScriptFile.exists()) { + logger.error("shell script not exist : {}", scriptPath); + alertResult.setMessage("shell script not exist : " + scriptPath); + return alertResult; + } + //validate is file + if (!shellScriptFile.isFile()) { + logger.error("shell script is not a file : {}", scriptPath); + alertResult.setMessage("shell script is not a file : " + scriptPath); + return alertResult; + } + String[] cmd = {"/bin/sh", "-c", scriptPath + ALERT_TITLE_OPTION + "'" + title + "'" + ALERT_CONTENT_OPTION + "'" + content + "'" + ALERT_USER_PARAMS_OPTION + "'" + userParams + "'"}; int exitCode = ProcessUtils.executeScript(cmd); From 29066ab20c8280877925d9f3846d4f37a75b4d42 Mon Sep 17 00:00:00 2001 From: JinyLeeChina Date: Sun, 3 Jul 2022 15:38:42 +0800 Subject: [PATCH 4/7] pick 8811/9020/9834 --- script/dolphinscheduler-daemon.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/script/dolphinscheduler-daemon.sh b/script/dolphinscheduler-daemon.sh index d26ec85c0cf6..c2823f04ea8f 100755 --- a/script/dolphinscheduler-daemon.sh +++ b/script/dolphinscheduler-daemon.sh @@ -35,7 +35,7 @@ BIN_DIR=`dirname $0` BIN_DIR=`cd "$BIN_DIR"; pwd` export DOLPHINSCHEDULER_HOME=`cd "$BIN_DIR/.."; pwd` -chmod -R 700 ${DOLPHINSCHEDULER_HOME}/config +chmod -R 700 ${DOLPHINSCHEDULER_HOME}/conf/config source /etc/profile set -a source "${DOLPHINSCHEDULER_HOME}/conf/env/dolphinscheduler_env.sh" From c2aaca011082e7883858131820a232283f675bf8 Mon Sep 17 00:00:00 2001 From: JinyLeeChina Date: Sun, 3 Jul 2022 16:59:17 +0800 Subject: [PATCH 5/7] fix ut --- .../dolphinscheduler/plugin/alert/script/ScriptSender.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-script/src/main/java/org/apache/dolphinscheduler/plugin/alert/script/ScriptSender.java b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-script/src/main/java/org/apache/dolphinscheduler/plugin/alert/script/ScriptSender.java index 0d422f279b6b..2791fc1d96a6 100644 --- a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-script/src/main/java/org/apache/dolphinscheduler/plugin/alert/script/ScriptSender.java +++ b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-script/src/main/java/org/apache/dolphinscheduler/plugin/alert/script/ScriptSender.java @@ -57,13 +57,13 @@ private AlertResult executeShellScript(String title, String content) { File shellScriptFile = new File(scriptPath); //validate existence if (!shellScriptFile.exists()) { - logger.error("shell script not exist : {}", scriptPath); + log.error("shell script not exist : {}", scriptPath); alertResult.setMessage("shell script not exist : " + scriptPath); return alertResult; } //validate is file if (!shellScriptFile.isFile()) { - logger.error("shell script is not a file : {}", scriptPath); + log.error("shell script is not a file : {}", scriptPath); alertResult.setMessage("shell script is not a file : " + scriptPath); return alertResult; } From 0601ecad3c4205f5c8c9774d04f460e5c5d86bee Mon Sep 17 00:00:00 2001 From: JinyLeeChina Date: Mon, 4 Jul 2022 11:17:26 +0800 Subject: [PATCH 6/7] fix pick error --- .../dolphinscheduler/plugin/alert/script/ScriptSender.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-script/src/main/java/org/apache/dolphinscheduler/plugin/alert/script/ScriptSender.java b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-script/src/main/java/org/apache/dolphinscheduler/plugin/alert/script/ScriptSender.java index 0d422f279b6b..2791fc1d96a6 100644 --- a/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-script/src/main/java/org/apache/dolphinscheduler/plugin/alert/script/ScriptSender.java +++ b/dolphinscheduler-alert/dolphinscheduler-alert-plugins/dolphinscheduler-alert-script/src/main/java/org/apache/dolphinscheduler/plugin/alert/script/ScriptSender.java @@ -57,13 +57,13 @@ private AlertResult executeShellScript(String title, String content) { File shellScriptFile = new File(scriptPath); //validate existence if (!shellScriptFile.exists()) { - logger.error("shell script not exist : {}", scriptPath); + log.error("shell script not exist : {}", scriptPath); alertResult.setMessage("shell script not exist : " + scriptPath); return alertResult; } //validate is file if (!shellScriptFile.isFile()) { - logger.error("shell script is not a file : {}", scriptPath); + log.error("shell script is not a file : {}", scriptPath); alertResult.setMessage("shell script is not a file : " + scriptPath); return alertResult; } From b27f2d3dbe9abba38da27d7b1f88b1f7a8f39cc4 Mon Sep 17 00:00:00 2001 From: JinyLeeChina Date: Mon, 4 Jul 2022 15:08:43 +0800 Subject: [PATCH 7/7] fix 10762 --- .../service/impl/ResourcesServiceImpl.java | 1 + .../server/utils/DependentExecute.java | 19 ++++++++++++------- 2 files changed, 13 insertions(+), 7 deletions(-) diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ResourcesServiceImpl.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ResourcesServiceImpl.java index 3698ea0bf87f..c00d45461455 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ResourcesServiceImpl.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ResourcesServiceImpl.java @@ -64,6 +64,7 @@ import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.Set; import java.util.UUID; import java.util.regex.Matcher; diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/utils/DependentExecute.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/utils/DependentExecute.java index 77af8699f552..a26e8c6e29d5 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/utils/DependentExecute.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/utils/DependentExecute.java @@ -115,16 +115,16 @@ private DependResult calculateResultForTasks(DependentItem dependentItem, List