From d596593c07eac399d5b28bcd133af4832c274eb6 Mon Sep 17 00:00:00 2001 From: xdu-chenrj Date: Sat, 1 Jul 2023 00:05:41 +0800 Subject: [PATCH 01/15] support for zeppelin connections in the connection center and external connections in zeppelin tasks --- .../service/impl/DataSourceServiceImpl.java | 9 ++ .../dolphinscheduler-datasource-all/pom.xml | 5 + .../pom.xml | 36 +++++ .../zeppelin/ZeppelinDataSourceChannel.java | 30 ++++ .../ZeppelinDataSourceChannelFactory.java | 36 +++++ .../zeppelin/ZeppelinDataSourceClient.java | 35 +++++ .../param/ZeppelinConnectionParam.java | 33 +++++ .../param/ZeppelinDataSourceParamDTO.java | 32 +++++ .../param/ZeppelinDataSourceProcessor.java | 131 ++++++++++++++++++ dolphinscheduler-datasource-plugin/pom.xml | 1 + .../e2e/pages/datasource/DataSourcePage.java | 6 + .../dolphinscheduler/spi/enums/DbType.java | 3 +- .../dolphinscheduler-task-zeppelin/pom.xml | 5 + .../task/zeppelin/ZeppelinParameters.java | 31 ++++- .../plugin/task/zeppelin/ZeppelinTask.java | 33 ++--- .../task/zeppelin/ZeppelinTaskChannel.java | 2 +- .../ZeppelinTaskExecutionContext.java | 46 ++++++ .../src/locales/zh_CN/datasource.ts | 14 +- .../src/service/modules/data-source/types.ts | 4 + .../src/views/datasource/list/detail.tsx | 16 +++ .../src/views/datasource/list/use-form.ts | 41 ++++-- .../components/node/fields/use-datasource.ts | 5 + .../components/node/fields/use-zeppelin.ts | 33 ----- .../task/components/node/format-data.ts | 2 + .../components/node/tasks/use-zeppelin.ts | 8 +- 25 files changed, 518 insertions(+), 79 deletions(-) create mode 100644 dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-zeppelin/pom.xml create mode 100644 dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-zeppelin/src/main/java/org/apache/dolphinscheduler/plugin/datasource/zeppelin/ZeppelinDataSourceChannel.java create mode 100644 dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-zeppelin/src/main/java/org/apache/dolphinscheduler/plugin/datasource/zeppelin/ZeppelinDataSourceChannelFactory.java create mode 100644 dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-zeppelin/src/main/java/org/apache/dolphinscheduler/plugin/datasource/zeppelin/ZeppelinDataSourceClient.java create mode 100644 dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-zeppelin/src/main/java/org/apache/dolphinscheduler/plugin/datasource/zeppelin/param/ZeppelinConnectionParam.java create mode 100644 dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-zeppelin/src/main/java/org/apache/dolphinscheduler/plugin/datasource/zeppelin/param/ZeppelinDataSourceParamDTO.java create mode 100644 dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-zeppelin/src/main/java/org/apache/dolphinscheduler/plugin/datasource/zeppelin/param/ZeppelinDataSourceProcessor.java create mode 100644 dolphinscheduler-task-plugin/dolphinscheduler-task-zeppelin/src/main/java/org/apache/dolphinscheduler/plugin/task/zeppelin/ZeppelinTaskExecutionContext.java diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/DataSourceServiceImpl.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/DataSourceServiceImpl.java index fc39e1b72dcc..e679756964b6 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/DataSourceServiceImpl.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/DataSourceServiceImpl.java @@ -394,6 +394,15 @@ public Result checkConnection(DbType type, ConnectionParam connectionPar } return result; } + if (type == DbType.ZEPPELIN) { + DataSourceProcessor zeppelinDataSourceProcessor = DataSourceUtils.getDatasourceProcessor(type); + if (zeppelinDataSourceProcessor.testConnection(connectionParam)) { + putMsg(result, Status.SUCCESS); + } else { + putMsg(result, Status.CONNECT_DATASOURCE_FAILURE); + } + return result; + } try (Connection connection = DataSourceClientProvider.getInstance().getConnection(type, connectionParam)) { if (connection == null) { log.error("Connection test to {} datasource failed, connectionParam:{}.", type.getDescp(), diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-all/pom.xml b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-all/pom.xml index 5eda89cc1248..3b70c3d98902 100644 --- a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-all/pom.xml +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-all/pom.xml @@ -117,5 +117,10 @@ dolphinscheduler-datasource-kyuubi ${project.version} + + org.apache.dolphinscheduler + dolphinscheduler-datasource-zeppelin + ${project.version} + diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-zeppelin/pom.xml b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-zeppelin/pom.xml new file mode 100644 index 000000000000..2873104df667 --- /dev/null +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-zeppelin/pom.xml @@ -0,0 +1,36 @@ + + + 4.0.0 + + org.apache.dolphinscheduler + dolphinscheduler-datasource-plugin + dev-SNAPSHOT + + + dolphinscheduler-datasource-zeppelin + jar + ${project.artifactId} + + + + org.apache.dolphinscheduler + dolphinscheduler-spi + provided + + + + org.apache.dolphinscheduler + dolphinscheduler-datasource-api + ${project.version} + + + + org.apache.zeppelin + zeppelin-client + 0.10.1 + + + + \ No newline at end of file diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-zeppelin/src/main/java/org/apache/dolphinscheduler/plugin/datasource/zeppelin/ZeppelinDataSourceChannel.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-zeppelin/src/main/java/org/apache/dolphinscheduler/plugin/datasource/zeppelin/ZeppelinDataSourceChannel.java new file mode 100644 index 000000000000..1abc7e33b024 --- /dev/null +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-zeppelin/src/main/java/org/apache/dolphinscheduler/plugin/datasource/zeppelin/ZeppelinDataSourceChannel.java @@ -0,0 +1,30 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.plugin.datasource.zeppelin; + +import org.apache.dolphinscheduler.spi.datasource.BaseConnectionParam; +import org.apache.dolphinscheduler.spi.datasource.DataSourceChannel; +import org.apache.dolphinscheduler.spi.datasource.DataSourceClient; +import org.apache.dolphinscheduler.spi.enums.DbType; + +public class ZeppelinDataSourceChannel implements DataSourceChannel { + @Override + public DataSourceClient createDataSourceClient(BaseConnectionParam baseConnectionParam, DbType dbType) { + return new ZeppelinDataSourceClient(baseConnectionParam, dbType); + } +} diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-zeppelin/src/main/java/org/apache/dolphinscheduler/plugin/datasource/zeppelin/ZeppelinDataSourceChannelFactory.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-zeppelin/src/main/java/org/apache/dolphinscheduler/plugin/datasource/zeppelin/ZeppelinDataSourceChannelFactory.java new file mode 100644 index 000000000000..47d08e7df58f --- /dev/null +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-zeppelin/src/main/java/org/apache/dolphinscheduler/plugin/datasource/zeppelin/ZeppelinDataSourceChannelFactory.java @@ -0,0 +1,36 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.plugin.datasource.zeppelin; + +import com.google.auto.service.AutoService; +import org.apache.dolphinscheduler.spi.datasource.DataSourceChannel; +import org.apache.dolphinscheduler.spi.datasource.DataSourceChannelFactory; + +@AutoService(DataSourceChannelFactory.class) +public class ZeppelinDataSourceChannelFactory implements DataSourceChannelFactory { + @Override + public DataSourceChannel create() { + return new ZeppelinDataSourceChannel(); + } + + @Override + public String getName() { + return "zeppelin"; + } + +} diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-zeppelin/src/main/java/org/apache/dolphinscheduler/plugin/datasource/zeppelin/ZeppelinDataSourceClient.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-zeppelin/src/main/java/org/apache/dolphinscheduler/plugin/datasource/zeppelin/ZeppelinDataSourceClient.java new file mode 100644 index 000000000000..ea5907b95515 --- /dev/null +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-zeppelin/src/main/java/org/apache/dolphinscheduler/plugin/datasource/zeppelin/ZeppelinDataSourceClient.java @@ -0,0 +1,35 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.plugin.datasource.zeppelin; + +import org.apache.dolphinscheduler.plugin.datasource.api.client.CommonDataSourceClient; +import org.apache.dolphinscheduler.spi.datasource.BaseConnectionParam; +import org.apache.dolphinscheduler.spi.enums.DbType; + +import java.sql.Connection; + +public class ZeppelinDataSourceClient extends CommonDataSourceClient { + public ZeppelinDataSourceClient(BaseConnectionParam baseConnectionParam, DbType dbType) { + super(baseConnectionParam, dbType); + } + + @Override + public Connection getConnection() { + return (Connection) this.baseConnectionParam; + } +} diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-zeppelin/src/main/java/org/apache/dolphinscheduler/plugin/datasource/zeppelin/param/ZeppelinConnectionParam.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-zeppelin/src/main/java/org/apache/dolphinscheduler/plugin/datasource/zeppelin/param/ZeppelinConnectionParam.java new file mode 100644 index 000000000000..85bec74a17e2 --- /dev/null +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-zeppelin/src/main/java/org/apache/dolphinscheduler/plugin/datasource/zeppelin/param/ZeppelinConnectionParam.java @@ -0,0 +1,33 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.plugin.datasource.zeppelin.param; + +import com.fasterxml.jackson.annotation.JsonInclude; +import lombok.Data; +import org.apache.dolphinscheduler.spi.datasource.ConnectionParam; + +@Data +@JsonInclude(JsonInclude.Include.NON_NULL) +public class ZeppelinConnectionParam implements ConnectionParam { + + protected String username; + + protected String password; + + protected String restEndpoint; +} diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-zeppelin/src/main/java/org/apache/dolphinscheduler/plugin/datasource/zeppelin/param/ZeppelinDataSourceParamDTO.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-zeppelin/src/main/java/org/apache/dolphinscheduler/plugin/datasource/zeppelin/param/ZeppelinDataSourceParamDTO.java new file mode 100644 index 000000000000..9afef55a00bb --- /dev/null +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-zeppelin/src/main/java/org/apache/dolphinscheduler/plugin/datasource/zeppelin/param/ZeppelinDataSourceParamDTO.java @@ -0,0 +1,32 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.plugin.datasource.zeppelin.param; + +import lombok.Data; +import org.apache.dolphinscheduler.plugin.datasource.api.datasource.BaseDataSourceParamDTO; +import org.apache.dolphinscheduler.spi.enums.DbType; + +@Data +public class ZeppelinDataSourceParamDTO extends BaseDataSourceParamDTO { + protected String restEndpoint; + + @Override + public DbType getType() { + return DbType.ZEPPELIN; + } +} diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-zeppelin/src/main/java/org/apache/dolphinscheduler/plugin/datasource/zeppelin/param/ZeppelinDataSourceProcessor.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-zeppelin/src/main/java/org/apache/dolphinscheduler/plugin/datasource/zeppelin/param/ZeppelinDataSourceProcessor.java new file mode 100644 index 000000000000..1399d3ce2b64 --- /dev/null +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-zeppelin/src/main/java/org/apache/dolphinscheduler/plugin/datasource/zeppelin/param/ZeppelinDataSourceProcessor.java @@ -0,0 +1,131 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.plugin.datasource.zeppelin.param; + +import com.google.auto.service.AutoService; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.StringUtils; +import org.apache.dolphinscheduler.common.utils.JSONUtils; +import org.apache.dolphinscheduler.plugin.datasource.api.datasource.BaseDataSourceParamDTO; +import org.apache.dolphinscheduler.plugin.datasource.api.datasource.DataSourceProcessor; +import org.apache.dolphinscheduler.plugin.datasource.api.utils.PasswordUtils; +import org.apache.dolphinscheduler.spi.datasource.ConnectionParam; +import org.apache.dolphinscheduler.spi.enums.DbType; +import org.apache.zeppelin.client.ClientConfig; +import org.apache.zeppelin.client.ZeppelinClient; + +import java.sql.Connection; +import java.text.MessageFormat; + +@AutoService(DataSourceProcessor.class) +@Slf4j +public class ZeppelinDataSourceProcessor implements DataSourceProcessor { + @Override + public BaseDataSourceParamDTO castDatasourceParamDTO(String paramJson) { + return JSONUtils.parseObject(paramJson, ZeppelinDataSourceParamDTO.class); + } + + @Override + public void checkDatasourceParam(BaseDataSourceParamDTO datasourceParamDTO) { + ZeppelinDataSourceParamDTO zeppelinDataSourceParamDTO = (ZeppelinDataSourceParamDTO) datasourceParamDTO; + if (StringUtils.isEmpty(zeppelinDataSourceParamDTO.getRestEndpoint()) || StringUtils.isEmpty(zeppelinDataSourceParamDTO.getUserName())) { + throw new IllegalArgumentException("zeppelin datasource param is not valid"); + } + } + + @Override + public String getDatasourceUniqueId(ConnectionParam connectionParam, DbType dbType) { + ZeppelinConnectionParam baseConnectionParam = (ZeppelinConnectionParam) connectionParam; + return MessageFormat.format("{0}@{1}@{2}@{3}", dbType.getDescp(), baseConnectionParam.getRestEndpoint(), baseConnectionParam.getUsername(), PasswordUtils.encodePassword(baseConnectionParam.getPassword())); + } + + @Override + public BaseDataSourceParamDTO createDatasourceParamDTO(String connectionJson) { + ZeppelinConnectionParam connectionParams = (ZeppelinConnectionParam) createConnectionParams(connectionJson); + ZeppelinDataSourceParamDTO zeppelinDataSourceParamDTO = new ZeppelinDataSourceParamDTO(); + + zeppelinDataSourceParamDTO.setUserName(connectionParams.getUsername()); + zeppelinDataSourceParamDTO.setPassword(connectionParams.getPassword()); + zeppelinDataSourceParamDTO.setRestEndpoint(connectionParams.getRestEndpoint()); + return zeppelinDataSourceParamDTO; + } + + @Override + public ZeppelinConnectionParam createConnectionParams(BaseDataSourceParamDTO datasourceParam) { + ZeppelinDataSourceParamDTO zeppelinDataSourceParam = (ZeppelinDataSourceParamDTO) datasourceParam; + ZeppelinConnectionParam zeppelinConnectionParam = new ZeppelinConnectionParam(); + zeppelinConnectionParam.setUsername(zeppelinDataSourceParam.getUserName()); + zeppelinConnectionParam.setPassword(zeppelinDataSourceParam.getPassword()); + zeppelinConnectionParam.setRestEndpoint(zeppelinDataSourceParam.getRestEndpoint()); + + return zeppelinConnectionParam; + } + + @Override + public ConnectionParam createConnectionParams(String connectionJson) { + return JSONUtils.parseObject(connectionJson, ZeppelinConnectionParam.class); + } + + @Override + public String getDatasourceDriver() { + return ""; + } + + @Override + public String getValidationQuery() { + return ""; + } + + @Override + public String getJdbcUrl(ConnectionParam connectionParam) { + return ""; + } + + @Override + public Connection getConnection(ConnectionParam connectionParam) { + return null; + } + + @Override + public boolean testConnection(ConnectionParam connectionParam) { + ZeppelinConnectionParam baseConnectionParam = (ZeppelinConnectionParam) connectionParam; + ClientConfig clientConfig = new ClientConfig(baseConnectionParam.getRestEndpoint()); + + try { + // If the login fails, an exception will be thrown directly + ZeppelinClient zeppelinClient = new ZeppelinClient(clientConfig); + zeppelinClient.login(baseConnectionParam.username, baseConnectionParam.password); + String version = zeppelinClient.getVersion(); + log.info("zeppelin client connects to server successfully, version is {}", version); + return true; + } catch (Exception e) { + log.info("zeppelin client failed to connect to the server"); + return false; + } + } + + @Override + public DbType getDbType() { + return DbType.ZEPPELIN; + } + + @Override + public DataSourceProcessor create() { + return new ZeppelinDataSourceProcessor(); + } +} diff --git a/dolphinscheduler-datasource-plugin/pom.xml b/dolphinscheduler-datasource-plugin/pom.xml index 97ece76fa8dc..fd293603b3e9 100644 --- a/dolphinscheduler-datasource-plugin/pom.xml +++ b/dolphinscheduler-datasource-plugin/pom.xml @@ -48,6 +48,7 @@ dolphinscheduler-datasource-azure-sql dolphinscheduler-datasource-dameng dolphinscheduler-datasource-ssh + dolphinscheduler-datasource-zeppelin diff --git a/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/java/org/apache/dolphinscheduler/e2e/pages/datasource/DataSourcePage.java b/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/java/org/apache/dolphinscheduler/e2e/pages/datasource/DataSourcePage.java index 521cc6c97283..0a2661f5656a 100644 --- a/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/java/org/apache/dolphinscheduler/e2e/pages/datasource/DataSourcePage.java +++ b/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/java/org/apache/dolphinscheduler/e2e/pages/datasource/DataSourcePage.java @@ -197,5 +197,11 @@ public class CreateDataSourceForm { @FindBy(className = "btn-test-connection") private WebElement radioTestConnection; + @FindBys({ + @FindBy(className = "input-zeppelin_rest_endpoint"), + @FindBy(tagName = "input"), + }) + private WebElement inputZeppelinRestEndpoint; + } } diff --git a/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/enums/DbType.java b/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/enums/DbType.java index 3244f2b9ea22..ed9553642a49 100644 --- a/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/enums/DbType.java +++ b/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/enums/DbType.java @@ -46,7 +46,8 @@ public enum DbType { DAMENG(15, "dameng"), OCEANBASE(16, "oceanbase"), SSH(17, "ssh"), - KYUUBI(18, "kyuubi"); + KYUUBI(18, "kyuubi"), + ZEPPELIN(19, "zeppelin"); private static final Map DB_TYPE_MAP = Arrays.stream(DbType.values()).collect(toMap(DbType::getCode, Functions.identity())); diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-zeppelin/pom.xml b/dolphinscheduler-task-plugin/dolphinscheduler-task-zeppelin/pom.xml index 69a5a6699465..d136977526b6 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-zeppelin/pom.xml +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-zeppelin/pom.xml @@ -37,6 +37,11 @@ dolphinscheduler-task-api ${project.version} + + org.apache.dolphinscheduler + dolphinscheduler-datasource-all + ${project.version} + org.apache.zeppelin zeppelin-client diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-zeppelin/src/main/java/org/apache/dolphinscheduler/plugin/task/zeppelin/ZeppelinParameters.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-zeppelin/src/main/java/org/apache/dolphinscheduler/plugin/task/zeppelin/ZeppelinParameters.java index 8b1c1a341d34..605fa18a7bdc 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-zeppelin/src/main/java/org/apache/dolphinscheduler/plugin/task/zeppelin/ZeppelinParameters.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-zeppelin/src/main/java/org/apache/dolphinscheduler/plugin/task/zeppelin/ZeppelinParameters.java @@ -17,17 +17,19 @@ package org.apache.dolphinscheduler.plugin.task.zeppelin; +import lombok.Getter; +import lombok.Setter; +import lombok.ToString; +import org.apache.commons.lang3.StringUtils; +import org.apache.dolphinscheduler.plugin.task.api.enums.ResourceType; import org.apache.dolphinscheduler.plugin.task.api.model.ResourceInfo; import org.apache.dolphinscheduler.plugin.task.api.parameters.AbstractParameters; - -import org.apache.commons.lang3.StringUtils; +import org.apache.dolphinscheduler.plugin.task.api.parameters.resource.DataSourceParameters; +import org.apache.dolphinscheduler.plugin.task.api.parameters.resource.ResourceParametersHelper; import java.util.Collections; import java.util.List; - -import lombok.Getter; -import lombok.Setter; -import lombok.ToString; +import java.util.Objects; @Getter @Setter @@ -45,10 +47,12 @@ public class ZeppelinParameters extends AbstractParameters { private String parameters; private String username; private String password; + private int datasource; + private String type; @Override public boolean checkParameters() { - return StringUtils.isNotEmpty(this.noteId) && StringUtils.isNotEmpty(this.restEndpoint); + return StringUtils.isNotEmpty(this.noteId); } @Override @@ -56,4 +60,17 @@ public List getResourceFilesList() { return Collections.emptyList(); } + public ZeppelinTaskExecutionContext generateExtendedContext(ResourceParametersHelper parametersHelper) { + DataSourceParameters dataSourceParameters = (DataSourceParameters) parametersHelper.getResourceParameters(ResourceType.DATASOURCE, datasource); + ZeppelinTaskExecutionContext zeppelinTaskExecutionContext = new ZeppelinTaskExecutionContext(); + zeppelinTaskExecutionContext.setConnectionParams(Objects.nonNull(dataSourceParameters) ? dataSourceParameters.getConnectionParams() : null); + return zeppelinTaskExecutionContext; + } + + @Override + public ResourceParametersHelper getResources() { + ResourceParametersHelper resources = super.getResources(); + resources.put(ResourceType.DATASOURCE, datasource); + return resources; + } } diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-zeppelin/src/main/java/org/apache/dolphinscheduler/plugin/task/zeppelin/ZeppelinTask.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-zeppelin/src/main/java/org/apache/dolphinscheduler/plugin/task/zeppelin/ZeppelinTask.java index 0f25527c08f0..87a39183fd31 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-zeppelin/src/main/java/org/apache/dolphinscheduler/plugin/task/zeppelin/ZeppelinTask.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-zeppelin/src/main/java/org/apache/dolphinscheduler/plugin/task/zeppelin/ZeppelinTask.java @@ -17,31 +17,23 @@ package org.apache.dolphinscheduler.plugin.task.zeppelin; +import com.fasterxml.jackson.databind.ObjectMapper; +import kong.unirest.Unirest; +import org.apache.commons.lang3.StringUtils; import org.apache.dolphinscheduler.common.utils.DateUtils; import org.apache.dolphinscheduler.common.utils.JSONUtils; -import org.apache.dolphinscheduler.plugin.task.api.AbstractRemoteTask; -import org.apache.dolphinscheduler.plugin.task.api.TaskCallBack; -import org.apache.dolphinscheduler.plugin.task.api.TaskConstants; -import org.apache.dolphinscheduler.plugin.task.api.TaskException; -import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext; +import org.apache.dolphinscheduler.plugin.datasource.api.utils.DataSourceUtils; +import org.apache.dolphinscheduler.plugin.datasource.zeppelin.param.ZeppelinConnectionParam; +import org.apache.dolphinscheduler.plugin.task.api.*; import org.apache.dolphinscheduler.plugin.task.api.parameters.AbstractParameters; - -import org.apache.commons.lang3.StringUtils; -import org.apache.zeppelin.client.ClientConfig; -import org.apache.zeppelin.client.NoteResult; -import org.apache.zeppelin.client.ParagraphResult; -import org.apache.zeppelin.client.Status; -import org.apache.zeppelin.client.ZeppelinClient; +import org.apache.dolphinscheduler.spi.enums.DbType; +import org.apache.zeppelin.client.*; import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; -import kong.unirest.Unirest; - -import com.fasterxml.jackson.databind.ObjectMapper; - public class ZeppelinTask extends AbstractRemoteTask { /** @@ -59,6 +51,10 @@ public class ZeppelinTask extends AbstractRemoteTask { */ private ZeppelinClient zClient; + private ZeppelinConnectionParam zeppelinConnectionParam; + + private ZeppelinTaskExecutionContext zeppelinTaskExecutionContext; + /** * constructor * @@ -76,6 +72,11 @@ public void init() { if (this.zeppelinParameters == null || !this.zeppelinParameters.checkParameters()) { throw new ZeppelinTaskException("zeppelin task params is not valid"); } + zeppelinTaskExecutionContext = zeppelinParameters.generateExtendedContext(taskExecutionContext.getResourceParametersHelper()); + zeppelinConnectionParam = (ZeppelinConnectionParam) DataSourceUtils.buildConnectionParams(DbType.valueOf("ZEPPELIN"), zeppelinTaskExecutionContext.getConnectionParams()); + zeppelinParameters.setUsername(zeppelinConnectionParam.getUsername()); + zeppelinParameters.setPassword(zeppelinConnectionParam.getPassword()); + zeppelinParameters.setRestEndpoint(zeppelinConnectionParam.getRestEndpoint()); log.info("Initialize zeppelin task params:{}", JSONUtils.toPrettyJsonString(taskParams)); this.zClient = getZeppelinClient(); } diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-zeppelin/src/main/java/org/apache/dolphinscheduler/plugin/task/zeppelin/ZeppelinTaskChannel.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-zeppelin/src/main/java/org/apache/dolphinscheduler/plugin/task/zeppelin/ZeppelinTaskChannel.java index c2f63f3cf64e..d9e7318dc497 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-zeppelin/src/main/java/org/apache/dolphinscheduler/plugin/task/zeppelin/ZeppelinTaskChannel.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-zeppelin/src/main/java/org/apache/dolphinscheduler/plugin/task/zeppelin/ZeppelinTaskChannel.java @@ -44,6 +44,6 @@ public AbstractParameters parseParameters(ParametersNode parametersNode) { @Override public ResourceParametersHelper getResources(String parameters) { - return null; + return JSONUtils.parseObject(parameters, ZeppelinParameters.class).getResources(); } } diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-zeppelin/src/main/java/org/apache/dolphinscheduler/plugin/task/zeppelin/ZeppelinTaskExecutionContext.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-zeppelin/src/main/java/org/apache/dolphinscheduler/plugin/task/zeppelin/ZeppelinTaskExecutionContext.java new file mode 100644 index 000000000000..4cc09e5f4f49 --- /dev/null +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-zeppelin/src/main/java/org/apache/dolphinscheduler/plugin/task/zeppelin/ZeppelinTaskExecutionContext.java @@ -0,0 +1,46 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.plugin.task.zeppelin; + +import java.io.Serializable; + +/** + * master/worker task transport + */ +public class ZeppelinTaskExecutionContext implements Serializable { + + /** + * connectionParams + */ + private String connectionParams; + + public String getConnectionParams() { + return connectionParams; + } + + public void setConnectionParams(String connectionParams) { + this.connectionParams = connectionParams; + } + + @Override + public String toString() { + return "ZeppelinTaskExecutionContext{" + + "connectionParams='" + connectionParams + '\'' + + '}'; + } +} diff --git a/dolphinscheduler-ui/src/locales/zh_CN/datasource.ts b/dolphinscheduler-ui/src/locales/zh_CN/datasource.ts index 702ad9d6dd7c..c4c312b3b22e 100644 --- a/dolphinscheduler-ui/src/locales/zh_CN/datasource.ts +++ b/dolphinscheduler-ui/src/locales/zh_CN/datasource.ts @@ -85,10 +85,12 @@ export default { clientSecret: 'ClientSecret', OAuth_token_endpoint: 'OAuth 2.0 token endpoint', endpoint_tips: '请输入OAuth', - AccessKeyID:'AccessKeyID', - AccessKeyID_tips:'请输入AccessKeyID', - SecretAccessKey:'SecretAccessKey', - SecretAccessKey_tips:'请输入SecretAccessKey', - dbUser:'DbUser', - dbUser_tips:'请输入DbUser', + AccessKeyID: 'AccessKeyID', + AccessKeyID_tips: '请输入AccessKeyID', + SecretAccessKey: 'SecretAccessKey', + SecretAccessKey_tips: '请输入SecretAccessKey', + dbUser: 'DbUser', + dbUser_tips: '请输入DbUser', + zeppelin_rest_endpoint: 'zeppelinRestEndpoint', + zeppelin_rest_endpoint_tips: '请输入zeppelin server的rest endpoint' } diff --git a/dolphinscheduler-ui/src/service/modules/data-source/types.ts b/dolphinscheduler-ui/src/service/modules/data-source/types.ts index 10a200c9dbba..7ad81c8af4b5 100644 --- a/dolphinscheduler-ui/src/service/modules/data-source/types.ts +++ b/dolphinscheduler-ui/src/service/modules/data-source/types.ts @@ -33,6 +33,7 @@ type IDataBase = | 'DAMENG' | 'OCEANBASE' | 'SSH' + | 'ZEPPELIN' type IDataBaseLabel = | 'MYSQL' @@ -52,6 +53,8 @@ type IDataBaseLabel = | 'DAMENG' | 'OCEANBASE' | 'SSH' +| 'ZEPPELIN' + interface IDataSource { id?: number @@ -75,6 +78,7 @@ interface IDataSource { testFlag?: number bindTestId?: number endpoint?: string + restEndpoint?: string MSIClientId?: string dbUser?: string compatibleMode?: string diff --git a/dolphinscheduler-ui/src/views/datasource/list/detail.tsx b/dolphinscheduler-ui/src/views/datasource/list/detail.tsx index 8b5d6d2cf5d1..16f064e51a89 100644 --- a/dolphinscheduler-ui/src/views/datasource/list/detail.tsx +++ b/dolphinscheduler-ui/src/views/datasource/list/detail.tsx @@ -157,6 +157,7 @@ const DetailModal = defineComponent({ requiredDataBase, showHost, showPort, + showRestEndpoint, showAwsRegion, showCompatibleMode, showConnectType, @@ -246,6 +247,21 @@ const DetailModal = defineComponent({ placeholder={t('datasource.ip_tips')} /> + + + { - // @ts-ignore - if (state.detailForm.id && state.detailForm.id === value.id) - return false - return true - }) - .map((TestDataSourceExample: { name: string; id: number }) => ({ + .filter((value: { label: string; value: string }) => { + // @ts-ignore + if (state.detailForm.id && state.detailForm.id === value.id) + return false + return true + }) + .map((TestDataSourceExample: { name: string; id: number }) => ({ label: TestDataSourceExample.name, value: TestDataSourceExample.id })) @@ -409,9 +417,9 @@ export const datasourceType: IDataBaseOptionKeys = { defaultPort: 1433 }, STARROCKS: { - value: 'STARROCKS', - label: 'STARROCKS', - defaultPort: 9030 + value: 'STARROCKS', + label: 'STARROCKS', + defaultPort: 9030 }, DAMENG: { value: 'DAMENG', @@ -427,6 +435,11 @@ export const datasourceType: IDataBaseOptionKeys = { value: 'SSH', label: 'SSH', defaultPort: 22 + }, + ZEPPELIN: { + value: 'ZEPPELIN', + label: 'ZEPPELIN', + defaultPort: 8080 } } diff --git a/dolphinscheduler-ui/src/views/projects/task/components/node/fields/use-datasource.ts b/dolphinscheduler-ui/src/views/projects/task/components/node/fields/use-datasource.ts index 4f9f13294771..bfad0b7d055c 100644 --- a/dolphinscheduler-ui/src/views/projects/task/components/node/fields/use-datasource.ts +++ b/dolphinscheduler-ui/src/views/projects/task/components/node/fields/use-datasource.ts @@ -118,6 +118,11 @@ export function useDatasource( code: 'SSH', disabled: true }, + { + id: 16, + code: 'ZEPPELIN', + disabled: false + } ] const getDatasourceTypes = async () => { diff --git a/dolphinscheduler-ui/src/views/projects/task/components/node/fields/use-zeppelin.ts b/dolphinscheduler-ui/src/views/projects/task/components/node/fields/use-zeppelin.ts index a643e1a4bcef..eff477721557 100644 --- a/dolphinscheduler-ui/src/views/projects/task/components/node/fields/use-zeppelin.ts +++ b/dolphinscheduler-ui/src/views/projects/task/components/node/fields/use-zeppelin.ts @@ -47,23 +47,6 @@ export function useZeppelin(model: { [field: string]: any }): IJsonItem[] { placeholder: t('project.node.zeppelin_paragraph_id_tips') } }, - { - type: 'input', - field: 'restEndpoint', - name: t('project.node.zeppelin_rest_endpoint'), - props: { - placeholder: t('project.node.zeppelin_rest_endpoint_tips') - }, - validate: { - trigger: ['input', 'blur'], - required: true, - validator(validate: any, value: string) { - if (!value) { - return new Error(t('project.node.zeppelin_rest_endpoint_tips')) - } - } - } - }, { type: 'input', field: 'productionNoteDirectory', @@ -72,22 +55,6 @@ export function useZeppelin(model: { [field: string]: any }): IJsonItem[] { placeholder: t('project.node.zeppelin_production_note_directory_tips') } }, - { - type: 'input', - field: 'username', - name: t('project.node.zeppelin_username'), - props: { - placeholder: t('project.node.zeppelin_username_tips') - } - }, - { - type: 'input', - field: 'password', - name: t('project.node.zeppelin_password'), - props: { - placeholder: t('project.node.zeppelin_password_tips') - } - }, { type: 'input', field: 'parameters', diff --git a/dolphinscheduler-ui/src/views/projects/task/components/node/format-data.ts b/dolphinscheduler-ui/src/views/projects/task/components/node/format-data.ts index 09f6114c5e6e..06d304387dae 100644 --- a/dolphinscheduler-ui/src/views/projects/task/components/node/format-data.ts +++ b/dolphinscheduler-ui/src/views/projects/task/components/node/format-data.ts @@ -338,6 +338,8 @@ export function formatParams(data: INodeData): { taskParams.password = data.password taskParams.productionNoteDirectory = data.productionNoteDirectory taskParams.parameters = data.parameters + taskParams.datasource = data.datasource + taskParams.type = data.type } if (data.taskType === 'K8S') { diff --git a/dolphinscheduler-ui/src/views/projects/task/components/node/tasks/use-zeppelin.ts b/dolphinscheduler-ui/src/views/projects/task/components/node/tasks/use-zeppelin.ts index a3419f683e46..6516d013cff6 100644 --- a/dolphinscheduler-ui/src/views/projects/task/components/node/tasks/use-zeppelin.ts +++ b/dolphinscheduler-ui/src/views/projects/task/components/node/tasks/use-zeppelin.ts @@ -43,7 +43,12 @@ export function useZeppelin({ workerGroup: 'default', delayTime: 0, timeout: 30, - timeoutNotifyStrategy: ['WARN'] + type: 'ZEPPELIN', + displayRows: 10, + timeoutNotifyStrategy: ['WARN'], + restEndpoint: '', + username: '', + password: '' } as INodeData) return { @@ -60,6 +65,7 @@ export function useZeppelin({ ...Fields.useFailed(), Fields.useDelayTime(model), ...Fields.useTimeoutAlarm(model), + ...Fields.useDatasource(model), ...Fields.useZeppelin(model), Fields.usePreTasks() ] as IJsonItem[], From ab406486899ad4b2943c8cb334aab5335995bc42 Mon Sep 17 00:00:00 2001 From: xdu-chenrj Date: Mon, 3 Jul 2023 10:56:04 +0800 Subject: [PATCH 02/15] repaired the method for retrieving task types --- .../dolphinscheduler/plugin/task/zeppelin/ZeppelinTask.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-zeppelin/src/main/java/org/apache/dolphinscheduler/plugin/task/zeppelin/ZeppelinTask.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-zeppelin/src/main/java/org/apache/dolphinscheduler/plugin/task/zeppelin/ZeppelinTask.java index b409a0dd4261..584d58d7c736 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-zeppelin/src/main/java/org/apache/dolphinscheduler/plugin/task/zeppelin/ZeppelinTask.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-zeppelin/src/main/java/org/apache/dolphinscheduler/plugin/task/zeppelin/ZeppelinTask.java @@ -86,7 +86,7 @@ public void init() { zeppelinTaskExecutionContext = zeppelinParameters.generateExtendedContext(taskExecutionContext.getResourceParametersHelper()); zeppelinConnectionParam = (ZeppelinConnectionParam) DataSourceUtils - .buildConnectionParams(DbType.valueOf("ZEPPELIN"), zeppelinTaskExecutionContext.getConnectionParams()); + .buildConnectionParams(DbType.valueOf(zeppelinParameters.getType()), zeppelinTaskExecutionContext.getConnectionParams()); zeppelinParameters.setUsername(zeppelinConnectionParam.getUsername()); zeppelinParameters.setPassword(zeppelinConnectionParam.getPassword()); zeppelinParameters.setRestEndpoint(zeppelinConnectionParam.getRestEndpoint()); From c0bf0723d45c60ac2dc34fc54cf90eb5748c72df Mon Sep 17 00:00:00 2001 From: xdu-chenrj Date: Sat, 8 Jul 2023 11:40:55 +0800 Subject: [PATCH 03/15] recovered code with incorrect changes --- .asf.yaml | 2 +- .../dolphinscheduler/templates/_helpers.tpl | 103 ++++++++++++++++++ 2 files changed, 104 insertions(+), 1 deletion(-) diff --git a/.asf.yaml b/.asf.yaml index 9043e652a792..d403bc7d46fe 100644 --- a/.asf.yaml +++ b/.asf.yaml @@ -16,7 +16,7 @@ # github: - description: Apache DolphinScheduler is the modern data workflow orchestration platform with powerful user interface, dedicated to solving complex task dependencies in the data pipeline and providing various types of jobs available `out of the box` + description: Apache DolphinScheduler is the modern data orchestration platform. Agile to create high performance workflow with low-code homepage: https://dolphinscheduler.apache.org/ labels: - cloud-native diff --git a/deploy/kubernetes/dolphinscheduler/templates/_helpers.tpl b/deploy/kubernetes/dolphinscheduler/templates/_helpers.tpl index 3991a198d910..3e68b07c1645 100644 --- a/deploy/kubernetes/dolphinscheduler/templates/_helpers.tpl +++ b/deploy/kubernetes/dolphinscheduler/templates/_helpers.tpl @@ -177,6 +177,38 @@ Create a database environment variables. {{- end }} {{- end -}} +{{/* +Create a security environment variables. +*/}} +{{- define "dolphinscheduler.security.env_vars" -}} +- name: SECURITY_AUTHENTICATION_TYPE + value: {{ .Values.security.authentication.type | quote }} +{{- if eq .Values.security.authentication.type "LDAP" }} +- name: SECURITY_AUTHENTICATION_LDAP_URLS + value: {{ .Values.security.authentication.ldap.urls | quote }} +- name: SECURITY_AUTHENTICATION_LDAP_BASE_DN + value: {{ .Values.security.authentication.ldap.basedn | quote }} +- name: SECURITY_AUTHENTICATION_LDAP_USERNAME + value: {{ .Values.security.authentication.ldap.username | quote }} +- name: SECURITY_AUTHENTICATION_LDAP_PASSWORD + value: {{ .Values.security.authentication.ldap.password | quote }} +- name: SECURITY_AUTHENTICATION_LDAP_USER_ADMIN + value: {{ .Values.security.authentication.ldap.user.admin | quote }} +- name: SECURITY_AUTHENTICATION_LDAP_USER_IDENTITY_ATTRIBUTE + value: {{ .Values.security.authentication.ldap.user.identityattribute | quote }} +- name: SECURITY_AUTHENTICATION_LDAP_USER_EMAIL_ATTRIBUTE + value: {{ .Values.security.authentication.ldap.user.emailattribute | quote }} +- name: SECURITY_AUTHENTICATION_LDAP_USER_NOT_EXIST_ACTION + value: {{ .Values.security.authentication.ldap.user.notexistaction | quote }} +- name: SECURITY_AUTHENTICATION_LDAP_SSL_ENABLE + value: {{ .Values.security.authentication.ldap.ssl.enable | quote }} +- name: SECURITY_AUTHENTICATION_LDAP_SSL_TRUST_STORE + value: {{ .Values.security.authentication.ldap.ssl.truststore | quote }} +- name: SECURITY_AUTHENTICATION_LDAP_SSL_TRUST_STORE_PASSWORD + value: {{ .Values.security.authentication.ldap.ssl.truststorepassword | quote }} +{{- end }} +{{- end -}} + {{/* Wait for database to be ready. */}} @@ -212,15 +244,36 @@ Create a registry environment variables. - name: REGISTRY_TYPE {{- if .Values.zookeeper.enabled }} value: "zookeeper" + {{- else if .Values.etcd.enabled }} + value: "etcd" {{- else }} value: {{ .Values.externalRegistry.registryPluginName }} {{- end }} +{{- if .Values.etcd.enabled }} +- name: REGISTRY_ENDPOINTS + value: {{ .Values.etcd.endpoints }} +- name: REGISTRY_NAMESPACE + value: {{ .Values.etcd.namespace }} +- name: REGISTRY_USER + value: {{ .Values.etcd.user }} +- name: REGISTRY_PASSWORD + value: {{ .Values.etcd.passWord }} +- name: REGISTRY_AUTHORITY + value: {{ .Values.etcd.authority }} +- name: REGISTRY_CERT_FILE + value: {{ .Values.etcd.ssl.certFile }} +- name: REGISTRY_KEY_CERT_CHAIN_FILE + value: {{ .Values.etcd.ssl.keyCertChainFile }} +- name: REGISTRY_KEY_FILE + value: {{ .Values.etcd.ssl.keyFile }} +{{- else }} - name: REGISTRY_ZOOKEEPER_CONNECT_STRING {{- if .Values.zookeeper.enabled }} value: {{ template "dolphinscheduler.zookeeper.quorum" . }} {{- else }} value: {{ .Values.externalRegistry.registryServers }} {{- end }} +{{- end }} {{- end -}} {{/* @@ -264,3 +317,53 @@ Create a fsFileResourcePersistence volumeMount. name: {{ include "dolphinscheduler.fullname" . }}-fs-file {{- end -}} {{- end -}} + +{{/* +Create a etcd ssl volume. +*/}} +{{- define "dolphinscheduler.etcd.ssl.volume" -}} +{{- if .Values.etcd.ssl.enabled -}} +- name: etcd-ssl + secret: + secretName: {{ include "dolphinscheduler.fullname" . }}-etcd-ssl +{{- end -}} +{{- end -}} + +{{/* +Create a etcd ssl volumeMount. +*/}} +{{- define "dolphinscheduler.etcd.ssl.volumeMount" -}} +{{- if .Values.etcd.ssl.enabled -}} +- mountPath: /opt/dolphinscheduler/{{ .Values.etcd.ssl.certFile }} + name: etcd-ssl + subPath: cert-file +- mountPath: /opt/dolphinscheduler/{{ .Values.etcd.ssl.keyCertChainFile }} + name: etcd-ssl + subPath: key-cert-chain-file +- mountPath: /opt/dolphinscheduler/{{ .Values.etcd.ssl.keyFile }} + name: etcd-ssl + subPath: key-file +{{- end -}} +{{- end -}} + +{{/* +Create a ldap ssl volume. +*/}} +{{- define "dolphinscheduler.ldap.ssl.volume" -}} +{{- if .Values.security.authentication.ldap.ssl.enable -}} +- name: jks-file + secret: + secretName: {{ include "dolphinscheduler.fullname" . }}-ldap-ssl +{{- end -}} +{{- end -}} + +{{/* +Create a ldap ssl volumeMount. +*/}} +{{- define "dolphinscheduler.ldap.ssl.volumeMount" -}} +{{- if .Values.security.authentication.ldap.ssl.enable -}} +- mountPath: {{ .Values.security.authentication.ldap.ssl.truststore }} + name: jks-file + subPath: jks-file +{{- end -}} +{{- end -}} From 925ba1150dcf97e18f289314ad055fc24767856f Mon Sep 17 00:00:00 2001 From: xdu-chenrj Date: Sat, 8 Jul 2023 20:45:56 +0800 Subject: [PATCH 04/15] added zeppelin connection UT and formatted the code --- dolphinscheduler-bom/pom.xml | 7 ++ .../pom.xml | 1 - .../ZeppelinDataSourceProcessorTest.java | 106 ++++++++++++++++++ .../plugin/task/zeppelin/ZeppelinTask.java | 3 +- 4 files changed, 115 insertions(+), 2 deletions(-) create mode 100644 dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-zeppelin/src/test/java/org/apache/dolphinscheduler/plugin/datasource/zeppelin/ZeppelinDataSourceProcessorTest.java diff --git a/dolphinscheduler-bom/pom.xml b/dolphinscheduler-bom/pom.xml index af622b1a8687..acd9a6a082c8 100644 --- a/dolphinscheduler-bom/pom.xml +++ b/dolphinscheduler-bom/pom.xml @@ -114,6 +114,7 @@ 6.0.0 1.6.0 1.2.10 + 0.10.1 @@ -888,6 +889,12 @@ snowflake-jdbc ${snowflake-jdbc.version} + + + org.apache.zeppelin + zeppelin-client + ${zeppelin-client.version} + diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-zeppelin/pom.xml b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-zeppelin/pom.xml index 5e6d22ab96fd..90c7f1ece5aa 100644 --- a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-zeppelin/pom.xml +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-zeppelin/pom.xml @@ -44,7 +44,6 @@ org.apache.zeppelin zeppelin-client - 0.10.1 diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-zeppelin/src/test/java/org/apache/dolphinscheduler/plugin/datasource/zeppelin/ZeppelinDataSourceProcessorTest.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-zeppelin/src/test/java/org/apache/dolphinscheduler/plugin/datasource/zeppelin/ZeppelinDataSourceProcessorTest.java new file mode 100644 index 000000000000..4dc8f8ae8101 --- /dev/null +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-zeppelin/src/test/java/org/apache/dolphinscheduler/plugin/datasource/zeppelin/ZeppelinDataSourceProcessorTest.java @@ -0,0 +1,106 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.plugin.datasource.zeppelin; + +import org.apache.dolphinscheduler.plugin.datasource.zeppelin.param.ZeppelinConnectionParam; +import org.apache.dolphinscheduler.plugin.datasource.zeppelin.param.ZeppelinDataSourceParamDTO; +import org.apache.dolphinscheduler.plugin.datasource.zeppelin.param.ZeppelinDataSourceProcessor; +import org.apache.dolphinscheduler.spi.enums.DbType; + +import org.apache.zeppelin.client.ZeppelinClient; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; +import org.mockito.Mockito; +import org.mockito.junit.jupiter.MockitoExtension; + +@ExtendWith(MockitoExtension.class) +public class ZeppelinDataSourceProcessorTest { + + private ZeppelinDataSourceProcessor zeppelinDataSourceProcessor; + + @Mock + private ZeppelinClient zeppelinClient; + private String connectJson = + "{\"username\":\"lucky\",\"password\":\"123456\",\"restEndpoint\":\"https://dolphinscheduler.com:8080\"}"; + + @BeforeEach + public void init() { + zeppelinDataSourceProcessor = new ZeppelinDataSourceProcessor(); + } + + @Test + void testCheckDatasourceParam() { + ZeppelinDataSourceParamDTO zeppelinDataSourceParamDTO = new ZeppelinDataSourceParamDTO(); + Assertions.assertThrows(IllegalArgumentException.class, + () -> zeppelinDataSourceProcessor.checkDatasourceParam(zeppelinDataSourceParamDTO)); + zeppelinDataSourceParamDTO.setRestEndpoint("http://dolphinscheduler.com:8080"); + Assertions.assertThrows(IllegalArgumentException.class, + () -> zeppelinDataSourceProcessor.checkDatasourceParam(zeppelinDataSourceParamDTO)); + zeppelinDataSourceParamDTO.setUserName("root"); + Assertions + .assertDoesNotThrow(() -> zeppelinDataSourceProcessor.checkDatasourceParam(zeppelinDataSourceParamDTO)); + } + + @Test + void testGetDatasourceUniqueId() { + ZeppelinConnectionParam zeppelinConnectionParam = new ZeppelinConnectionParam(); + zeppelinConnectionParam.setRestEndpoint("https://dolphinscheduler.com:8080"); + zeppelinConnectionParam.setUsername("root"); + zeppelinConnectionParam.setPassword("123456"); + Assertions.assertEquals("zeppelin@https://dolphinscheduler.com:8080@root@123456", + zeppelinDataSourceProcessor.getDatasourceUniqueId(zeppelinConnectionParam, DbType.ZEPPELIN)); + + } + + @Test + void testCreateDatasourceParamDTO() { + ZeppelinDataSourceParamDTO zeppelinDataSourceParamDTO = + (ZeppelinDataSourceParamDTO) zeppelinDataSourceProcessor.createDatasourceParamDTO(connectJson); + Assertions.assertEquals("lucky", zeppelinDataSourceParamDTO.getUserName()); + Assertions.assertEquals("123456", zeppelinDataSourceParamDTO.getPassword()); + Assertions.assertEquals("https://dolphinscheduler.com:8080", zeppelinDataSourceParamDTO.getRestEndpoint()); + } + + @Test + void testCreateConnectionParams() { + ZeppelinDataSourceParamDTO zeppelinDataSourceParamDTO = + (ZeppelinDataSourceParamDTO) zeppelinDataSourceProcessor.createDatasourceParamDTO(connectJson); + ZeppelinConnectionParam zeppelinConnectionParam = + zeppelinDataSourceProcessor.createConnectionParams(zeppelinDataSourceParamDTO); + Assertions.assertEquals("lucky", zeppelinConnectionParam.getUsername()); + Assertions.assertEquals("123456", zeppelinConnectionParam.getPassword()); + Assertions.assertEquals("https://dolphinscheduler.com:8080", zeppelinConnectionParam.getRestEndpoint()); + } + + @Test + void testTestConnection() throws Exception { + ZeppelinDataSourceParamDTO zeppelinDataSourceParamDTO = + (ZeppelinDataSourceParamDTO) zeppelinDataSourceProcessor.createDatasourceParamDTO(connectJson); + ZeppelinConnectionParam zeppelinConnectionParam = + zeppelinDataSourceProcessor.createConnectionParams(zeppelinDataSourceParamDTO); + + Mockito.doNothing().when(zeppelinClient).login(zeppelinConnectionParam.getUsername(), + zeppelinConnectionParam.getPassword()); + boolean isConnected = zeppelinDataSourceProcessor.testConnection(zeppelinConnectionParam); + Assertions.assertTrue(isConnected); + } +} diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-zeppelin/src/main/java/org/apache/dolphinscheduler/plugin/task/zeppelin/ZeppelinTask.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-zeppelin/src/main/java/org/apache/dolphinscheduler/plugin/task/zeppelin/ZeppelinTask.java index 584d58d7c736..f6fa8e89ffcc 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-zeppelin/src/main/java/org/apache/dolphinscheduler/plugin/task/zeppelin/ZeppelinTask.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-zeppelin/src/main/java/org/apache/dolphinscheduler/plugin/task/zeppelin/ZeppelinTask.java @@ -86,7 +86,8 @@ public void init() { zeppelinTaskExecutionContext = zeppelinParameters.generateExtendedContext(taskExecutionContext.getResourceParametersHelper()); zeppelinConnectionParam = (ZeppelinConnectionParam) DataSourceUtils - .buildConnectionParams(DbType.valueOf(zeppelinParameters.getType()), zeppelinTaskExecutionContext.getConnectionParams()); + .buildConnectionParams(DbType.valueOf(zeppelinParameters.getType()), + zeppelinTaskExecutionContext.getConnectionParams()); zeppelinParameters.setUsername(zeppelinConnectionParam.getUsername()); zeppelinParameters.setPassword(zeppelinConnectionParam.getPassword()); zeppelinParameters.setRestEndpoint(zeppelinConnectionParam.getRestEndpoint()); From 9e4ca63c1c2aa581822bec625219bcb0f4eef3b7 Mon Sep 17 00:00:00 2001 From: xdu-chenrj Date: Wed, 12 Jul 2023 18:56:56 +0800 Subject: [PATCH 05/15] fixed zeppelin connection UT --- .../service/impl/DataSourceServiceImpl.java | 1 + .../datasource/zeppelin/ZeppelinUtils.java | 36 +++++++++++++++++++ .../param/ZeppelinDataSourceProcessor.java | 6 ++-- .../ZeppelinDataSourceProcessorTest.java | 24 ++++++++----- 4 files changed, 55 insertions(+), 12 deletions(-) create mode 100644 dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-zeppelin/src/main/java/org/apache/dolphinscheduler/plugin/datasource/zeppelin/ZeppelinUtils.java diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/DataSourceServiceImpl.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/DataSourceServiceImpl.java index 438ce26a45f5..10e6f032b272 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/DataSourceServiceImpl.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/DataSourceServiceImpl.java @@ -385,6 +385,7 @@ public Result checkConnection(DbType type, ConnectionParam connectionPar } return result; } + // TODO: When merging datasource plugin and task plugin, DbType will be changed to ConnectionType. if (type == DbType.ZEPPELIN) { DataSourceProcessor zeppelinDataSourceProcessor = DataSourceUtils.getDatasourceProcessor(type); if (zeppelinDataSourceProcessor.testConnection(connectionParam)) { diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-zeppelin/src/main/java/org/apache/dolphinscheduler/plugin/datasource/zeppelin/ZeppelinUtils.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-zeppelin/src/main/java/org/apache/dolphinscheduler/plugin/datasource/zeppelin/ZeppelinUtils.java new file mode 100644 index 000000000000..308af03d8f4c --- /dev/null +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-zeppelin/src/main/java/org/apache/dolphinscheduler/plugin/datasource/zeppelin/ZeppelinUtils.java @@ -0,0 +1,36 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.plugin.datasource.zeppelin; + +import org.apache.dolphinscheduler.plugin.datasource.zeppelin.param.ZeppelinConnectionParam; + +import org.apache.zeppelin.client.ClientConfig; +import org.apache.zeppelin.client.ZeppelinClient; + +public class ZeppelinUtils { + + private ZeppelinUtils() { + throw new IllegalStateException("Utility class"); + } + + public static ZeppelinClient getZeppelinClient(ZeppelinConnectionParam connectionParam) throws Exception { + ClientConfig clientConfig = new ClientConfig(connectionParam.getRestEndpoint()); + return new ZeppelinClient(clientConfig); + } + +} diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-zeppelin/src/main/java/org/apache/dolphinscheduler/plugin/datasource/zeppelin/param/ZeppelinDataSourceProcessor.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-zeppelin/src/main/java/org/apache/dolphinscheduler/plugin/datasource/zeppelin/param/ZeppelinDataSourceProcessor.java index bf59fc14ebdd..66039ae8c39b 100644 --- a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-zeppelin/src/main/java/org/apache/dolphinscheduler/plugin/datasource/zeppelin/param/ZeppelinDataSourceProcessor.java +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-zeppelin/src/main/java/org/apache/dolphinscheduler/plugin/datasource/zeppelin/param/ZeppelinDataSourceProcessor.java @@ -21,11 +21,11 @@ import org.apache.dolphinscheduler.plugin.datasource.api.datasource.BaseDataSourceParamDTO; import org.apache.dolphinscheduler.plugin.datasource.api.datasource.DataSourceProcessor; import org.apache.dolphinscheduler.plugin.datasource.api.utils.PasswordUtils; +import org.apache.dolphinscheduler.plugin.datasource.zeppelin.ZeppelinUtils; import org.apache.dolphinscheduler.spi.datasource.ConnectionParam; import org.apache.dolphinscheduler.spi.enums.DbType; import org.apache.commons.lang3.StringUtils; -import org.apache.zeppelin.client.ClientConfig; import org.apache.zeppelin.client.ZeppelinClient; import java.sql.Connection; @@ -110,11 +110,9 @@ public Connection getConnection(ConnectionParam connectionParam) { @Override public boolean testConnection(ConnectionParam connectionParam) { ZeppelinConnectionParam baseConnectionParam = (ZeppelinConnectionParam) connectionParam; - ClientConfig clientConfig = new ClientConfig(baseConnectionParam.getRestEndpoint()); - try { // If the login fails, an exception will be thrown directly - ZeppelinClient zeppelinClient = new ZeppelinClient(clientConfig); + ZeppelinClient zeppelinClient = ZeppelinUtils.getZeppelinClient(baseConnectionParam); zeppelinClient.login(baseConnectionParam.username, baseConnectionParam.password); String version = zeppelinClient.getVersion(); log.info("zeppelin client connects to server successfully, version is {}", version); diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-zeppelin/src/test/java/org/apache/dolphinscheduler/plugin/datasource/zeppelin/ZeppelinDataSourceProcessorTest.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-zeppelin/src/test/java/org/apache/dolphinscheduler/plugin/datasource/zeppelin/ZeppelinDataSourceProcessorTest.java index 4dc8f8ae8101..11b3d77e58ae 100644 --- a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-zeppelin/src/test/java/org/apache/dolphinscheduler/plugin/datasource/zeppelin/ZeppelinDataSourceProcessorTest.java +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-zeppelin/src/test/java/org/apache/dolphinscheduler/plugin/datasource/zeppelin/ZeppelinDataSourceProcessorTest.java @@ -17,6 +17,9 @@ package org.apache.dolphinscheduler.plugin.datasource.zeppelin; +import static org.mockito.Mockito.RETURNS_DEEP_STUBS; +import static org.mockito.Mockito.when; + import org.apache.dolphinscheduler.plugin.datasource.zeppelin.param.ZeppelinConnectionParam; import org.apache.dolphinscheduler.plugin.datasource.zeppelin.param.ZeppelinDataSourceParamDTO; import org.apache.dolphinscheduler.plugin.datasource.zeppelin.param.ZeppelinDataSourceProcessor; @@ -28,7 +31,7 @@ import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; -import org.mockito.Mock; +import org.mockito.MockedStatic; import org.mockito.Mockito; import org.mockito.junit.jupiter.MockitoExtension; @@ -37,8 +40,6 @@ public class ZeppelinDataSourceProcessorTest { private ZeppelinDataSourceProcessor zeppelinDataSourceProcessor; - @Mock - private ZeppelinClient zeppelinClient; private String connectJson = "{\"username\":\"lucky\",\"password\":\"123456\",\"restEndpoint\":\"https://dolphinscheduler.com:8080\"}"; @@ -95,12 +96,19 @@ void testCreateConnectionParams() { void testTestConnection() throws Exception { ZeppelinDataSourceParamDTO zeppelinDataSourceParamDTO = (ZeppelinDataSourceParamDTO) zeppelinDataSourceProcessor.createDatasourceParamDTO(connectJson); - ZeppelinConnectionParam zeppelinConnectionParam = + ZeppelinConnectionParam connectionParam = zeppelinDataSourceProcessor.createConnectionParams(zeppelinDataSourceParamDTO); - Mockito.doNothing().when(zeppelinClient).login(zeppelinConnectionParam.getUsername(), - zeppelinConnectionParam.getPassword()); - boolean isConnected = zeppelinDataSourceProcessor.testConnection(zeppelinConnectionParam); - Assertions.assertTrue(isConnected); + MockedStatic zeppelinConnectionUtilsMockedStatic = + Mockito.mockStatic(ZeppelinUtils.class); + zeppelinConnectionUtilsMockedStatic.when(() -> ZeppelinUtils.getZeppelinClient(Mockito.any())).thenReturn(null); + Assertions.assertFalse(zeppelinDataSourceProcessor.testConnection(connectionParam)); + + ZeppelinClient zeppelinClient = Mockito.mock(ZeppelinClient.class, RETURNS_DEEP_STUBS); + zeppelinConnectionUtilsMockedStatic.when(() -> ZeppelinUtils.getZeppelinClient(Mockito.any())) + .thenReturn(zeppelinClient); + Mockito.doNothing().when(zeppelinClient).login(Mockito.any(), Mockito.any()); + when(zeppelinClient.getVersion()).thenReturn("1.0"); + Assertions.assertTrue(zeppelinDataSourceProcessor.testConnection(connectionParam)); } } From 0b8078dbbde5c8098b9e2a83b65eabf514be5aa3 Mon Sep 17 00:00:00 2001 From: xdu-chenrj Date: Wed, 12 Jul 2023 19:29:32 +0800 Subject: [PATCH 06/15] recovered incorrect modifications --- .github/workflows/api-test.yml | 10 ++++++++-- .../service/impl/AlertPluginInstanceServiceImpl.java | 6 +++--- 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/.github/workflows/api-test.yml b/.github/workflows/api-test.yml index 14a5bcec1de8..7462619a5eb9 100644 --- a/.github/workflows/api-test.yml +++ b/.github/workflows/api-test.yml @@ -67,7 +67,7 @@ jobs: ./mvnw -B clean install \ -Dmaven.test.skip \ -Dmaven.javadoc.skip \ - -Dcheckstyle.skip=true \ + -Dspotless.skip=true \ -Pdocker,release -Ddocker.tag=ci \ -pl dolphinscheduler-standalone-server -am - name: Export Docker Images @@ -94,6 +94,12 @@ jobs: class: org.apache.dolphinscheduler.api.test.cases.WorkerGroupAPITest - name: Project class: org.apache.dolphinscheduler.api.test.cases.ProjectAPITest + - name: Workflow + class: org.apache.dolphinscheduler.api.test.cases.ProcessDefinitionAPITest + - name: Scheduler + class: org.apache.dolphinscheduler.api.test.cases.SchedulerAPITest + - name: Executor + class: org.apache.dolphinscheduler.api.test.cases.ExecutorAPITest env: RECORDING_PATH: /tmp/recording-${{ matrix.case.name }} steps: @@ -118,7 +124,7 @@ jobs: run: | ./mvnw -B -f dolphinscheduler-api-test/pom.xml -am \ -DfailIfNoTests=false \ - -Dcheckstyle.skip=false \ + -Dspotless.skip=false \ -Dtest=${{ matrix.case.class }} test - uses: actions/upload-artifact@v2 if: always() diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/AlertPluginInstanceServiceImpl.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/AlertPluginInstanceServiceImpl.java index d7c7581f1156..8a7782145363 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/AlertPluginInstanceServiceImpl.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/AlertPluginInstanceServiceImpl.java @@ -243,16 +243,16 @@ public Result listPaging(User loginUser, String searchVal, int pageNo, int pageS } private List buildPluginInstanceVOList(List alertPluginInstances) { + List alertPluginInstanceVOS = new ArrayList<>(); if (CollectionUtils.isEmpty(alertPluginInstances)) { - return null; + return alertPluginInstanceVOS; } List pluginDefineList = pluginDefineMapper.queryAllPluginDefineList(); if (CollectionUtils.isEmpty(pluginDefineList)) { - return null; + return alertPluginInstanceVOS; } Map pluginDefineMap = pluginDefineList.stream().collect(Collectors.toMap(PluginDefine::getId, Function.identity())); - List alertPluginInstanceVOS = new ArrayList<>(); alertPluginInstances.forEach(alertPluginInstance -> { AlertPluginInstanceVO alertPluginInstanceVO = new AlertPluginInstanceVO(); From 1a638bb6b0b83e904caba9e9b6487b66a10fd74b Mon Sep 17 00:00:00 2001 From: xdu-chenrj Date: Wed, 12 Jul 2023 19:29:32 +0800 Subject: [PATCH 07/15] recovered incorrect modifications --- .../plugin/task/api/AbstractCommandExecutor.java | 6 ------ 1 file changed, 6 deletions(-) diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/AbstractCommandExecutor.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/AbstractCommandExecutor.java index 5fcdab0f5d75..17c52084ad89 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/AbstractCommandExecutor.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/AbstractCommandExecutor.java @@ -93,11 +93,6 @@ public abstract class AbstractCommandExecutor { protected boolean podLogOutputIsFinished = false; - /* - * SHELL result string - */ - protected String taskResultString; - /** * taskRequest */ @@ -370,7 +365,6 @@ private void parseProcessOutput(Process process) { varPool.append("$VarPool$"); } else { logBuffer.add(line); - taskResultString = line; } } processLogOutputIsSuccess = true; From 1d06f31d5c3adc17e588cfb68f3c090beb595a1d Mon Sep 17 00:00:00 2001 From: xdu-chenrj Date: Thu, 13 Jul 2023 15:32:05 +0800 Subject: [PATCH 08/15] fixed zeppelin task UT --- .../task/zeppelin/ZeppelinTaskTest.java | 76 +++++++++++-------- 1 file changed, 43 insertions(+), 33 deletions(-) diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-zeppelin/src/test/java/org/apache/dolphinscheduler/plugin/task/zeppelin/ZeppelinTaskTest.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-zeppelin/src/test/java/org/apache/dolphinscheduler/plugin/task/zeppelin/ZeppelinTaskTest.java index 8c5cb11377f2..d65e79cc049c 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-zeppelin/src/test/java/org/apache/dolphinscheduler/plugin/task/zeppelin/ZeppelinTaskTest.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-zeppelin/src/test/java/org/apache/dolphinscheduler/plugin/task/zeppelin/ZeppelinTaskTest.java @@ -17,29 +17,21 @@ package org.apache.dolphinscheduler.plugin.task.zeppelin; -import static org.apache.dolphinscheduler.plugin.task.api.TaskConstants.EXIT_CODE_FAILURE; -import static org.apache.dolphinscheduler.plugin.task.api.TaskConstants.EXIT_CODE_KILL; -import static org.apache.dolphinscheduler.plugin.task.api.TaskConstants.EXIT_CODE_SUCCESS; -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.Mockito.doReturn; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.spy; -import static org.mockito.Mockito.when; - +import com.fasterxml.jackson.databind.ObjectMapper; import org.apache.dolphinscheduler.common.utils.DateUtils; import org.apache.dolphinscheduler.common.utils.JSONUtils; +import org.apache.dolphinscheduler.plugin.datasource.api.utils.DataSourceUtils; +import org.apache.dolphinscheduler.plugin.datasource.zeppelin.param.ZeppelinConnectionParam; import org.apache.dolphinscheduler.plugin.task.api.TaskCallBack; import org.apache.dolphinscheduler.plugin.task.api.TaskException; import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext; import org.apache.dolphinscheduler.plugin.task.api.model.ApplicationInfo; - +import org.apache.dolphinscheduler.plugin.task.api.parameters.resource.ResourceParametersHelper; import org.apache.zeppelin.client.NoteResult; import org.apache.zeppelin.client.ParagraphResult; import org.apache.zeppelin.client.Status; import org.apache.zeppelin.client.ZeppelinClient; - -import java.util.Map; - +import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -48,7 +40,11 @@ import org.mockito.Mockito; import org.mockito.junit.jupiter.MockitoExtension; -import com.fasterxml.jackson.databind.ObjectMapper; +import java.util.Map; + +import static org.apache.dolphinscheduler.plugin.task.api.TaskConstants.*; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.*; @ExtendWith(MockitoExtension.class) public class ZeppelinTaskTest { @@ -59,6 +55,8 @@ public class ZeppelinTaskTest { private static final String MOCK_REST_ENDPOINT = "localhost:8080"; private static final String MOCK_CLONE_NOTE_ID = "3GYJR92R8"; private static final String MOCK_PRODUCTION_DIRECTORY = "/prod/"; + private static final String MOCK_TYPE = "ZEPPELIN"; + private static MockedStatic dataSourceUtilsStaticMock = null; private final ObjectMapper mapper = new ObjectMapper(); private ZeppelinClient zClient; @@ -80,9 +78,15 @@ public void updateTaskInstanceInfo(int taskInstanceId) { @BeforeEach public void before() throws Exception { - String zeppelinParameters = buildZeppelinTaskParameters(); + String zeppelinTaskParameters = buildZeppelinTaskParameters(); TaskExecutionContext taskExecutionContext = mock(TaskExecutionContext.class); - when(taskExecutionContext.getTaskParams()).thenReturn(zeppelinParameters); + ResourceParametersHelper resourceParametersHelper = mock(ResourceParametersHelper.class); + ZeppelinConnectionParam zeppelinConnectionParam = mock(ZeppelinConnectionParam.class); + when(taskExecutionContext.getTaskParams()).thenReturn(zeppelinTaskParameters); + when(taskExecutionContext.getResourceParametersHelper()).thenReturn(resourceParametersHelper); + + dataSourceUtilsStaticMock = Mockito.mockStatic(DataSourceUtils.class); + dataSourceUtilsStaticMock.when(() -> DataSourceUtils.buildConnectionParams(Mockito.any(), Mockito.any())).thenReturn(zeppelinConnectionParam); this.zeppelinTask = spy(new ZeppelinTask(taskExecutionContext)); this.zClient = mock(ZeppelinClient.class); @@ -93,6 +97,11 @@ public void before() throws Exception { this.zeppelinTask.init(); } + @AfterEach + public void afterEach() { + dataSourceUtilsStaticMock.close(); + } + @Test public void testHandleWithParagraphExecutionSuccess() throws Exception { when(this.zClient.executeParagraph(any(), any(), any(Map.class))).thenReturn(this.paragraphResult); @@ -131,9 +140,7 @@ public void testHandleWithParagraphExecutionError() throws Exception { this.zeppelinTask.handle(taskCallBack); - Mockito.verify(this.zClient).executeParagraph(MOCK_NOTE_ID, - MOCK_PARAGRAPH_ID, - (Map) mapper.readValue(MOCK_PARAMETERS, Map.class)); + Mockito.verify(this.zClient).executeParagraph(MOCK_NOTE_ID, MOCK_PARAGRAPH_ID, (Map) mapper.readValue(MOCK_PARAMETERS, Map.class)); Mockito.verify(this.paragraphResult).getResultInText(); Mockito.verify(this.paragraphResult).getStatus(); Assertions.assertEquals(EXIT_CODE_FAILURE, this.zeppelinTask.getExitStatusCode()); @@ -141,16 +148,13 @@ public void testHandleWithParagraphExecutionError() throws Exception { @Test public void testHandleWithParagraphExecutionException() throws Exception { - when(this.zClient.executeParagraph(any(), any(), any(Map.class))) - .thenThrow(new TaskException("Something wrong happens from zeppelin side")); + when(this.zClient.executeParagraph(any(), any(), any(Map.class))).thenThrow(new TaskException("Something wrong happens from zeppelin side")); Assertions.assertThrows(TaskException.class, () -> { this.zeppelinTask.handle(taskCallBack); }); - Mockito.verify(this.zClient).executeParagraph(MOCK_NOTE_ID, - MOCK_PARAGRAPH_ID, - (Map) mapper.readValue(MOCK_PARAMETERS, Map.class)); + Mockito.verify(this.zClient).executeParagraph(MOCK_NOTE_ID, MOCK_PARAGRAPH_ID, (Map) mapper.readValue(MOCK_PARAMETERS, Map.class)); Mockito.verify(this.paragraphResult, Mockito.times(0)).getResultInText(); Mockito.verify(this.paragraphResult, Mockito.times(0)).getStatus(); Assertions.assertEquals(EXIT_CODE_FAILURE, this.zeppelinTask.getExitStatusCode()); @@ -158,20 +162,23 @@ public void testHandleWithParagraphExecutionException() throws Exception { @Test public void testHandleWithNoteExecutionSuccess() throws Exception { - String zeppelinParametersWithNoParagraphId = buildZeppelinTaskParametersWithNoParagraphId(); + String zeppelinTaskParametersWithNoParagraphId = buildZeppelinTaskParametersWithNoParagraphId(); TaskExecutionContext taskExecutionContext = mock(TaskExecutionContext.class); - when(taskExecutionContext.getTaskParams()).thenReturn(zeppelinParametersWithNoParagraphId); + ResourceParametersHelper resourceParametersHelper = mock(ResourceParametersHelper.class); + when(taskExecutionContext.getTaskParams()).thenReturn(zeppelinTaskParametersWithNoParagraphId); + when(taskExecutionContext.getResourceParametersHelper()).thenReturn(resourceParametersHelper); + this.zeppelinTask = spy(new ZeppelinTask(taskExecutionContext)); this.zClient = mock(ZeppelinClient.class); this.noteResult = mock(NoteResult.class); doReturn(this.zClient).when(this.zeppelinTask).getZeppelinClient(); when(this.zClient.executeNote(any(), any(Map.class))).thenReturn(this.noteResult); + this.zeppelinTask.init(); this.zeppelinTask.handle(taskCallBack); - Mockito.verify(this.zClient).executeNote(MOCK_NOTE_ID, - (Map) mapper.readValue(MOCK_PARAMETERS, Map.class)); + Mockito.verify(this.zClient).executeNote(MOCK_NOTE_ID, (Map) mapper.readValue(MOCK_PARAMETERS, Map.class)); Mockito.verify(this.noteResult).getParagraphResultList(); Assertions.assertEquals(EXIT_CODE_SUCCESS, this.zeppelinTask.getExitStatusCode()); } @@ -183,6 +190,9 @@ public void testHandleWithNoteExecutionSuccessWithProductionSetting() throws Exc try (MockedStatic mockedStaticDateUtils = Mockito.mockStatic(DateUtils.class)) { when(taskExecutionContext.getTaskParams()).thenReturn(zeppelinParametersWithNoParagraphId); + ResourceParametersHelper resourceParametersHelper = mock(ResourceParametersHelper.class); + when(taskExecutionContext.getResourceParametersHelper()).thenReturn(resourceParametersHelper); + this.zeppelinTask = spy(new ZeppelinTask(taskExecutionContext)); this.zClient = mock(ZeppelinClient.class); @@ -194,11 +204,8 @@ public void testHandleWithNoteExecutionSuccessWithProductionSetting() throws Exc this.zeppelinTask.init(); when(DateUtils.getTimestampString()).thenReturn("123456789"); this.zeppelinTask.handle(taskCallBack); - Mockito.verify(this.zClient).cloneNote( - MOCK_NOTE_ID, - String.format("%s%s_%s", MOCK_PRODUCTION_DIRECTORY, MOCK_NOTE_ID, "123456789")); - Mockito.verify(this.zClient).executeNote(MOCK_CLONE_NOTE_ID, - (Map) mapper.readValue(MOCK_PARAMETERS, Map.class)); + Mockito.verify(this.zClient).cloneNote(MOCK_NOTE_ID, String.format("%s%s_%s", MOCK_PRODUCTION_DIRECTORY, MOCK_NOTE_ID, "123456789")); + Mockito.verify(this.zClient).executeNote(MOCK_CLONE_NOTE_ID, (Map) mapper.readValue(MOCK_PARAMETERS, Map.class)); Mockito.verify(this.noteResult).getParagraphResultList(); Mockito.verify(this.zClient).deleteNote(MOCK_CLONE_NOTE_ID); Assertions.assertEquals(EXIT_CODE_SUCCESS, this.zeppelinTask.getExitStatusCode()); @@ -211,6 +218,7 @@ private String buildZeppelinTaskParameters() { zeppelinParameters.setParagraphId(MOCK_PARAGRAPH_ID); zeppelinParameters.setRestEndpoint(MOCK_REST_ENDPOINT); zeppelinParameters.setParameters(MOCK_PARAMETERS); + zeppelinParameters.setType(MOCK_TYPE); return JSONUtils.toJsonString(zeppelinParameters); } @@ -220,6 +228,7 @@ private String buildZeppelinTaskParametersWithNoParagraphId() { zeppelinParameters.setNoteId(MOCK_NOTE_ID); zeppelinParameters.setParameters(MOCK_PARAMETERS); zeppelinParameters.setRestEndpoint(MOCK_REST_ENDPOINT); + zeppelinParameters.setType(MOCK_TYPE); return JSONUtils.toJsonString(zeppelinParameters); } @@ -230,6 +239,7 @@ private String buildZeppelinTaskParametersWithProductionSetting() { zeppelinParameters.setParameters(MOCK_PARAMETERS); zeppelinParameters.setRestEndpoint(MOCK_REST_ENDPOINT); zeppelinParameters.setProductionNoteDirectory(MOCK_PRODUCTION_DIRECTORY); + zeppelinParameters.setType(MOCK_TYPE); return JSONUtils.toJsonString(zeppelinParameters); } From d2f9924df26a979b4b49673b669d9c446ce6e854 Mon Sep 17 00:00:00 2001 From: xdu-chenrj Date: Thu, 13 Jul 2023 16:14:58 +0800 Subject: [PATCH 09/15] fixed zeppelin task UT --- .../task/zeppelin/ZeppelinTaskTest.java | 23 ++++++++++++------- 1 file changed, 15 insertions(+), 8 deletions(-) diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-zeppelin/src/test/java/org/apache/dolphinscheduler/plugin/task/zeppelin/ZeppelinTaskTest.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-zeppelin/src/test/java/org/apache/dolphinscheduler/plugin/task/zeppelin/ZeppelinTaskTest.java index d65e79cc049c..8fa299ebca4e 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-zeppelin/src/test/java/org/apache/dolphinscheduler/plugin/task/zeppelin/ZeppelinTaskTest.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-zeppelin/src/test/java/org/apache/dolphinscheduler/plugin/task/zeppelin/ZeppelinTaskTest.java @@ -84,7 +84,6 @@ public void before() throws Exception { ZeppelinConnectionParam zeppelinConnectionParam = mock(ZeppelinConnectionParam.class); when(taskExecutionContext.getTaskParams()).thenReturn(zeppelinTaskParameters); when(taskExecutionContext.getResourceParametersHelper()).thenReturn(resourceParametersHelper); - dataSourceUtilsStaticMock = Mockito.mockStatic(DataSourceUtils.class); dataSourceUtilsStaticMock.when(() -> DataSourceUtils.buildConnectionParams(Mockito.any(), Mockito.any())).thenReturn(zeppelinConnectionParam); this.zeppelinTask = spy(new ZeppelinTask(taskExecutionContext)); @@ -140,7 +139,9 @@ public void testHandleWithParagraphExecutionError() throws Exception { this.zeppelinTask.handle(taskCallBack); - Mockito.verify(this.zClient).executeParagraph(MOCK_NOTE_ID, MOCK_PARAGRAPH_ID, (Map) mapper.readValue(MOCK_PARAMETERS, Map.class)); + Mockito.verify(this.zClient).executeParagraph(MOCK_NOTE_ID, + MOCK_PARAGRAPH_ID, + (Map) mapper.readValue(MOCK_PARAMETERS, Map.class)); Mockito.verify(this.paragraphResult).getResultInText(); Mockito.verify(this.paragraphResult).getStatus(); Assertions.assertEquals(EXIT_CODE_FAILURE, this.zeppelinTask.getExitStatusCode()); @@ -148,13 +149,16 @@ public void testHandleWithParagraphExecutionError() throws Exception { @Test public void testHandleWithParagraphExecutionException() throws Exception { - when(this.zClient.executeParagraph(any(), any(), any(Map.class))).thenThrow(new TaskException("Something wrong happens from zeppelin side")); + when(this.zClient.executeParagraph(any(), any(), any(Map.class))) + .thenThrow(new TaskException("Something wrong happens from zeppelin side")); Assertions.assertThrows(TaskException.class, () -> { this.zeppelinTask.handle(taskCallBack); }); - Mockito.verify(this.zClient).executeParagraph(MOCK_NOTE_ID, MOCK_PARAGRAPH_ID, (Map) mapper.readValue(MOCK_PARAMETERS, Map.class)); + Mockito.verify(this.zClient).executeParagraph(MOCK_NOTE_ID, + MOCK_PARAGRAPH_ID, + (Map) mapper.readValue(MOCK_PARAMETERS, Map.class)); Mockito.verify(this.paragraphResult, Mockito.times(0)).getResultInText(); Mockito.verify(this.paragraphResult, Mockito.times(0)).getStatus(); Assertions.assertEquals(EXIT_CODE_FAILURE, this.zeppelinTask.getExitStatusCode()); @@ -167,7 +171,6 @@ public void testHandleWithNoteExecutionSuccess() throws Exception { ResourceParametersHelper resourceParametersHelper = mock(ResourceParametersHelper.class); when(taskExecutionContext.getTaskParams()).thenReturn(zeppelinTaskParametersWithNoParagraphId); when(taskExecutionContext.getResourceParametersHelper()).thenReturn(resourceParametersHelper); - this.zeppelinTask = spy(new ZeppelinTask(taskExecutionContext)); this.zClient = mock(ZeppelinClient.class); this.noteResult = mock(NoteResult.class); @@ -178,7 +181,8 @@ public void testHandleWithNoteExecutionSuccess() throws Exception { this.zeppelinTask.init(); this.zeppelinTask.handle(taskCallBack); - Mockito.verify(this.zClient).executeNote(MOCK_NOTE_ID, (Map) mapper.readValue(MOCK_PARAMETERS, Map.class)); + Mockito.verify(this.zClient).executeNote(MOCK_NOTE_ID, + (Map) mapper.readValue(MOCK_PARAMETERS, Map.class)); Mockito.verify(this.noteResult).getParagraphResultList(); Assertions.assertEquals(EXIT_CODE_SUCCESS, this.zeppelinTask.getExitStatusCode()); } @@ -204,8 +208,11 @@ public void testHandleWithNoteExecutionSuccessWithProductionSetting() throws Exc this.zeppelinTask.init(); when(DateUtils.getTimestampString()).thenReturn("123456789"); this.zeppelinTask.handle(taskCallBack); - Mockito.verify(this.zClient).cloneNote(MOCK_NOTE_ID, String.format("%s%s_%s", MOCK_PRODUCTION_DIRECTORY, MOCK_NOTE_ID, "123456789")); - Mockito.verify(this.zClient).executeNote(MOCK_CLONE_NOTE_ID, (Map) mapper.readValue(MOCK_PARAMETERS, Map.class)); + Mockito.verify(this.zClient).cloneNote( + MOCK_NOTE_ID, + String.format("%s%s_%s", MOCK_PRODUCTION_DIRECTORY, MOCK_NOTE_ID, "123456789")); + Mockito.verify(this.zClient).executeNote(MOCK_CLONE_NOTE_ID, + (Map) mapper.readValue(MOCK_PARAMETERS, Map.class)); Mockito.verify(this.noteResult).getParagraphResultList(); Mockito.verify(this.zClient).deleteNote(MOCK_CLONE_NOTE_ID); Assertions.assertEquals(EXIT_CODE_SUCCESS, this.zeppelinTask.getExitStatusCode()); From 9f4a0f0274e3467aec154481e8a54196ce3688f6 Mon Sep 17 00:00:00 2001 From: xdu-chenrj Date: Thu, 20 Jul 2023 16:22:18 +0800 Subject: [PATCH 10/15] fixed zeppelin task UT --- .../plugin/task/zeppelin/ZeppelinTaskTest.java | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-zeppelin/src/test/java/org/apache/dolphinscheduler/plugin/task/zeppelin/ZeppelinTaskTest.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-zeppelin/src/test/java/org/apache/dolphinscheduler/plugin/task/zeppelin/ZeppelinTaskTest.java index 8fa299ebca4e..8daf06d83ea3 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-zeppelin/src/test/java/org/apache/dolphinscheduler/plugin/task/zeppelin/ZeppelinTaskTest.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-zeppelin/src/test/java/org/apache/dolphinscheduler/plugin/task/zeppelin/ZeppelinTaskTest.java @@ -44,7 +44,10 @@ import static org.apache.dolphinscheduler.plugin.task.api.TaskConstants.*; import static org.mockito.ArgumentMatchers.any; -import static org.mockito.Mockito.*; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; +import static org.mockito.Mockito.spy; +import static org.mockito.Mockito.doReturn; @ExtendWith(MockitoExtension.class) public class ZeppelinTaskTest { From 50fb4cb12f09647b8359e15b45f07beb4dcb0b04 Mon Sep 17 00:00:00 2001 From: xdu-chenrj Date: Thu, 20 Jul 2023 17:39:48 +0800 Subject: [PATCH 11/15] fixed zeppelin task UT --- .../task/zeppelin/ZeppelinTaskTest.java | 27 +++++++++++-------- 1 file changed, 16 insertions(+), 11 deletions(-) diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-zeppelin/src/test/java/org/apache/dolphinscheduler/plugin/task/zeppelin/ZeppelinTaskTest.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-zeppelin/src/test/java/org/apache/dolphinscheduler/plugin/task/zeppelin/ZeppelinTaskTest.java index 8daf06d83ea3..ffaf85b207b0 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-zeppelin/src/test/java/org/apache/dolphinscheduler/plugin/task/zeppelin/ZeppelinTaskTest.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-zeppelin/src/test/java/org/apache/dolphinscheduler/plugin/task/zeppelin/ZeppelinTaskTest.java @@ -17,7 +17,15 @@ package org.apache.dolphinscheduler.plugin.task.zeppelin; -import com.fasterxml.jackson.databind.ObjectMapper; +import static org.apache.dolphinscheduler.plugin.task.api.TaskConstants.EXIT_CODE_FAILURE; +import static org.apache.dolphinscheduler.plugin.task.api.TaskConstants.EXIT_CODE_KILL; +import static org.apache.dolphinscheduler.plugin.task.api.TaskConstants.EXIT_CODE_SUCCESS; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.doReturn; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.spy; +import static org.mockito.Mockito.when; + import org.apache.dolphinscheduler.common.utils.DateUtils; import org.apache.dolphinscheduler.common.utils.JSONUtils; import org.apache.dolphinscheduler.plugin.datasource.api.utils.DataSourceUtils; @@ -27,10 +35,14 @@ import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext; import org.apache.dolphinscheduler.plugin.task.api.model.ApplicationInfo; import org.apache.dolphinscheduler.plugin.task.api.parameters.resource.ResourceParametersHelper; + import org.apache.zeppelin.client.NoteResult; import org.apache.zeppelin.client.ParagraphResult; import org.apache.zeppelin.client.Status; import org.apache.zeppelin.client.ZeppelinClient; + +import java.util.Map; + import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.BeforeEach; @@ -40,14 +52,7 @@ import org.mockito.Mockito; import org.mockito.junit.jupiter.MockitoExtension; -import java.util.Map; - -import static org.apache.dolphinscheduler.plugin.task.api.TaskConstants.*; -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; -import static org.mockito.Mockito.spy; -import static org.mockito.Mockito.doReturn; +import com.fasterxml.jackson.databind.ObjectMapper; @ExtendWith(MockitoExtension.class) public class ZeppelinTaskTest { @@ -88,7 +93,8 @@ public void before() throws Exception { when(taskExecutionContext.getTaskParams()).thenReturn(zeppelinTaskParameters); when(taskExecutionContext.getResourceParametersHelper()).thenReturn(resourceParametersHelper); dataSourceUtilsStaticMock = Mockito.mockStatic(DataSourceUtils.class); - dataSourceUtilsStaticMock.when(() -> DataSourceUtils.buildConnectionParams(Mockito.any(), Mockito.any())).thenReturn(zeppelinConnectionParam); + dataSourceUtilsStaticMock.when(() -> DataSourceUtils.buildConnectionParams(Mockito.any(), Mockito.any())) + .thenReturn(zeppelinConnectionParam); this.zeppelinTask = spy(new ZeppelinTask(taskExecutionContext)); this.zClient = mock(ZeppelinClient.class); @@ -180,7 +186,6 @@ public void testHandleWithNoteExecutionSuccess() throws Exception { doReturn(this.zClient).when(this.zeppelinTask).getZeppelinClient(); when(this.zClient.executeNote(any(), any(Map.class))).thenReturn(this.noteResult); - this.zeppelinTask.init(); this.zeppelinTask.handle(taskCallBack); From a335054c44eb7260488ab344a4c5bc59aeed0f53 Mon Sep 17 00:00:00 2001 From: xdu-chenrj Date: Thu, 20 Jul 2023 19:30:45 +0800 Subject: [PATCH 12/15] modified zeppelin code --- .../java/org/apache/dolphinscheduler/spi/enums/DbType.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/enums/DbType.java b/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/enums/DbType.java index 5c4b65b97049..967eec3b863b 100644 --- a/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/enums/DbType.java +++ b/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/enums/DbType.java @@ -50,9 +50,9 @@ public enum DbType { DATABEND(19, "databend"), SNOWFLAKE(20, "snowflake"), VERTICA(21, "vertica"), - ZEPPELIN(23, "zeppelin"), HANA(22, "hana"), - DORIS(23, "doris"); + DORIS(23, "doris"), + ZEPPELIN(24, "zeppelin"); private static final Map DB_TYPE_MAP = Arrays.stream(DbType.values()).collect(toMap(DbType::getCode, Functions.identity())); From b8165a42c12d66dc6235899a43eb63f05ae96611 Mon Sep 17 00:00:00 2001 From: xdu-chenrj Date: Thu, 14 Sep 2023 17:31:27 +0800 Subject: [PATCH 13/15] supported zeppelin connection --- .../zeppelin/ZeppelinClientWrapper.java | 55 +++++++++++++++++++ .../zeppelin/ZeppelinDataSourceChannel.java | 12 +++- .../zeppelin/ZeppelinDataSourceClient.java | 36 ------------ .../param/ZeppelinDataSourceProcessor.java | 18 +++--- .../ZeppelinDataSourceProcessorTest.java | 31 ++++------- 5 files changed, 83 insertions(+), 69 deletions(-) create mode 100644 dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-zeppelin/src/main/java/org/apache/dolphinscheduler/plugin/datasource/zeppelin/ZeppelinClientWrapper.java delete mode 100644 dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-zeppelin/src/main/java/org/apache/dolphinscheduler/plugin/datasource/zeppelin/ZeppelinDataSourceClient.java diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-zeppelin/src/main/java/org/apache/dolphinscheduler/plugin/datasource/zeppelin/ZeppelinClientWrapper.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-zeppelin/src/main/java/org/apache/dolphinscheduler/plugin/datasource/zeppelin/ZeppelinClientWrapper.java new file mode 100644 index 000000000000..4a6c840e4055 --- /dev/null +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-zeppelin/src/main/java/org/apache/dolphinscheduler/plugin/datasource/zeppelin/ZeppelinClientWrapper.java @@ -0,0 +1,55 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.plugin.datasource.zeppelin; + +import static com.google.common.base.Preconditions.checkNotNull; + +import org.apache.zeppelin.client.ClientConfig; +import org.apache.zeppelin.client.ZeppelinClient; + +import lombok.extern.slf4j.Slf4j; +@Slf4j +public class ZeppelinClientWrapper implements AutoCloseable { + + private ZeppelinClient zeppelinClient; + + public ZeppelinClientWrapper(String restEndpoint) + throws Exception { + checkNotNull(restEndpoint); + ClientConfig clientConfig = new ClientConfig(restEndpoint); + zeppelinClient = new ZeppelinClient(clientConfig); + } + + public boolean checkConnect(String username, String password) { + try { + // If the login fails, an exception will be thrown directly + zeppelinClient.login(username, password); + String version = zeppelinClient.getVersion(); + log.info("zeppelin client connects to server successfully, version is {}", version); + return true; + } catch (Exception e) { + log.info("zeppelin client failed to connect to the server"); + return false; + } + } + + @Override + public void close() throws Exception { + + } +} diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-zeppelin/src/main/java/org/apache/dolphinscheduler/plugin/datasource/zeppelin/ZeppelinDataSourceChannel.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-zeppelin/src/main/java/org/apache/dolphinscheduler/plugin/datasource/zeppelin/ZeppelinDataSourceChannel.java index eca7d0490ae4..c8e33611e722 100644 --- a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-zeppelin/src/main/java/org/apache/dolphinscheduler/plugin/datasource/zeppelin/ZeppelinDataSourceChannel.java +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-zeppelin/src/main/java/org/apache/dolphinscheduler/plugin/datasource/zeppelin/ZeppelinDataSourceChannel.java @@ -17,15 +17,21 @@ package org.apache.dolphinscheduler.plugin.datasource.zeppelin; +import org.apache.dolphinscheduler.spi.datasource.AdHocDataSourceClient; import org.apache.dolphinscheduler.spi.datasource.BaseConnectionParam; import org.apache.dolphinscheduler.spi.datasource.DataSourceChannel; -import org.apache.dolphinscheduler.spi.datasource.DataSourceClient; +import org.apache.dolphinscheduler.spi.datasource.PooledDataSourceClient; import org.apache.dolphinscheduler.spi.enums.DbType; public class ZeppelinDataSourceChannel implements DataSourceChannel { @Override - public DataSourceClient createDataSourceClient(BaseConnectionParam baseConnectionParam, DbType dbType) { - return new ZeppelinDataSourceClient(baseConnectionParam, dbType); + public AdHocDataSourceClient createAdHocDataSourceClient(BaseConnectionParam baseConnectionParam, DbType dbType) { + throw new UnsupportedOperationException("Zeppelin AdHocDataSourceClient is not supported"); + } + + @Override + public PooledDataSourceClient createPooledDataSourceClient(BaseConnectionParam baseConnectionParam, DbType dbType) { + throw new UnsupportedOperationException("Zeppelin AdHocDataSourceClient is not supported"); } } diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-zeppelin/src/main/java/org/apache/dolphinscheduler/plugin/datasource/zeppelin/ZeppelinDataSourceClient.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-zeppelin/src/main/java/org/apache/dolphinscheduler/plugin/datasource/zeppelin/ZeppelinDataSourceClient.java deleted file mode 100644 index c20a314c4a62..000000000000 --- a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-zeppelin/src/main/java/org/apache/dolphinscheduler/plugin/datasource/zeppelin/ZeppelinDataSourceClient.java +++ /dev/null @@ -1,36 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.dolphinscheduler.plugin.datasource.zeppelin; - -import org.apache.dolphinscheduler.plugin.datasource.api.client.CommonDataSourceClient; -import org.apache.dolphinscheduler.spi.datasource.BaseConnectionParam; -import org.apache.dolphinscheduler.spi.enums.DbType; - -import java.sql.Connection; - -public class ZeppelinDataSourceClient extends CommonDataSourceClient { - - public ZeppelinDataSourceClient(BaseConnectionParam baseConnectionParam, DbType dbType) { - super(baseConnectionParam, dbType); - } - - @Override - public Connection getConnection() { - return (Connection) this.baseConnectionParam; - } -} diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-zeppelin/src/main/java/org/apache/dolphinscheduler/plugin/datasource/zeppelin/param/ZeppelinDataSourceProcessor.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-zeppelin/src/main/java/org/apache/dolphinscheduler/plugin/datasource/zeppelin/param/ZeppelinDataSourceProcessor.java index 66039ae8c39b..bf2795959e67 100644 --- a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-zeppelin/src/main/java/org/apache/dolphinscheduler/plugin/datasource/zeppelin/param/ZeppelinDataSourceProcessor.java +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-zeppelin/src/main/java/org/apache/dolphinscheduler/plugin/datasource/zeppelin/param/ZeppelinDataSourceProcessor.java @@ -21,12 +21,11 @@ import org.apache.dolphinscheduler.plugin.datasource.api.datasource.BaseDataSourceParamDTO; import org.apache.dolphinscheduler.plugin.datasource.api.datasource.DataSourceProcessor; import org.apache.dolphinscheduler.plugin.datasource.api.utils.PasswordUtils; -import org.apache.dolphinscheduler.plugin.datasource.zeppelin.ZeppelinUtils; +import org.apache.dolphinscheduler.plugin.datasource.zeppelin.ZeppelinClientWrapper; import org.apache.dolphinscheduler.spi.datasource.ConnectionParam; import org.apache.dolphinscheduler.spi.enums.DbType; import org.apache.commons.lang3.StringUtils; -import org.apache.zeppelin.client.ZeppelinClient; import java.sql.Connection; import java.text.MessageFormat; @@ -108,17 +107,14 @@ public Connection getConnection(ConnectionParam connectionParam) { } @Override - public boolean testConnection(ConnectionParam connectionParam) { + public boolean checkDataSourceConnectivity(ConnectionParam connectionParam) { ZeppelinConnectionParam baseConnectionParam = (ZeppelinConnectionParam) connectionParam; - try { - // If the login fails, an exception will be thrown directly - ZeppelinClient zeppelinClient = ZeppelinUtils.getZeppelinClient(baseConnectionParam); - zeppelinClient.login(baseConnectionParam.username, baseConnectionParam.password); - String version = zeppelinClient.getVersion(); - log.info("zeppelin client connects to server successfully, version is {}", version); - return true; + try ( + ZeppelinClientWrapper zeppelinClientWrapper = + new ZeppelinClientWrapper(baseConnectionParam.getRestEndpoint())) { + return zeppelinClientWrapper.checkConnect(baseConnectionParam.username, baseConnectionParam.password); } catch (Exception e) { - log.info("zeppelin client failed to connect to the server"); + log.error("zeppelin client failed to connect to the server", e); return false; } } diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-zeppelin/src/test/java/org/apache/dolphinscheduler/plugin/datasource/zeppelin/ZeppelinDataSourceProcessorTest.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-zeppelin/src/test/java/org/apache/dolphinscheduler/plugin/datasource/zeppelin/ZeppelinDataSourceProcessorTest.java index 11b3d77e58ae..05be02e72231 100644 --- a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-zeppelin/src/test/java/org/apache/dolphinscheduler/plugin/datasource/zeppelin/ZeppelinDataSourceProcessorTest.java +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-zeppelin/src/test/java/org/apache/dolphinscheduler/plugin/datasource/zeppelin/ZeppelinDataSourceProcessorTest.java @@ -17,21 +17,16 @@ package org.apache.dolphinscheduler.plugin.datasource.zeppelin; -import static org.mockito.Mockito.RETURNS_DEEP_STUBS; -import static org.mockito.Mockito.when; - import org.apache.dolphinscheduler.plugin.datasource.zeppelin.param.ZeppelinConnectionParam; import org.apache.dolphinscheduler.plugin.datasource.zeppelin.param.ZeppelinDataSourceParamDTO; import org.apache.dolphinscheduler.plugin.datasource.zeppelin.param.ZeppelinDataSourceProcessor; import org.apache.dolphinscheduler.spi.enums.DbType; -import org.apache.zeppelin.client.ZeppelinClient; - import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; -import org.mockito.MockedStatic; +import org.mockito.MockedConstruction; import org.mockito.Mockito; import org.mockito.junit.jupiter.MockitoExtension; @@ -93,22 +88,20 @@ void testCreateConnectionParams() { } @Test - void testTestConnection() throws Exception { + void testTestConnection() { ZeppelinDataSourceParamDTO zeppelinDataSourceParamDTO = (ZeppelinDataSourceParamDTO) zeppelinDataSourceProcessor.createDatasourceParamDTO(connectJson); ZeppelinConnectionParam connectionParam = zeppelinDataSourceProcessor.createConnectionParams(zeppelinDataSourceParamDTO); - - MockedStatic zeppelinConnectionUtilsMockedStatic = - Mockito.mockStatic(ZeppelinUtils.class); - zeppelinConnectionUtilsMockedStatic.when(() -> ZeppelinUtils.getZeppelinClient(Mockito.any())).thenReturn(null); - Assertions.assertFalse(zeppelinDataSourceProcessor.testConnection(connectionParam)); - - ZeppelinClient zeppelinClient = Mockito.mock(ZeppelinClient.class, RETURNS_DEEP_STUBS); - zeppelinConnectionUtilsMockedStatic.when(() -> ZeppelinUtils.getZeppelinClient(Mockito.any())) - .thenReturn(zeppelinClient); - Mockito.doNothing().when(zeppelinClient).login(Mockito.any(), Mockito.any()); - when(zeppelinClient.getVersion()).thenReturn("1.0"); - Assertions.assertTrue(zeppelinDataSourceProcessor.testConnection(connectionParam)); + Assertions.assertFalse(zeppelinDataSourceProcessor.checkDataSourceConnectivity(connectionParam)); + try ( + MockedConstruction sshClientWrapperMockedConstruction = + Mockito.mockConstruction(ZeppelinClientWrapper.class, (mock, context) -> { + Mockito.when( + mock.checkConnect(connectionParam.getUsername(), connectionParam.getPassword())) + .thenReturn(true); + })) { + Assertions.assertTrue(zeppelinDataSourceProcessor.checkDataSourceConnectivity(connectionParam)); + } } } From 47bdc47ed288117b95f36975c757921bf8217dfc Mon Sep 17 00:00:00 2001 From: xdu-chenrj Date: Thu, 28 Sep 2023 20:02:53 +0800 Subject: [PATCH 14/15] partial sagemaker --- .../pom.xml | 39 +++++ .../sagemaker/SagemakerClientWrapper.java | 65 +++++++++ .../sagemaker/SagemakerDataSourceChannel.java | 37 +++++ .../SagemakerDataSourceChannelFactory.java | 38 +++++ .../param/SagemakerConnectionParam.java | 35 +++++ .../param/SagemakerDataSourceParamDTO.java | 38 +++++ .../param/SagemakerDataSourceProcessor.java | 136 ++++++++++++++++++ 7 files changed, 388 insertions(+) create mode 100644 dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-sagemaker/pom.xml create mode 100644 dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-sagemaker/src/main/java/org/apache/dolphinscheduler/plugin/datasource/sagemaker/SagemakerClientWrapper.java create mode 100644 dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-sagemaker/src/main/java/org/apache/dolphinscheduler/plugin/datasource/sagemaker/SagemakerDataSourceChannel.java create mode 100644 dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-sagemaker/src/main/java/org/apache/dolphinscheduler/plugin/datasource/sagemaker/SagemakerDataSourceChannelFactory.java create mode 100644 dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-sagemaker/src/main/java/org/apache/dolphinscheduler/plugin/datasource/sagemaker/param/SagemakerConnectionParam.java create mode 100644 dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-sagemaker/src/main/java/org/apache/dolphinscheduler/plugin/datasource/sagemaker/param/SagemakerDataSourceParamDTO.java create mode 100644 dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-sagemaker/src/main/java/org/apache/dolphinscheduler/plugin/datasource/sagemaker/param/SagemakerDataSourceProcessor.java diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-sagemaker/pom.xml b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-sagemaker/pom.xml new file mode 100644 index 000000000000..49abd6a3970a --- /dev/null +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-sagemaker/pom.xml @@ -0,0 +1,39 @@ + + + 4.0.0 + + org.apache.dolphinscheduler + dolphinscheduler-datasource-plugin + dev-SNAPSHOT + + + org.apache.dolphinscheduler-plugin + dolphinscheduler-datasource-sagemaker + + + 8 + 8 + UTF-8 + + + + + org.apache.dolphinscheduler + dolphinscheduler-spi + provided + + + + org.apache.dolphinscheduler + dolphinscheduler-datasource-api + ${project.version} + + + + com.amazonaws + aws-java-sdk-sagemaker + + + + diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-sagemaker/src/main/java/org/apache/dolphinscheduler/plugin/datasource/sagemaker/SagemakerClientWrapper.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-sagemaker/src/main/java/org/apache/dolphinscheduler/plugin/datasource/sagemaker/SagemakerClientWrapper.java new file mode 100644 index 000000000000..2996789221bd --- /dev/null +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-sagemaker/src/main/java/org/apache/dolphinscheduler/plugin/datasource/sagemaker/SagemakerClientWrapper.java @@ -0,0 +1,65 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.plugin.datasource.sagemaker; + +import static com.google.common.base.Preconditions.checkNotNull; + +import lombok.extern.slf4j.Slf4j; + +import com.amazonaws.auth.AWSCredentialsProvider; +import com.amazonaws.auth.AWSStaticCredentialsProvider; +import com.amazonaws.auth.BasicAWSCredentials; +import com.amazonaws.services.sagemaker.AmazonSageMaker; +import com.amazonaws.services.sagemaker.AmazonSageMakerClientBuilder; +import com.amazonaws.services.sagemaker.model.ListNotebookInstancesRequest; + +@Slf4j +public class SagemakerClientWrapper implements AutoCloseable { + + private AmazonSageMaker amazonSageMaker; + + public SagemakerClientWrapper(String accessKey, String secretAccessKey, String region) { + checkNotNull(accessKey, "sagemaker accessKey cannot be null"); + checkNotNull(secretAccessKey, "sagemaker secretAccessKey cannot be null"); + checkNotNull(region, "sagemaker region cannot be null"); + + final BasicAWSCredentials basicAWSCredentials = new BasicAWSCredentials(accessKey, secretAccessKey); + final AWSCredentialsProvider awsCredentialsProvider = new AWSStaticCredentialsProvider(basicAWSCredentials); + // create a SageMaker client + amazonSageMaker = AmazonSageMakerClientBuilder.standard().withCredentials(awsCredentialsProvider) + .withRegion(region).build(); + } + + public boolean checkConnect() { + try { + // If listing notebook instances fails, an exception will be thrown directly + ListNotebookInstancesRequest request = new ListNotebookInstancesRequest(); + amazonSageMaker.listNotebookInstances(request); + log.info("sagemaker client connects to server successfully"); + return true; + } catch (Exception e) { + log.info("sagemaker client failed to connect to the server"); + } + return false; + } + + @Override + public void close() { + + } +} diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-sagemaker/src/main/java/org/apache/dolphinscheduler/plugin/datasource/sagemaker/SagemakerDataSourceChannel.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-sagemaker/src/main/java/org/apache/dolphinscheduler/plugin/datasource/sagemaker/SagemakerDataSourceChannel.java new file mode 100644 index 000000000000..03b96cf8a303 --- /dev/null +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-sagemaker/src/main/java/org/apache/dolphinscheduler/plugin/datasource/sagemaker/SagemakerDataSourceChannel.java @@ -0,0 +1,37 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.plugin.datasource.sagemaker; + +import org.apache.dolphinscheduler.spi.datasource.AdHocDataSourceClient; +import org.apache.dolphinscheduler.spi.datasource.BaseConnectionParam; +import org.apache.dolphinscheduler.spi.datasource.DataSourceChannel; +import org.apache.dolphinscheduler.spi.datasource.PooledDataSourceClient; +import org.apache.dolphinscheduler.spi.enums.DbType; + +public class SagemakerDataSourceChannel implements DataSourceChannel { + + @Override + public AdHocDataSourceClient createAdHocDataSourceClient(BaseConnectionParam baseConnectionParam, DbType dbType) { + throw new UnsupportedOperationException("Sagemaker AdHocDataSourceClient is not supported"); + } + + @Override + public PooledDataSourceClient createPooledDataSourceClient(BaseConnectionParam baseConnectionParam, DbType dbType) { + throw new UnsupportedOperationException("Sagemaker AdHocDataSourceClient is not supported"); + } +} diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-sagemaker/src/main/java/org/apache/dolphinscheduler/plugin/datasource/sagemaker/SagemakerDataSourceChannelFactory.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-sagemaker/src/main/java/org/apache/dolphinscheduler/plugin/datasource/sagemaker/SagemakerDataSourceChannelFactory.java new file mode 100644 index 000000000000..04ab93f36f9d --- /dev/null +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-sagemaker/src/main/java/org/apache/dolphinscheduler/plugin/datasource/sagemaker/SagemakerDataSourceChannelFactory.java @@ -0,0 +1,38 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.plugin.datasource.sagemaker; + +import org.apache.dolphinscheduler.spi.datasource.DataSourceChannel; +import org.apache.dolphinscheduler.spi.datasource.DataSourceChannelFactory; + +import com.google.auto.service.AutoService; + +@AutoService(DataSourceChannelFactory.class) +public class SagemakerDataSourceChannelFactory implements DataSourceChannelFactory { + + @Override + public DataSourceChannel create() { + return new SagemakerDataSourceChannel(); + } + + @Override + public String getName() { + return "sagemaker"; + } + +} diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-sagemaker/src/main/java/org/apache/dolphinscheduler/plugin/datasource/sagemaker/param/SagemakerConnectionParam.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-sagemaker/src/main/java/org/apache/dolphinscheduler/plugin/datasource/sagemaker/param/SagemakerConnectionParam.java new file mode 100644 index 000000000000..d0995d634832 --- /dev/null +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-sagemaker/src/main/java/org/apache/dolphinscheduler/plugin/datasource/sagemaker/param/SagemakerConnectionParam.java @@ -0,0 +1,35 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.plugin.datasource.sagemaker.param; + +import org.apache.dolphinscheduler.spi.datasource.ConnectionParam; + +import lombok.Data; + +import com.fasterxml.jackson.annotation.JsonInclude; + +@Data +@JsonInclude(JsonInclude.Include.NON_NULL) +public class SagemakerConnectionParam implements ConnectionParam { + + protected String accessKey; + + protected String secretAccessKey; + + protected String region; +} diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-sagemaker/src/main/java/org/apache/dolphinscheduler/plugin/datasource/sagemaker/param/SagemakerDataSourceParamDTO.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-sagemaker/src/main/java/org/apache/dolphinscheduler/plugin/datasource/sagemaker/param/SagemakerDataSourceParamDTO.java new file mode 100644 index 000000000000..406cfca5b58e --- /dev/null +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-sagemaker/src/main/java/org/apache/dolphinscheduler/plugin/datasource/sagemaker/param/SagemakerDataSourceParamDTO.java @@ -0,0 +1,38 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.plugin.datasource.sagemaker.param; + +import org.apache.dolphinscheduler.plugin.datasource.api.datasource.BaseDataSourceParamDTO; +import org.apache.dolphinscheduler.spi.enums.DbType; + +import lombok.Data; + +@Data +public class SagemakerDataSourceParamDTO extends BaseDataSourceParamDTO { + + protected String accessKey; + + protected String secretAccessKey; + + protected String region; + + @Override + public DbType getType() { + return DbType.SAGEMAKER; + } +} diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-sagemaker/src/main/java/org/apache/dolphinscheduler/plugin/datasource/sagemaker/param/SagemakerDataSourceProcessor.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-sagemaker/src/main/java/org/apache/dolphinscheduler/plugin/datasource/sagemaker/param/SagemakerDataSourceProcessor.java new file mode 100644 index 000000000000..d918e10e3931 --- /dev/null +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-sagemaker/src/main/java/org/apache/dolphinscheduler/plugin/datasource/sagemaker/param/SagemakerDataSourceProcessor.java @@ -0,0 +1,136 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.plugin.datasource.sagemaker.param; + +import org.apache.dolphinscheduler.common.utils.JSONUtils; +import org.apache.dolphinscheduler.plugin.datasource.api.datasource.BaseDataSourceParamDTO; +import org.apache.dolphinscheduler.plugin.datasource.api.datasource.DataSourceProcessor; +import org.apache.dolphinscheduler.plugin.datasource.api.utils.PasswordUtils; +import org.apache.dolphinscheduler.plugin.datasource.sagemaker.SagemakerClientWrapper; +import org.apache.dolphinscheduler.spi.datasource.ConnectionParam; +import org.apache.dolphinscheduler.spi.enums.DbType; + +import org.apache.commons.lang3.StringUtils; + +import java.sql.Connection; +import java.text.MessageFormat; + +import lombok.extern.slf4j.Slf4j; + +import com.google.auto.service.AutoService; + +@AutoService(DataSourceProcessor.class) +@Slf4j +public class SagemakerDataSourceProcessor implements DataSourceProcessor { + + @Override + public BaseDataSourceParamDTO castDatasourceParamDTO(String paramJson) { + return JSONUtils.parseObject(paramJson, SagemakerDataSourceParamDTO.class); + } + + @Override + public void checkDatasourceParam(BaseDataSourceParamDTO datasourceParamDTO) { + SagemakerDataSourceParamDTO sageMakerDataSourceParamDTO = (SagemakerDataSourceParamDTO) datasourceParamDTO; + if (StringUtils.isEmpty(sageMakerDataSourceParamDTO.getAccessKey()) + || StringUtils.isEmpty(sageMakerDataSourceParamDTO.getSecretAccessKey()) + || StringUtils.isEmpty(sageMakerDataSourceParamDTO.getRegion())) { + throw new IllegalArgumentException("sagemaker datasource param is not valid"); + } + } + + @Override + public String getDatasourceUniqueId(ConnectionParam connectionParam, DbType dbType) { + SagemakerConnectionParam baseConnectionParam = (SagemakerConnectionParam) connectionParam; + return MessageFormat.format("{0}@{1}@{2}@{3}", dbType.getDescp(), + PasswordUtils.encodePassword(baseConnectionParam.getAccessKey()), + PasswordUtils.encodePassword(baseConnectionParam.getSecretAccessKey()), + PasswordUtils.encodePassword(baseConnectionParam.getRegion())); + } + + // SageMaker + @Override + public BaseDataSourceParamDTO createDatasourceParamDTO(String connectionJson) { + SagemakerConnectionParam connectionParams = (SagemakerConnectionParam) createConnectionParams(connectionJson); + SagemakerDataSourceParamDTO SageMakerDataSourceParamDTO = new SagemakerDataSourceParamDTO(); + + SageMakerDataSourceParamDTO.setUserName(connectionParams.getAccessKey()); + SageMakerDataSourceParamDTO.setPassword(connectionParams.getSecretAccessKey()); + SageMakerDataSourceParamDTO.setRegion(connectionParams.getRegion()); + return SageMakerDataSourceParamDTO; + } + + @Override + public SagemakerConnectionParam createConnectionParams(BaseDataSourceParamDTO datasourceParam) { + SagemakerDataSourceParamDTO sageMakerDataSourceParam = (SagemakerDataSourceParamDTO) datasourceParam; + SagemakerConnectionParam sageMakerConnectionParam = new SagemakerConnectionParam(); + sageMakerConnectionParam.setAccessKey(sageMakerDataSourceParam.getAccessKey()); + sageMakerConnectionParam.setSecretAccessKey(sageMakerDataSourceParam.getSecretAccessKey()); + sageMakerConnectionParam.setRegion(sageMakerConnectionParam.getRegion()); + + return sageMakerConnectionParam; + } + + @Override + public ConnectionParam createConnectionParams(String connectionJson) { + return JSONUtils.parseObject(connectionJson, SagemakerConnectionParam.class); + } + + @Override + public String getDatasourceDriver() { + return ""; + } + + @Override + public String getValidationQuery() { + return ""; + } + + @Override + public String getJdbcUrl(ConnectionParam connectionParam) { + return ""; + } + + @Override + public Connection getConnection(ConnectionParam connectionParam) { + return null; + } + + @Override + public boolean checkDataSourceConnectivity(ConnectionParam connectionParam) { + SagemakerConnectionParam baseConnectionParam = (SagemakerConnectionParam) connectionParam; + try ( + SagemakerClientWrapper sagemakerClientWrapper = + new SagemakerClientWrapper(baseConnectionParam.getAccessKey(), + baseConnectionParam.getSecretAccessKey(), baseConnectionParam.getRegion())) { + return sagemakerClientWrapper.checkConnect(); + } catch (Exception e) { + log.error("sagemaker client failed to connect to the server", e); + return false; + } + } + + @Override + public DbType getDbType() { + return DbType.SAGEMAKER; + } + + @Override + public DataSourceProcessor create() { + return new SagemakerDataSourceProcessor(); + } +} From d4dc63e71a7a1f43e8ef25e04a5b730c586d2785 Mon Sep 17 00:00:00 2001 From: xdu-chenrj Date: Thu, 28 Sep 2023 20:04:56 +0800 Subject: [PATCH 15/15] partial sagemaker --- dolphinscheduler-datasource-plugin/pom.xml | 1 + .../apache/dolphinscheduler/spi/enums/DbType.java | 3 ++- dolphinscheduler-ui/src/locales/en_US/datasource.ts | 4 ++++ dolphinscheduler-ui/src/locales/zh_CN/datasource.ts | 4 ++++ .../src/service/modules/data-source/types.ts | 2 ++ .../src/views/datasource/list/use-form.ts | 13 +++++++++++-- .../task/components/node/fields/use-datasource.ts | 5 +++++ .../task/components/node/tasks/use-sagemaker.ts | 8 +++++++- 8 files changed, 36 insertions(+), 4 deletions(-) diff --git a/dolphinscheduler-datasource-plugin/pom.xml b/dolphinscheduler-datasource-plugin/pom.xml index 91882bb0b676..fb08aa7f6dd4 100644 --- a/dolphinscheduler-datasource-plugin/pom.xml +++ b/dolphinscheduler-datasource-plugin/pom.xml @@ -53,6 +53,7 @@ dolphinscheduler-datasource-snowflake dolphinscheduler-datasource-vertica dolphinscheduler-datasource-doris + dolphinscheduler-datasource-sagemaker diff --git a/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/enums/DbType.java b/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/enums/DbType.java index 967eec3b863b..01ac12fab6cb 100644 --- a/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/enums/DbType.java +++ b/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/enums/DbType.java @@ -52,7 +52,8 @@ public enum DbType { VERTICA(21, "vertica"), HANA(22, "hana"), DORIS(23, "doris"), - ZEPPELIN(24, "zeppelin"); + ZEPPELIN(24, "zeppelin"), + SAGEMAKER(25, "sagemaker"); private static final Map DB_TYPE_MAP = Arrays.stream(DbType.values()).collect(toMap(DbType::getCode, Functions.identity())); diff --git a/dolphinscheduler-ui/src/locales/en_US/datasource.ts b/dolphinscheduler-ui/src/locales/en_US/datasource.ts index 134bf015b172..bb280a69172b 100644 --- a/dolphinscheduler-ui/src/locales/en_US/datasource.ts +++ b/dolphinscheduler-ui/src/locales/en_US/datasource.ts @@ -70,6 +70,10 @@ export default { user_password_tips: 'Please enter your password', aws_region: 'Aws Region', aws_region_tips: 'Please enter AwsRegion', + aws_access_key: 'AwsAccessKey', + aws_access_key_tips: 'Please enter AwsAccessKey', + aws_secret_access_key: 'AwsSecretAccessKey', + aws_secret_access_key_tips: 'Please enter AwsSecretAccessKey', validation: 'Validation', mode_tips: 'Please select a mode', jdbc_format_tips: 'jdbc connection parameters is not a correct JSON format', diff --git a/dolphinscheduler-ui/src/locales/zh_CN/datasource.ts b/dolphinscheduler-ui/src/locales/zh_CN/datasource.ts index 434f339dceae..3e175bcf0c77 100644 --- a/dolphinscheduler-ui/src/locales/zh_CN/datasource.ts +++ b/dolphinscheduler-ui/src/locales/zh_CN/datasource.ts @@ -67,6 +67,10 @@ export default { user_password_tips: '请输入密码', aws_region: 'AwsRegion', aws_region_tips: '请输入AwsRegion', + aws_access_key: 'AwsAccessKey', + aws_access_key_tips: '请输入AwsAccessKey', + aws_secret_access_key: 'AwsSecretAccessKey', + aws_secret_access_key_tips: '请输入AwsSecretAccessKey', validation: '验证', mode_tips: '请选择验证模式', jdbc_format_tips: 'jdbc连接参数不是一个正确的JSON格式', diff --git a/dolphinscheduler-ui/src/service/modules/data-source/types.ts b/dolphinscheduler-ui/src/service/modules/data-source/types.ts index a9d6addcf89a..f97a82c193b4 100644 --- a/dolphinscheduler-ui/src/service/modules/data-source/types.ts +++ b/dolphinscheduler-ui/src/service/modules/data-source/types.ts @@ -39,6 +39,7 @@ type IDataBase = | 'HANA' | 'ZEPPELIN' | 'DORIS' + | 'SAGEMAKER' type IDataBaseLabel = | 'MYSQL' @@ -59,6 +60,7 @@ type IDataBaseLabel = | 'OCEANBASE' | 'SSH' | 'ZEPPELIN' + | 'SAGEMAKER' interface IDataSource { id?: number diff --git a/dolphinscheduler-ui/src/views/datasource/list/use-form.ts b/dolphinscheduler-ui/src/views/datasource/list/use-form.ts index ef4bf3791144..7c6136c8ed3f 100644 --- a/dolphinscheduler-ui/src/views/datasource/list/use-form.ts +++ b/dolphinscheduler-ui/src/views/datasource/list/use-form.ts @@ -239,7 +239,7 @@ export function useForm(id?: number) { state.showHost = type !== 'ATHENA' state.showPort = type !== 'ATHENA' - state.showAwsRegion = type === 'ATHENA' + state.showAwsRegion = type === 'ATHENA' || type === 'SAGEMAKER' state.showMode = ['AZURESQL', 'REDSHIFT'].includes(type) if (type === 'ORACLE' && !id) { @@ -254,7 +254,7 @@ export function useForm(id?: number) { } else { state.showPrincipal = false } - if (type === 'SSH' || type === 'ZEPPELIN') { + if (type === 'SSH' || type === 'ZEPPELIN' || type == 'SAGEMAKER') { state.showDataBaseName = false state.requiredDataBase = false state.showJDBCConnectParameters = false @@ -267,6 +267,10 @@ export function useForm(id?: number) { state.showPort = false state.showRestEndpoint = true } + if (type == 'SAGEMAKER') { + state.showHost = false + state.showPort = false + } } else { state.showDataBaseName = true state.requiredDataBase = true @@ -420,6 +424,11 @@ export const datasourceType: IDataBaseOptionKeys = { value: 'DORIS', label: 'DORIS', defaultPort: 9030 + }, + SAGEMAKER: { + value: 'SAGEMAKER', + label: 'SAGEMAKER', + defaultPort: 0 } } diff --git a/dolphinscheduler-ui/src/views/projects/task/components/node/fields/use-datasource.ts b/dolphinscheduler-ui/src/views/projects/task/components/node/fields/use-datasource.ts index 3b2756b5e70c..d685b64de207 100644 --- a/dolphinscheduler-ui/src/views/projects/task/components/node/fields/use-datasource.ts +++ b/dolphinscheduler-ui/src/views/projects/task/components/node/fields/use-datasource.ts @@ -142,6 +142,11 @@ export function useDatasource( id: 23, code: 'DORIS', disabled: false + }, + { + id: 24, + code: 'SAGEMAKER', + disabled: false } ] diff --git a/dolphinscheduler-ui/src/views/projects/task/components/node/tasks/use-sagemaker.ts b/dolphinscheduler-ui/src/views/projects/task/components/node/tasks/use-sagemaker.ts index 3bcb13e05217..f89afe2cbcbe 100644 --- a/dolphinscheduler-ui/src/views/projects/task/components/node/tasks/use-sagemaker.ts +++ b/dolphinscheduler-ui/src/views/projects/task/components/node/tasks/use-sagemaker.ts @@ -43,7 +43,12 @@ export function userSagemaker({ workerGroup: 'default', delayTime: 0, timeout: 30, - timeoutNotifyStrategy: ['WARN'] + type: 'SAGEMAKER', + displayRows: 10, + timeoutNotifyStrategy: ['WARN'], + accessKey: '', + secretAccessKey: '', + region: '' } as INodeData) return { @@ -60,6 +65,7 @@ export function userSagemaker({ ...Fields.useFailed(), Fields.useDelayTime(model), ...Fields.useTimeoutAlarm(model), + ...Fields.useDatasource(model), ...Fields.useSagemaker(model), Fields.usePreTasks() ] as IJsonItem[],