diff --git a/exchangis-datasource/exchangis-datasource-core/src/main/java/com/webank/wedatasphere/exchangis/datasource/core/exception/ExchangisDataSourceExceptionCode.java b/exchangis-datasource/exchangis-datasource-core/src/main/java/com/webank/wedatasphere/exchangis/datasource/core/exception/ExchangisDataSourceExceptionCode.java index 52c4eb848..cf5e6083d 100644 --- a/exchangis-datasource/exchangis-datasource-core/src/main/java/com/webank/wedatasphere/exchangis/datasource/core/exception/ExchangisDataSourceExceptionCode.java +++ b/exchangis-datasource/exchangis-datasource-core/src/main/java/com/webank/wedatasphere/exchangis/datasource/core/exception/ExchangisDataSourceExceptionCode.java @@ -21,6 +21,7 @@ public enum ExchangisDataSourceExceptionCode { CLIENT_DATASOURCE_GET_TYPES_ERROR(31015), CLIENT_DATASOURCE_GET_KEY_DEFINES_ERROR(31016), CLIENT_METADATA_GET_PARTITION_PROPS(31017), + CLIENT_METADATA_GET_PARTITION(31018), // 其他错误 PARSE_JSON_ERROR(39000), // Parse Json Error CONTEXT_GET_DATASOURCE_NULL(39001), // DataSource Context Error diff --git a/exchangis-datasource/exchangis-datasource-core/src/main/java/com/webank/wedatasphere/exchangis/datasource/core/service/MetadataInfoService.java b/exchangis-datasource/exchangis-datasource-core/src/main/java/com/webank/wedatasphere/exchangis/datasource/core/service/MetadataInfoService.java index 521a276bc..58d380cea 100644 --- a/exchangis-datasource/exchangis-datasource-core/src/main/java/com/webank/wedatasphere/exchangis/datasource/core/service/MetadataInfoService.java +++ b/exchangis-datasource/exchangis-datasource-core/src/main/java/com/webank/wedatasphere/exchangis/datasource/core/service/MetadataInfoService.java @@ -3,9 +3,10 @@ import com.webank.wedatasphere.exchangis.datasource.core.exception.ExchangisDataSourceException; import com.webank.wedatasphere.exchangis.datasource.core.service.rpc.ServiceRpcClient; +import java.util.List; import java.util.Map; -public interface MetadataInfoService extends ServiceRpcInf { +public interface MetadataInfoService extends ServiceRpcInf { /** * Get properties of partition @@ -17,7 +18,7 @@ public interface MetadataInfoService extends ServiceRpcInf { Map getPartitionProps(String userName, Long dataSourceId, String database, String table, String partition) throws ExchangisDataSourceException; - Map getPartitionProps(ServiceRpcClient rpcClient, + Map getPartitionProps(ServiceRpcClient rpcClient, String userName, Long dataSourceId, String database, String table, String partition) throws ExchangisDataSourceException; @@ -31,6 +32,17 @@ Map getPartitionProps(ServiceRpcClient rpcClient, Map getTableProps(String userName, Long dataSourceId, String database, String table) throws ExchangisDataSourceException; - Map getTableProps(ServiceRpcClient rpcClient, String userName, Long dataSourceId, + Map getTableProps(ServiceRpcClient rpcClient, String userName, Long dataSourceId, String database, String table) throws ExchangisDataSourceException; + + /** + * Get partition keys + * @param userName userName + * @param dataSourceId data source id + * @param database database + * @param table table + * @return + * @throws ExchangisDataSourceException + */ + List getPartitionKeys(String userName, Long dataSourceId, String database, String table) throws ExchangisDataSourceException; } diff --git a/exchangis-datasource/exchangis-datasource-core/src/main/java/com/webank/wedatasphere/exchangis/datasource/core/service/ServiceRpcInf.java b/exchangis-datasource/exchangis-datasource-core/src/main/java/com/webank/wedatasphere/exchangis/datasource/core/service/ServiceRpcInf.java index fa29758fd..b7c5ca7d8 100644 --- a/exchangis-datasource/exchangis-datasource-core/src/main/java/com/webank/wedatasphere/exchangis/datasource/core/service/ServiceRpcInf.java +++ b/exchangis-datasource/exchangis-datasource-core/src/main/java/com/webank/wedatasphere/exchangis/datasource/core/service/ServiceRpcInf.java @@ -2,8 +2,8 @@ /** * RPC service - * @param */ -public interface ServiceRpcInf { +public interface ServiceRpcInf { + Class getClientClass(); } diff --git a/exchangis-datasource/exchangis-datasource-core/src/main/java/com/webank/wedatasphere/exchangis/datasource/core/service/rpc/AbstractServiceRpcDispatcher.java b/exchangis-datasource/exchangis-datasource-core/src/main/java/com/webank/wedatasphere/exchangis/datasource/core/service/rpc/AbstractServiceRpcDispatcher.java index b784d3d58..ab8fae6eb 100644 --- a/exchangis-datasource/exchangis-datasource-core/src/main/java/com/webank/wedatasphere/exchangis/datasource/core/service/rpc/AbstractServiceRpcDispatcher.java +++ b/exchangis-datasource/exchangis-datasource-core/src/main/java/com/webank/wedatasphere/exchangis/datasource/core/service/rpc/AbstractServiceRpcDispatcher.java @@ -24,6 +24,7 @@ public U dispatch(C remoteClient, T operation) throws ExchangisServiceRpcExc try { return execute(remoteClient, operation); }catch(Exception e){ + if (e instanceof ExchangisServiceRpcException){ throw e; } diff --git a/exchangis-datasource/exchangis-datasource-core/src/main/java/com/webank/wedatasphere/exchangis/datasource/core/ui/ElementUI.java b/exchangis-datasource/exchangis-datasource-core/src/main/java/com/webank/wedatasphere/exchangis/datasource/core/ui/ElementUI.java index 6e7575531..5589218fd 100644 --- a/exchangis-datasource/exchangis-datasource-core/src/main/java/com/webank/wedatasphere/exchangis/datasource/core/ui/ElementUI.java +++ b/exchangis-datasource/exchangis-datasource-core/src/main/java/com/webank/wedatasphere/exchangis/datasource/core/ui/ElementUI.java @@ -1,22 +1,51 @@ package com.webank.wedatasphere.exchangis.datasource.core.ui; -public interface ElementUI { - String TEXTAREA = "TEXTAREA"; - String INPUT = "INPUT"; - String OPTION = "OPTION"; - String MAP = "MAP"; +import java.util.Map; +public interface ElementUI { + /** + * Type enum + */ + enum Type { + NONE, TEXTAREA, INPUT, OPTION, MAP + } + + /** + * Field name + * @return string + */ String getField(); + /** + * Label + * @return label string + */ String getLabel(); - // 类型 + /** + * Type name + * @return string + */ String getType(); Integer getSort(); + /** + * Value store + * @return + */ T getValue(); + /** + * Default value + * @return + */ T getDefaultValue(); + /** + * Get value from params + * @param params + * @return + */ + void setValue(Map params); } diff --git a/exchangis-datasource/exchangis-datasource-core/src/main/java/com/webank/wedatasphere/exchangis/datasource/core/ui/InputElementUI.java b/exchangis-datasource/exchangis-datasource-core/src/main/java/com/webank/wedatasphere/exchangis/datasource/core/ui/InputElementUI.java index 419277769..1c2585228 100644 --- a/exchangis-datasource/exchangis-datasource-core/src/main/java/com/webank/wedatasphere/exchangis/datasource/core/ui/InputElementUI.java +++ b/exchangis-datasource/exchangis-datasource-core/src/main/java/com/webank/wedatasphere/exchangis/datasource/core/ui/InputElementUI.java @@ -1,5 +1,9 @@ package com.webank.wedatasphere.exchangis.datasource.core.ui; +import com.webank.wedatasphere.exchangis.datasource.core.utils.Json; + +import java.util.Map; + public class InputElementUI implements ElementUI { private String key; private String field; @@ -50,7 +54,7 @@ public void setLabel(String label) { @Override public String getType() { - return ElementUI.INPUT; + return Type.INPUT.name(); } @Override @@ -74,6 +78,12 @@ public void setValue(String value) { @Override public String getDefaultValue() { return defaultValue; } + @Override + public void setValue(Map params) { + // Convert to json string directly + this.value = Json.toJson(params, null); + } + public void setDefaultValue(String defaultValue) { this.defaultValue = defaultValue; } public String getUnit() { diff --git a/exchangis-datasource/exchangis-datasource-core/src/main/java/com/webank/wedatasphere/exchangis/datasource/core/ui/MapElementUI.java b/exchangis-datasource/exchangis-datasource-core/src/main/java/com/webank/wedatasphere/exchangis/datasource/core/ui/MapElementUI.java index 86a239e90..b8f5d8945 100644 --- a/exchangis-datasource/exchangis-datasource-core/src/main/java/com/webank/wedatasphere/exchangis/datasource/core/ui/MapElementUI.java +++ b/exchangis-datasource/exchangis-datasource-core/src/main/java/com/webank/wedatasphere/exchangis/datasource/core/ui/MapElementUI.java @@ -1,6 +1,9 @@ package com.webank.wedatasphere.exchangis.datasource.core.ui; +import org.apache.commons.lang.StringUtils; + import java.util.Map; +import java.util.Objects; public class MapElementUI implements ElementUI> { private String key; @@ -52,7 +55,7 @@ public void setLabel(String label) { @Override public String getType() { - return ElementUI.MAP; + return Type.MAP.name(); } @Override @@ -69,6 +72,7 @@ public Map getValue() { return value; } + public void setValue(Map value) { this.value = value; } @@ -76,6 +80,7 @@ public void setValue(Map value) { @Override public Map getDefaultValue() { return defaultValue; } + public void setDefaultValue(Map defaultValue) { this.defaultValue = defaultValue; } public String getUnit() { @@ -113,4 +118,8 @@ public void setValidateRange(String validateRange) { public String getValidateMsg() { return validateMsg; } public void setValidateMsg(String validateMsg) { this.validateMsg = validateMsg; } + + private boolean isBasicType(Class clz){ + return false; + } } diff --git a/exchangis-datasource/exchangis-datasource-core/src/main/java/com/webank/wedatasphere/exchangis/datasource/core/ui/OptionElementUI.java b/exchangis-datasource/exchangis-datasource-core/src/main/java/com/webank/wedatasphere/exchangis/datasource/core/ui/OptionElementUI.java index 69103654e..93e2da083 100644 --- a/exchangis-datasource/exchangis-datasource-core/src/main/java/com/webank/wedatasphere/exchangis/datasource/core/ui/OptionElementUI.java +++ b/exchangis-datasource/exchangis-datasource-core/src/main/java/com/webank/wedatasphere/exchangis/datasource/core/ui/OptionElementUI.java @@ -1,6 +1,9 @@ package com.webank.wedatasphere.exchangis.datasource.core.ui; +import com.webank.wedatasphere.exchangis.datasource.core.utils.Json; + import java.util.Collection; +import java.util.Map; public class OptionElementUI implements ElementUI { private String key; @@ -27,7 +30,7 @@ public String getField() { @Override public String getType() { - return ElementUI.OPTION; + return Type.OPTION.name(); } public void setField(String field) { @@ -62,6 +65,11 @@ public void setValue(String value) { @Override public String getDefaultValue() { return defaultValue; } + @Override + public void setValue(Map params) { + this.value = Json.toJson(params.values(), null); + } + public void setDefaultValue(String defaultValue) { this.defaultValue = defaultValue; } @Override diff --git a/exchangis-datasource/exchangis-datasource-core/src/main/java/com/webank/wedatasphere/exchangis/datasource/core/ui/builder/DefaultElementUIFactory.java b/exchangis-datasource/exchangis-datasource-core/src/main/java/com/webank/wedatasphere/exchangis/datasource/core/ui/builder/DefaultElementUIFactory.java new file mode 100644 index 000000000..bc87979e2 --- /dev/null +++ b/exchangis-datasource/exchangis-datasource-core/src/main/java/com/webank/wedatasphere/exchangis/datasource/core/ui/builder/DefaultElementUIFactory.java @@ -0,0 +1,83 @@ +package com.webank.wedatasphere.exchangis.datasource.core.ui.builder; + +import com.webank.wedatasphere.exchangis.datasource.core.ui.ElementUI; +import com.webank.wedatasphere.exchangis.datasource.core.ui.InputElementUI; + +import java.lang.annotation.ElementType; +import java.util.HashMap; +import java.util.Map; +import java.util.Objects; +import java.util.Optional; +import java.util.concurrent.atomic.AtomicReference; +import java.util.function.Function; + +/** + * Default element factory + */ +public class DefaultElementUIFactory implements ElementUIFactory{ + /** + * Element builders holder + */ + private Map>> builders = new HashMap<>(); + + + @Override + @SuppressWarnings("unchecked") + public void register(String type, Function> builder, Class inputType) { + builders.putIfAbsent(new Identify(type, inputType), (Function>) builder); + } + + @Override + @SuppressWarnings("unchecked") + public ElementUI createElement(String type, Object input, Class inputType) { + Identify identify = new Identify(type, inputType); + AtomicReference> elementUI = new AtomicReference<>(); + Optional.ofNullable(builders.get(identify)).ifPresent(builder -> { + elementUI.set((ElementUI) builder.apply(input)); + }); + return elementUI.get(); + } + + /** + * Identify for element builder + */ + private static class Identify{ + + /** + * Type + */ + private final String type; + + /** + * Input class + */ + private final Class inputClass; + + public Identify(String type, Class inputClass){ + this.type = type; + this.inputClass = inputClass; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + Identify identify = (Identify) o; + return Objects.equals(type, identify.type) && Objects.equals(inputClass, identify.inputClass); + } + + @Override + public int hashCode() { + return Objects.hash(type, inputClass); + } + + @Override + public String toString() { + return "Identify{" + + "type='" + type + '\'' + + ", inputClass=" + inputClass.getCanonicalName() + + '}'; + } + } + +} diff --git a/exchangis-datasource/exchangis-datasource-core/src/main/java/com/webank/wedatasphere/exchangis/datasource/core/ui/builder/ElementUIFactory.java b/exchangis-datasource/exchangis-datasource-core/src/main/java/com/webank/wedatasphere/exchangis/datasource/core/ui/builder/ElementUIFactory.java new file mode 100644 index 000000000..62befc4dd --- /dev/null +++ b/exchangis-datasource/exchangis-datasource-core/src/main/java/com/webank/wedatasphere/exchangis/datasource/core/ui/builder/ElementUIFactory.java @@ -0,0 +1,31 @@ +package com.webank.wedatasphere.exchangis.datasource.core.ui.builder; + +import com.webank.wedatasphere.exchangis.datasource.core.ui.ElementUI; + +import java.util.function.Function; + +/** + * Element factory + */ +public interface ElementUIFactory { + + /** + * Register the element builder + * @param type type + * @param builder builder + * @param inputType input type + * @param T + * @param R + */ + void register(String type, Function> builder, Class inputType); + + + /** + * Create element + * @param type type + * @param input input object + * @param element value type + * @return element + */ + ElementUI createElement(String type, Object input, Class inputType); +} diff --git a/exchangis-datasource/exchangis-datasource-core/src/main/java/com/webank/wedatasphere/exchangis/datasource/core/ui/builder/SpringElementUIFactory.java b/exchangis-datasource/exchangis-datasource-core/src/main/java/com/webank/wedatasphere/exchangis/datasource/core/ui/builder/SpringElementUIFactory.java new file mode 100644 index 000000000..15872ea3f --- /dev/null +++ b/exchangis-datasource/exchangis-datasource-core/src/main/java/com/webank/wedatasphere/exchangis/datasource/core/ui/builder/SpringElementUIFactory.java @@ -0,0 +1,49 @@ +package com.webank.wedatasphere.exchangis.datasource.core.ui.builder; + +import com.webank.wedatasphere.exchangis.datasource.core.ui.ElementUI; +import com.webank.wedatasphere.exchangis.datasource.core.ui.InputElementUI; +import com.webank.wedatasphere.exchangis.datasource.core.ui.MapElementUI; +import com.webank.wedatasphere.exchangis.datasource.core.ui.OptionElementUI; +import org.springframework.stereotype.Component; + +import javax.annotation.PostConstruct; +import java.util.HashMap; +import java.util.Map; +import java.util.function.Function; + +/** + * Default element factory in spring + */ +@Component +public class SpringElementUIFactory extends DefaultElementUIFactory{ + + @PostConstruct + public void init(){ + super.register(ElementUI.Type.MAP.name(), (Function, MapElementUI>) params -> + setElementValue(new MapElementUI(), params), Map.class); + super.register(ElementUI.Type.INPUT.name(), (Function, InputElementUI>) params -> + setElementValue(new InputElementUI(), params), Map.class); + super.register(ElementUI.Type.OPTION.name(), (Function, OptionElementUI>) params -> + setElementValue(new OptionElementUI(), params), Map.class); + } + + /** + * Set the params into element and return + * @param element element + * @param params params Map + * @param R + * @return + */ + private >R setElementValue(R element, Map params){ + element.setValue(params); + return element; + } + + public static void main(String[] args){ + SpringElementUIFactory elementUIFactory = new SpringElementUIFactory(); + elementUIFactory.init(); + Map map = new HashMap<>(); + map.putIfAbsent("hello", "world"); + System.out.println(elementUIFactory.createElement(ElementUI.Type.MAP.name(), map, Map.class).getValue()); + } +} diff --git a/exchangis-datasource/exchangis-datasource-linkis/src/main/java/com/webank/wedatasphere/exchangis/datasource/linkis/service/LinkisMetadataInfoService.java b/exchangis-datasource/exchangis-datasource-linkis/src/main/java/com/webank/wedatasphere/exchangis/datasource/linkis/service/LinkisMetadataInfoService.java index 49effec77..ecf9bd872 100644 --- a/exchangis-datasource/exchangis-datasource-linkis/src/main/java/com/webank/wedatasphere/exchangis/datasource/linkis/service/LinkisMetadataInfoService.java +++ b/exchangis-datasource/exchangis-datasource-linkis/src/main/java/com/webank/wedatasphere/exchangis/datasource/linkis/service/LinkisMetadataInfoService.java @@ -4,23 +4,37 @@ import com.webank.wedatasphere.exchangis.datasource.core.exception.ExchangisServiceRpcException; import com.webank.wedatasphere.exchangis.datasource.core.service.MetadataInfoService; import com.webank.wedatasphere.exchangis.datasource.core.service.rpc.ServiceRpcClient; +import com.webank.wedatasphere.exchangis.datasource.linkis.ExchangisLinkisRemoteClient; import com.webank.wedatasphere.exchangis.datasource.linkis.request.MetadataGetPartitionPropsAction; import com.webank.wedatasphere.exchangis.datasource.linkis.response.MetadataGetPartitionPropsResult; import com.webank.wedatasphere.exchangis.datasource.linkis.service.rpc.LinkisDataSourceServiceOperation; import com.webank.wedatasphere.exchangis.datasource.linkis.service.rpc.LinkisDataSourceServiceRpcDispatcher; import org.apache.linkis.datasource.client.impl.LinkisMetaDataRemoteClient; +import org.apache.linkis.datasource.client.request.MetadataGetPartitionsAction; import org.apache.linkis.datasource.client.request.MetadataGetTablePropsAction; +import org.apache.linkis.datasource.client.response.MetadataGetPartitionsResult; import org.apache.linkis.datasource.client.response.MetadataGetTablePropsResult; +import java.util.List; import java.util.Map; -import static com.webank.wedatasphere.exchangis.datasource.core.exception.ExchangisDataSourceExceptionCode.CLIENT_METADATA_GET_PARTITION_PROPS; +import static com.webank.wedatasphere.exchangis.datasource.core.exception.ExchangisDataSourceExceptionCode.*; /** * Linkis to fetch metadata info */ public class LinkisMetadataInfoService extends LinkisDataSourceServiceRpcDispatcher - implements MetadataInfoService { + implements MetadataInfoService { + + @Override + public Class getClientClass() { + return LinkisMetaDataRemoteClient.class; + } + + @Override + public ServiceRpcClient getDefaultRemoteClient() { + return ExchangisLinkisRemoteClient::getLinkisMetadataRemoteClient; + } @Override public Map getPartitionProps(String userName, Long dataSourceId, @@ -29,10 +43,11 @@ public Map getPartitionProps(String userName, Long dataSourceId, } @Override - public Map getPartitionProps(ServiceRpcClient rpcClient, + @SuppressWarnings("unchecked") + public Map getPartitionProps(ServiceRpcClient rpcClient, String userName, Long dataSourceId, String database, String table, String partition) throws ExchangisDataSourceException { - MetadataGetPartitionPropsResult result = dispatch(rpcClient, new LinkisDataSourceServiceOperation(() -> { + MetadataGetPartitionPropsResult result = dispatch((ServiceRpcClient) rpcClient, new LinkisDataSourceServiceOperation(() -> { MetadataGetPartitionPropsAction action = new MetadataGetPartitionPropsAction(dataSourceId, database, table, partition, LINKIS_RPC_CLIENT_SYSTEM.getValue()); action.setUser(userName); @@ -47,11 +62,20 @@ public Map getTableProps(String userName, Long dataSourceId, Str } @Override - public Map getTableProps(ServiceRpcClient rpcClient, String userName, Long dataSourceId, String database, String table) throws ExchangisDataSourceException { - MetadataGetTablePropsResult result = dispatch(rpcClient, new LinkisDataSourceServiceOperation(() -> MetadataGetTablePropsAction.builder() + @SuppressWarnings("unchecked") + public Map getTableProps(ServiceRpcClient rpcClient, String userName, Long dataSourceId, String database, String table) throws ExchangisDataSourceException { + MetadataGetTablePropsResult result = dispatch((ServiceRpcClient) rpcClient, new LinkisDataSourceServiceOperation(() -> MetadataGetTablePropsAction.builder() .setDataSourceId(dataSourceId).setDatabase(database).setTable(table) - .setUser(userName).setSystem(LINKIS_RPC_CLIENT_SYSTEM.getValue()).build()), 0, ""); + .setUser(userName).setSystem(LINKIS_RPC_CLIENT_SYSTEM.getValue()).build()), CLIENT_METADATA_GET_TABLES_ERROR.getCode(), "getTableProps"); return result.props(); +} + + @Override + public List getPartitionKeys(String userName, Long dataSourceId, String database, String table) throws ExchangisDataSourceException { + MetadataGetPartitionsResult result = dispatch(getDefaultRemoteClient(), new LinkisDataSourceServiceOperation(() -> MetadataGetPartitionsAction.builder() + .setDataSourceId(String.valueOf(dataSourceId)).setDatabase(database).setTable(table) + .setUser(userName).setSystem(LINKIS_RPC_CLIENT_SYSTEM.getValue()).build()), CLIENT_METADATA_GET_PARTITION.getCode(), "getPartitionKeys"); + return result.getPartitionInfo().getPartKeys(); } diff --git a/exchangis-datasource/exchangis-datasource-server/src/main/java/com/webank/wedatasphere/exchangis/datasource/server/ExchangisDataSourceAutoConfiguration.java b/exchangis-datasource/exchangis-datasource-server/src/main/java/com/webank/wedatasphere/exchangis/datasource/server/ExchangisDataSourceAutoConfiguration.java new file mode 100644 index 000000000..2146c2595 --- /dev/null +++ b/exchangis-datasource/exchangis-datasource-server/src/main/java/com/webank/wedatasphere/exchangis/datasource/server/ExchangisDataSourceAutoConfiguration.java @@ -0,0 +1,20 @@ +package com.webank.wedatasphere.exchangis.datasource.server; + +import com.webank.wedatasphere.exchangis.datasource.core.service.MetadataInfoService; +import com.webank.wedatasphere.exchangis.datasource.linkis.service.LinkisMetadataInfoService; +import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; + +/** + * Auto configure the core beans + */ +@Configuration +public class ExchangisDataSourceAutoConfiguration { + + @Bean + @ConditionalOnMissingBean(MetadataInfoService.class) + public MetadataInfoService metadataInfoService(){ + return new LinkisMetadataInfoService(); + } +} diff --git a/exchangis-datasource/exchangis-datasource-server/src/main/java/com/webank/wedatasphere/exchangis/datasource/server/restful/api/ExchangisDataSourceRenderRestfulApi.java b/exchangis-datasource/exchangis-datasource-server/src/main/java/com/webank/wedatasphere/exchangis/datasource/server/restful/api/ExchangisDataSourceRenderRestfulApi.java new file mode 100644 index 000000000..e64ba041b --- /dev/null +++ b/exchangis-datasource/exchangis-datasource-server/src/main/java/com/webank/wedatasphere/exchangis/datasource/server/restful/api/ExchangisDataSourceRenderRestfulApi.java @@ -0,0 +1,62 @@ +package com.webank.wedatasphere.exchangis.datasource.server.restful.api; + +import com.webank.wedatasphere.exchangis.datasource.core.service.MetadataInfoService; +import com.webank.wedatasphere.exchangis.datasource.core.ui.ElementUI; +import com.webank.wedatasphere.exchangis.datasource.core.ui.builder.ElementUIFactory; +import com.webank.wedatasphere.exchangis.datasource.service.DataSourceRenderService; +import org.apache.linkis.datasource.client.impl.LinkisDataSourceRemoteClient; +import org.apache.linkis.server.Message; +import org.apache.linkis.server.security.SecurityFilter; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.web.bind.annotation.*; + +import javax.annotation.Resource; +import javax.servlet.http.HttpServletRequest; +import java.util.Locale; +import java.util.Objects; + +/** + * Expose the ui interface to front-end rendering + */ +@RestController +@RequestMapping(value = "exchangis/datasources/render", produces = {"application/json;charset=utf-8"}) +public class ExchangisDataSourceRenderRestfulApi { + + private static final Logger LOG = LoggerFactory.getLogger(ExchangisDataSourceRenderRestfulApi.class); + + @Resource + private DataSourceRenderService renderService; + + @RequestMapping(value = "/partition/element/{elementType:\\w+}", method = RequestMethod.GET) + public Message partition(@PathVariable("elementType") String type, + @RequestParam("dataSourceId") Long dataSourceId, + @RequestParam("database") String database, + @RequestParam("table") String table, HttpServletRequest request){ + String userName = SecurityFilter.getLoginUsername(request); + ElementUI.Type uiType; + try { + uiType = ElementUI.Type.valueOf(type.toUpperCase(Locale.ROOT)); + } catch (Exception e){ + return Message.error("Element type: [" + type +"] is not support (不兼容的元素类型)"); + } + Message result = Message.ok(); + try{ + ElementUI elementUI = renderService.getPartitionAndRender(userName, dataSourceId, database, table, uiType); + result.data("type", uiType.name()); + if (Objects.nonNull(elementUI)){ + result.data("render", elementUI.getValue()); + } + }catch(Exception e){ + String uiMessage = "Load to render partition info Failed (加载渲染分区信息失败)"; + LOG.error(uiMessage + ", reason: " + e.getMessage(), e); + result = Message.error(uiMessage); + } + result.setMethod("/api/rest_j/v1/exchangis/datasources/render/partition/element/" + type); + return result; + } + + public static void main(String[] args){ + ElementUI.Type.valueOf("map".toUpperCase(Locale.ROOT)); + } +} diff --git a/exchangis-datasource/exchangis-datasource-server/src/main/java/com/webank/wedatasphere/exchangis/datasource/server/restful/api/ExchangisDataSourceRestfulApi.java b/exchangis-datasource/exchangis-datasource-server/src/main/java/com/webank/wedatasphere/exchangis/datasource/server/restful/api/ExchangisDataSourceRestfulApi.java index 98c1c1aaa..6ac152306 100644 --- a/exchangis-datasource/exchangis-datasource-server/src/main/java/com/webank/wedatasphere/exchangis/datasource/server/restful/api/ExchangisDataSourceRestfulApi.java +++ b/exchangis-datasource/exchangis-datasource-server/src/main/java/com/webank/wedatasphere/exchangis/datasource/server/restful/api/ExchangisDataSourceRestfulApi.java @@ -17,7 +17,7 @@ import java.util.Map; @RestController -@RequestMapping(value = "exchangis", produces = {"application/json;charset=utf-8"}) +@RequestMapping(value = "exchangis/datasources", produces = {"application/json;charset=utf-8"}) public class ExchangisDataSourceRestfulApi { private static final Logger LOG = LoggerFactory.getLogger(ExchangisDataSourceRestfulApi.class); @@ -30,19 +30,19 @@ public ExchangisDataSourceRestfulApi(ExchangisDataSourceService exchangisDataSou } // list all datasource types - @RequestMapping( value = "datasources/type", method = RequestMethod.GET) + @RequestMapping( value = "/type", method = RequestMethod.GET) public Message listDataSourceTypes(HttpServletRequest request) throws Exception { return this.exchangisDataSourceService.listDataSources(request); } // query paged datasource - @RequestMapping( value = "datasources/query", method = {RequestMethod.GET,RequestMethod.POST}) + @RequestMapping( value = "/query", method = {RequestMethod.GET,RequestMethod.POST}) public Message create(HttpServletRequest request, @RequestBody DataSourceQueryVO vo) throws Exception { return this.exchangisDataSourceService.queryDataSources(request, vo); } // list all datasources - @RequestMapping( value = "datasources", method = RequestMethod.GET) + @RequestMapping( value = "", method = RequestMethod.GET) @Deprecated public Message listAllDataSources( HttpServletRequest request, @@ -55,7 +55,7 @@ public Message listAllDataSources( } // get datasource key define - @RequestMapping( value = "datasources/types/{dataSourceTypeId}/keydefines", method = RequestMethod.GET) + @RequestMapping( value = "/types/{dataSourceTypeId}/keydefines", method = RequestMethod.GET) public Message getDataSourceKeyDefine( HttpServletRequest request, @PathVariable("dataSourceTypeId") Long dataSourceTypeId @@ -65,13 +65,13 @@ public Message getDataSourceKeyDefine( // get datasource version list - @RequestMapping( value = "datasources/{id}/versions", method = RequestMethod.GET) + @RequestMapping( value = "/{id}/versions", method = RequestMethod.GET) public Message getDataSourceVersionsById(HttpServletRequest request, @PathVariable("id") Long id) throws Exception { return this.exchangisDataSourceService.getDataSourceVersionsById(request, id); } // create datasource - @RequestMapping( value = "datasources", method = RequestMethod.POST) + @RequestMapping( value = "", method = RequestMethod.POST) public Message create(HttpServletRequest request, /*@PathParam("type") String type, */@RequestBody Map json) throws Exception { Message message = null; try{ @@ -86,37 +86,37 @@ public Message create(HttpServletRequest request, /*@PathParam("type") String ty } // get datasource details - @RequestMapping( value = "datasources/{id}", method = RequestMethod.GET) + @RequestMapping( value = "/{id}", method = RequestMethod.GET) public Message getDataSourceInfoById(HttpServletRequest request, @PathVariable("id") Long id, @QueryParam(value = "versionId") String versionId) throws Exception { return this.exchangisDataSourceService.getDataSource(request, id, versionId); } - @RequestMapping( value = "datasources/{id}/connect_params", method = RequestMethod.GET) + @RequestMapping( value = "/{id}/connect_params", method = RequestMethod.GET) public Message getDataSourceConnectParamsById(HttpServletRequest request, @PathVariable("id") Long id) throws Exception { return this.exchangisDataSourceService.getDataSourceConnectParamsById(request, id); } // update datasource and parameters (insert new record in datasource_version table) - @RequestMapping( value = "datasources/{id}", method = RequestMethod.PUT) + @RequestMapping( value = "/{id}", method = RequestMethod.PUT) public Message update(HttpServletRequest request,/* @PathParam("type") String type, */@PathVariable("id") Long id, @RequestBody Map json) throws Exception { return this.exchangisDataSourceService.updateDataSource(request, /*type, */id, json); } // publish datasource - @RequestMapping( value = "datasources/{id}/{version}/publish", method = RequestMethod.PUT) + @RequestMapping( value = "/{id}/{version}/publish", method = RequestMethod.PUT) public Message publishDataSource(HttpServletRequest request,/* @PathParam("type") String type, */@PathVariable("id") Long id, @PathVariable("version") Long version) throws Exception { return this.exchangisDataSourceService.publishDataSource(request, /*type, */id, version); } // expire datasource - @RequestMapping( value = "datasources/{id}/expire", method = RequestMethod.PUT) + @RequestMapping( value = "/{id}/expire", method = RequestMethod.PUT) public Message expireDataSource(HttpServletRequest request,/* @PathParam("type") String type, */@PathVariable("id") Long id) throws Exception { return this.exchangisDataSourceService.expireDataSource(request, /*type, */id); } // test datasource connect - @RequestMapping( value = "datasources/{id}/{version}/connect", method = RequestMethod.PUT) + @RequestMapping( value = "/{id}/{version}/connect", method = RequestMethod.PUT) public Message testConnect(HttpServletRequest request,/* @PathParam("type") String type, */@PathVariable("id") Long id, @PathVariable("version") Long version) throws Exception { Message message = null; @@ -132,7 +132,7 @@ public Message testConnect(HttpServletRequest request,/* @PathParam("type") Stri } // delete datasource (physical) - @RequestMapping( value = "datasources/{id}", method = RequestMethod.DELETE) + @RequestMapping( value = "/{id}", method = RequestMethod.DELETE) public Message delete(HttpServletRequest request, /*@PathParam("type") String type, */@PathVariable("id") Long id) throws Exception { Message message = null; try{ @@ -146,7 +146,7 @@ public Message delete(HttpServletRequest request, /*@PathParam("type") String ty //return this.exchangisDataSourceService.deleteDataSource(request, /*type, */id); } - @RequestMapping( value = "datasources/{type}/{id}/dbs", method = RequestMethod.GET) + @RequestMapping( value = "/{type}/{id}/dbs", method = RequestMethod.GET) public Message queryDataSourceDBs(HttpServletRequest request, @PathVariable("type") String type, @PathVariable("id") Long id) throws Exception { Message message = null; try{ @@ -160,25 +160,25 @@ public Message queryDataSourceDBs(HttpServletRequest request, @PathVariable("typ //return this.exchangisDataSourceService.queryDataSourceDBs(request, type, id); } - @RequestMapping( value = "datasources/{type}/{id}/dbs/{dbName}/tables", method = RequestMethod.GET) + @RequestMapping( value = "/{type}/{id}/dbs/{dbName}/tables", method = RequestMethod.GET) public Message queryDataSourceDBTables(HttpServletRequest request, @PathVariable("type") String type, @PathVariable("id") Long id, @PathVariable("dbName") String dbName) throws Exception { return this.exchangisDataSourceService.queryDataSourceDBTables(request, type, id, dbName); } - @RequestMapping( value = "datasources/{type}/{id}/dbs/{dbName}/tables/{tableName}/fields", method = RequestMethod.GET) + @RequestMapping( value = "/{type}/{id}/dbs/{dbName}/tables/{tableName}/fields", method = RequestMethod.GET) public Message queryDataSourceDBTableFields(HttpServletRequest request, @PathVariable("type") String type, @PathVariable("id") Long id, @PathVariable("dbName") String dbName, @PathVariable("tableName") String tableName) throws Exception { return this.exchangisDataSourceService.queryDataSourceDBTableFields(request, type, id, dbName, tableName); } - @RequestMapping( value = "datasources/fieldsmapping", method = RequestMethod.POST) + @RequestMapping( value = "/fieldsmapping", method = RequestMethod.POST) public Message queryDataSourceDBTableFieldsMapping(HttpServletRequest request, @RequestBody FieldMappingVO vo) throws Exception { return this.exchangisDataSourceService.queryDataSourceDBTableFieldsMapping(request, vo); } - @RequestMapping( value = "datasources/{engine}/{type}/params/ui", method = RequestMethod.GET) + @RequestMapping( value = "/{engine}/{type}/params/ui", method = RequestMethod.GET) public Message getParamsUI( HttpServletRequest request, @PathVariable("engine") String engine, diff --git a/exchangis-datasource/exchangis-datasource-service/src/main/java/com/webank/wedatasphere/exchangis/datasource/service/AbstractDataSourceService.java b/exchangis-datasource/exchangis-datasource-service/src/main/java/com/webank/wedatasphere/exchangis/datasource/service/AbstractDataSourceService.java index 5e3f18bd2..4c519c9cf 100644 --- a/exchangis-datasource/exchangis-datasource-service/src/main/java/com/webank/wedatasphere/exchangis/datasource/service/AbstractDataSourceService.java +++ b/exchangis-datasource/exchangis-datasource-service/src/main/java/com/webank/wedatasphere/exchangis/datasource/service/AbstractDataSourceService.java @@ -19,6 +19,7 @@ import com.webank.wedatasphere.exchangis.datasource.core.vo.ExchangisJobParamsContent; import com.webank.wedatasphere.exchangis.datasource.core.vo.ExchangisJobTransformsContent; import com.webank.wedatasphere.exchangis.datasource.dto.GetDataSourceInfoResultDTO; +import org.apache.commons.lang.StringUtils; import org.apache.linkis.datasource.client.impl.LinkisDataSourceRemoteClient; import org.apache.linkis.datasource.client.request.GetInfoByDataSourceIdAction; import org.apache.linkis.datasourcemanager.common.exception.JsonErrorException; @@ -277,12 +278,19 @@ private ExchangisJobParamsContent.ExchangisJobParamsItem getJobParamsItem(String private ElementUI fillElementUIValue(ExchangisJobParamConfig config, Object value) { String uiType = config.getUiType(); - switch (uiType) { - case ElementUI.OPTION: + ElementUI.Type uiTypeEnum; + try { + uiTypeEnum = StringUtils.isNotBlank(uiType)? + ElementUI.Type.valueOf(uiType.toUpperCase(Locale.ROOT)) : ElementUI.Type.NONE; + }catch (Exception e){ + uiTypeEnum = ElementUI.Type.NONE; + } + switch (uiTypeEnum) { + case OPTION: return fillOptionElementUIValue(config, String.valueOf(value)); - case ElementUI.INPUT: + case INPUT: return fillInputElementUIValue(config, String.valueOf(value)); - case ElementUI.MAP: + case MAP: Map mapElement = null; try { mapElement = Json.fromJson(String.valueOf(value), Map.class); diff --git a/exchangis-datasource/exchangis-datasource-service/src/main/java/com/webank/wedatasphere/exchangis/datasource/service/DataSourceRenderService.java b/exchangis-datasource/exchangis-datasource-service/src/main/java/com/webank/wedatasphere/exchangis/datasource/service/DataSourceRenderService.java new file mode 100644 index 000000000..8e237fa48 --- /dev/null +++ b/exchangis-datasource/exchangis-datasource-service/src/main/java/com/webank/wedatasphere/exchangis/datasource/service/DataSourceRenderService.java @@ -0,0 +1,16 @@ +package com.webank.wedatasphere.exchangis.datasource.service; + +import com.webank.wedatasphere.exchangis.datasource.core.exception.ExchangisDataSourceException; +import com.webank.wedatasphere.exchangis.datasource.core.ui.ElementUI; + +public interface DataSourceRenderService { + + /** + * Get the partition info and render to element + * @param userName userName + * @return element ui + */ + ElementUI getPartitionAndRender(String userName, + Long dataSourceId, String database, String table, ElementUI.Type uiType) throws ExchangisDataSourceException; + +} diff --git a/exchangis-datasource/exchangis-datasource-service/src/main/java/com/webank/wedatasphere/exchangis/datasource/service/ExchangisDataSourceService.java b/exchangis-datasource/exchangis-datasource-service/src/main/java/com/webank/wedatasphere/exchangis/datasource/service/ExchangisDataSourceService.java index 2b6b4d1c1..adde34f21 100644 --- a/exchangis-datasource/exchangis-datasource-service/src/main/java/com/webank/wedatasphere/exchangis/datasource/service/ExchangisDataSourceService.java +++ b/exchangis-datasource/exchangis-datasource-service/src/main/java/com/webank/wedatasphere/exchangis/datasource/service/ExchangisDataSourceService.java @@ -636,7 +636,7 @@ public Message queryDataSources(HttpServletRequest request, DataSourceQueryVO vo Message message = Message.ok(); message.data("list", dataSources); - message.data("total", dataSources.size()); + message.data("total", result.getTotalPage() * pageSize); return message; //return Message.ok().data("list", dataSources); } diff --git a/exchangis-datasource/exchangis-datasource-service/src/main/java/com/webank/wedatasphere/exchangis/datasource/service/impl/DefaultDataSourceRenderService.java b/exchangis-datasource/exchangis-datasource-service/src/main/java/com/webank/wedatasphere/exchangis/datasource/service/impl/DefaultDataSourceRenderService.java new file mode 100644 index 000000000..397cadc5c --- /dev/null +++ b/exchangis-datasource/exchangis-datasource-service/src/main/java/com/webank/wedatasphere/exchangis/datasource/service/impl/DefaultDataSourceRenderService.java @@ -0,0 +1,49 @@ +package com.webank.wedatasphere.exchangis.datasource.service.impl; + +import com.webank.wedatasphere.exchangis.datasource.core.exception.ExchangisDataSourceException; +import com.webank.wedatasphere.exchangis.datasource.core.service.MetadataInfoService; +import com.webank.wedatasphere.exchangis.datasource.core.ui.ElementUI; +import com.webank.wedatasphere.exchangis.datasource.core.ui.builder.ElementUIFactory; +import com.webank.wedatasphere.exchangis.datasource.service.DataSourceRenderService; +import org.springframework.stereotype.Service; + +import javax.annotation.Resource; +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +/** + * Default render service + */ +@Service +public class DefaultDataSourceRenderService implements DataSourceRenderService { + + /** + * Default placeholder stored in system + */ + private final static String[] DEFAULT_PLACEHOLDER = + new String[]{"${timestamp}", "${yyyyMMdd}","${yyyy-MM-dd}", "${run_date}", "${run_date-1}", "${run_month_begin}", "${run_month_begin-1}"}; + + /** + * Metadata info service + */ + @Resource + private MetadataInfoService metadataInfoService; + + /** + * Element Factory + */ + @Resource + private ElementUIFactory elementUIFactory; + + @Override + public ElementUI getPartitionAndRender(String userName, + Long dataSourceId, String database, String table, ElementUI.Type uiType) throws ExchangisDataSourceException { + List partitionKeys = metadataInfoService.getPartitionKeys(userName, dataSourceId, database, table); + Map renderParams = new HashMap<>(); + List placeHolder = Arrays.asList(DEFAULT_PLACEHOLDER); + partitionKeys.forEach(partition -> renderParams.putIfAbsent(partition, placeHolder)); + return elementUIFactory.createElement(uiType.name(), renderParams, Map.class); + } +} diff --git a/exchangis-datasource/exchangis-datasource-service/src/main/java/com/webank/wedatasphere/exchangis/datasource/service/impl/DefaultMetadataInfoService.java b/exchangis-datasource/exchangis-datasource-service/src/main/java/com/webank/wedatasphere/exchangis/datasource/service/impl/DefaultMetadataInfoService.java deleted file mode 100644 index 1fb11b540..000000000 --- a/exchangis-datasource/exchangis-datasource-service/src/main/java/com/webank/wedatasphere/exchangis/datasource/service/impl/DefaultMetadataInfoService.java +++ /dev/null @@ -1,5 +0,0 @@ -package com.webank.wedatasphere.exchangis.datasource.service.impl; - -public class DefaultMetadataInfoService { - -} diff --git a/exchangis-job/exchangis-job-builder/src/main/java/com/webank/wedatasphere/exchangis/job/builder/ExchangisJobBuilderContext.java b/exchangis-job/exchangis-job-builder/src/main/java/com/webank/wedatasphere/exchangis/job/builder/ExchangisJobBuilderContext.java index 81e969153..af221df16 100644 --- a/exchangis-job/exchangis-job-builder/src/main/java/com/webank/wedatasphere/exchangis/job/builder/ExchangisJobBuilderContext.java +++ b/exchangis-job/exchangis-job-builder/src/main/java/com/webank/wedatasphere/exchangis/job/builder/ExchangisJobBuilderContext.java @@ -1,6 +1,7 @@ package com.webank.wedatasphere.exchangis.job.builder; +import com.webank.wedatasphere.exchangis.datasource.core.service.MetadataInfoService; import com.webank.wedatasphere.exchangis.job.domain.ExchangisJobInfo; import com.webank.wedatasphere.exchangis.job.listener.JobLogListener; @@ -26,6 +27,8 @@ public class ExchangisJobBuilderContext { private Map> datasourceParams = new HashMap<>(); + private MetadataInfoService metadataInfoService; + public ExchangisJobBuilderContext() { } @@ -70,4 +73,11 @@ public boolean containsEnv(String name) { return this.env.containsKey(name); } + public MetadataInfoService getMetadataInfoService() { + return metadataInfoService; + } + + public void setMetadataInfoService(MetadataInfoService metadataInfoService) { + this.metadataInfoService = metadataInfoService; + } } diff --git a/exchangis-job/exchangis-job-common/src/main/java/com/webank/wedatasphere/exchangis/job/domain/ExchangisJobEntity.java b/exchangis-job/exchangis-job-common/src/main/java/com/webank/wedatasphere/exchangis/job/domain/ExchangisJobEntity.java index 036c3d156..67c84efb1 100644 --- a/exchangis-job/exchangis-job-common/src/main/java/com/webank/wedatasphere/exchangis/job/domain/ExchangisJobEntity.java +++ b/exchangis-job/exchangis-job-common/src/main/java/com/webank/wedatasphere/exchangis/job/domain/ExchangisJobEntity.java @@ -1,6 +1,5 @@ package com.webank.wedatasphere.exchangis.job.domain; -import java.util.Date; /** * diff --git a/exchangis-job/exchangis-job-common/src/main/java/com/webank/wedatasphere/exchangis/job/domain/ExchangisJobInfo.java b/exchangis-job/exchangis-job-common/src/main/java/com/webank/wedatasphere/exchangis/job/domain/ExchangisJobInfo.java index 6ea8b1343..af474cfe8 100644 --- a/exchangis-job/exchangis-job-common/src/main/java/com/webank/wedatasphere/exchangis/job/domain/ExchangisJobInfo.java +++ b/exchangis-job/exchangis-job-common/src/main/java/com/webank/wedatasphere/exchangis/job/domain/ExchangisJobInfo.java @@ -9,17 +9,17 @@ public class ExchangisJobInfo extends GenericExchangisJob { /** * Job content (JSON) */ - private String jobContent; + protected String jobContent; /** * Execute user */ - private String executeUser; + protected String executeUser; /** * Job params (JSON) */ - private String jobParams; + protected String jobParams; /** * Convert from view object diff --git a/exchangis-job/exchangis-job-launcher/src/main/java/com/webank/wedatasphere/exchangis/job/launcher/domain/LaunchableExchangisJob.java b/exchangis-job/exchangis-job-launcher/src/main/java/com/webank/wedatasphere/exchangis/job/launcher/domain/LaunchableExchangisJob.java index 7595dde86..78c918810 100644 --- a/exchangis-job/exchangis-job-launcher/src/main/java/com/webank/wedatasphere/exchangis/job/launcher/domain/LaunchableExchangisJob.java +++ b/exchangis-job/exchangis-job-launcher/src/main/java/com/webank/wedatasphere/exchangis/job/launcher/domain/LaunchableExchangisJob.java @@ -21,6 +21,11 @@ public class LaunchableExchangisJob extends GenericExchangisJob { */ private String jobExecutionId; + /** + * Execute user + */ + private String execUser; + private List launchableExchangisTasks = new ArrayList<>(); public ExchangisJobInfo getExchangisJobInfo() { @@ -46,4 +51,12 @@ public List getLaunchableExchangisTasks() { public void setLaunchableExchangisTasks(List launchableExchangisTasks) { this.launchableExchangisTasks = launchableExchangisTasks; } + + public String getExecUser() { + return execUser; + } + + public void setExecUser(String execUser) { + this.execUser = execUser; + } } diff --git a/exchangis-job/exchangis-job-launcher/src/main/java/com/webank/wedatasphere/exchangis/job/launcher/entity/LaunchedExchangisJobEntity.java b/exchangis-job/exchangis-job-launcher/src/main/java/com/webank/wedatasphere/exchangis/job/launcher/entity/LaunchedExchangisJobEntity.java index 8125652de..1aac7a1f0 100644 --- a/exchangis-job/exchangis-job-launcher/src/main/java/com/webank/wedatasphere/exchangis/job/launcher/entity/LaunchedExchangisJobEntity.java +++ b/exchangis-job/exchangis-job-launcher/src/main/java/com/webank/wedatasphere/exchangis/job/launcher/entity/LaunchedExchangisJobEntity.java @@ -49,8 +49,8 @@ public LaunchedExchangisJobEntity(LaunchableExchangisJob job){ this.engineType = job.getEngineType(); this.jobId = job.getId(); this.jobName = job.getName(); - // Use the create user of LaunchableExchangisJob as exec user - this.executeUser = job.getCreateUser(); + this.executeUser = job.getExecUser(); + this.createUser = job.getCreateUser(); this.createTime = job.getCreateTime(); this.lastUpdateTime = job.getLastUpdateTime(); this.jobExecutionId = job.getJobExecutionId(); diff --git a/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/ExchangisJobExecuteAutoConfiguration.java b/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/ExchangisJobExecuteAutoConfiguration.java index 7f8077920..4805e1c5a 100644 --- a/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/ExchangisJobExecuteAutoConfiguration.java +++ b/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/ExchangisJobExecuteAutoConfiguration.java @@ -1,5 +1,6 @@ package com.webank.wedatasphere.exchangis.job.server; +import com.webank.wedatasphere.exchangis.datasource.core.service.MetadataInfoService; import com.webank.wedatasphere.exchangis.job.builder.manager.ExchangisJobBuilderManager; import com.webank.wedatasphere.exchangis.job.launcher.ExchangisTaskLaunchManager; import com.webank.wedatasphere.exchangis.job.launcher.domain.LaunchableExchangisTask; @@ -63,8 +64,9 @@ public ExchangisJobBuilderManager jobBuilderManager(){ @Bean @ConditionalOnMissingBean(TaskGeneratorContext.class) - public TaskGeneratorContext taskGeneratorContext(JobLogListener jobLogListener){ - return new DefaultTaskGeneratorContext(jobLogListener); + public TaskGeneratorContext taskGeneratorContext(JobLogListener jobLogListener, + MetadataInfoService metadataInfoService){ + return new DefaultTaskGeneratorContext(jobLogListener, metadataInfoService); } /** diff --git a/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/builder/engine/SqoopExchangisEngineJobBuilder.java b/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/builder/engine/SqoopExchangisEngineJobBuilder.java index 47fd85258..786b7e4c8 100644 --- a/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/builder/engine/SqoopExchangisEngineJobBuilder.java +++ b/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/builder/engine/SqoopExchangisEngineJobBuilder.java @@ -68,6 +68,7 @@ public enum MODE_TYPE { IMPORT, EXPORT} * Whether hcatalog */ private static final JobParamDefine IS_USE_HCATALOG = JobParams.define("sqoop.use.hcatalog", (BiFunction)(k, job) ->{ +// getCurrentBuilderContext().getMetadataInfoService().getTableProps() return true; }); diff --git a/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/builder/transform/TransformExchangisJob.java b/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/builder/transform/TransformExchangisJob.java index 00220d9cf..fe9806975 100644 --- a/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/builder/transform/TransformExchangisJob.java +++ b/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/builder/transform/TransformExchangisJob.java @@ -82,7 +82,7 @@ private void convertContentToParams(ExchangisJobInfoContent content){ if(Objects.nonNull(content.getParams().getSources())) { JobParamSet paramSet = setIntoParams(REALM_JOB_CONTENT_SOURCE, () -> { List items = content.getParams().getSources(); - return items.stream().filter(item -> StringUtils.isNotBlank(item.getConfigKey())).collect + return items.stream().filter(item -> StringUtils.isNotBlank(item.getConfigKey()) && Objects.nonNull(item.getConfigValue())).collect (Collectors.toMap(ExchangisJobParamsContent.ExchangisJobParamsItem::getConfigKey, ExchangisJobParamsContent.ExchangisJobParamsItem::getConfigValue)); }); @@ -93,7 +93,7 @@ private void convertContentToParams(ExchangisJobInfoContent content){ if(Objects.nonNull(content.getParams().getSinks())) { JobParamSet paramSet = setIntoParams(REALM_JOB_CONTENT_SINK, () -> { List items = content.getParams().getSinks(); - return items.stream().filter(item -> StringUtils.isNotBlank(item.getConfigKey())).collect + return items.stream().filter(item -> StringUtils.isNotBlank(item.getConfigKey()) && Objects.nonNull(item.getConfigValue())).collect (Collectors.toMap(ExchangisJobParamsContent.ExchangisJobParamsItem::getConfigKey, ExchangisJobParamsContent.ExchangisJobParamsItem::getConfigValue)); }); diff --git a/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/execution/AbstractTaskManager.java b/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/execution/AbstractTaskManager.java index af74c710a..332ac0de7 100644 --- a/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/execution/AbstractTaskManager.java +++ b/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/execution/AbstractTaskManager.java @@ -58,7 +58,8 @@ public void cancelRunningTask(String taskId) { LaunchedExchangisTask task = runningTasks.get(taskId); if (Objects.nonNull(task)){ onEvent(new TaskStatusUpdateEvent(task, TaskStatus.Cancelled)); - info(task, "Status of task: [{}] change {} => {}", task.getTaskId(), task.getStatus(), TaskStatus.Cancelled); + info(task, "Status of task: [name: {}, id: {}] change {} => {}", + task.getName(), task.getTaskId(), task.getStatus(), TaskStatus.Cancelled); JobLogCacheUtils.flush(task.getJobExecutionId(), false); runningTasks.remove(taskId); JobWrapper wrapper = jobWrappers.get(task.getJobExecutionId()); @@ -73,7 +74,7 @@ public void addRunningTask(LaunchedExchangisTask task) { task.setStatus(TaskStatus.Running); task.setRunningTime(Calendar.getInstance().getTime()); onEvent(new TaskLaunchEvent(task)); - info(task, "Status of task: [{}] change to {}, info: [{}]", task.getTaskId(), task.getStatus(), Json.toJson(task, null)); + info(task, "Status of task: [name: {}, id: {}] change to {}, info: [{}]", task.getName(), task.getTaskId(), task.getStatus(), Json.toJson(task, null)); if (Objects.isNull(runningTasks.putIfAbsent(task.getTaskId(), task))){ jobWrappers.compute(task.getJobExecutionId(), (jobExecutionId, jobWrapper) -> { if (Objects.nonNull(jobWrapper) && jobWrapper.addTask(task)){ @@ -109,7 +110,8 @@ public boolean refreshRunningTaskMetrics(LaunchedExchangisTask task, Map {}", task.getTaskId(), beforeStatus, status); + info(task, "Status of task: [name: {}, id: {}] change {} => {}", + task.getName(), task.getTaskId(), beforeStatus, status); onEvent(new TaskStatusUpdateEvent(task, status)); removeRunningTaskInner(task.getTaskId(), false); return true; @@ -118,7 +120,8 @@ public boolean refreshRunningTaskStatus(LaunchedExchangisTask task, TaskStatus s if (Objects.nonNull(task) ) { onEvent(new TaskStatusUpdateEvent(task, status)); if (isTransition(task, status)) { - info(task, "Status of task: [{}] change {} => {}", task.getTaskId(), beforeStatus, status); + info(task, "Status of task: [name: {}, id: {}] change {} => {}", + task.getName(), task.getTaskId(), beforeStatus, status); } task.setStatus(status); return true; diff --git a/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/execution/generator/AbstractTaskGenerator.java b/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/execution/generator/AbstractTaskGenerator.java index 66de68999..f6be48332 100644 --- a/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/execution/generator/AbstractTaskGenerator.java +++ b/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/execution/generator/AbstractTaskGenerator.java @@ -44,7 +44,8 @@ public LaunchableExchangisJob init(ExchangisJobInfo jobInfo) throws ExchangisTas launchableExchangisJob.setCreateTime(calendar.getTime()); launchableExchangisJob.setLastUpdateTime(calendar.getTime()); launchableExchangisJob.setId(jobInfo.getId()); - launchableExchangisJob.setCreateUser(jobInfo.getExecuteUser()); + launchableExchangisJob.setExecUser(jobInfo.getExecuteUser()); + launchableExchangisJob.setCreateUser(jobInfo.getCreateUser()); // Generate launchable exchangis job id to UUID launchableExchangisJob.setJobExecutionId(UUID.randomUUID().toString()); LOG.info("Generate job execution id: [{}] for job: [{}]" , launchableExchangisJob.getJobExecutionId(), launchableExchangisJob.getExchangisJobInfo().getName()); diff --git a/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/execution/generator/DefaultTaskGenerator.java b/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/execution/generator/DefaultTaskGenerator.java index 870a8e6e0..7dcfc6e54 100644 --- a/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/execution/generator/DefaultTaskGenerator.java +++ b/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/execution/generator/DefaultTaskGenerator.java @@ -75,6 +75,8 @@ protected void execute(LaunchableExchangisJob launchableExchangisJob, } ExchangisJobBuilderManager jobBuilderManager = getExchangisJobBuilderManager(); ExchangisJobBuilderContext ctx = new ExchangisJobBuilderContext(jobInfo, generatorContext.getJobLogListener()); + // Set the metadata service + ctx.setMetadataInfoService(generatorContext.getMetadataInfoService()); ctx.putEnv("USER_NAME", tenancy); // ExchangisJobInfo -> TransformExchangisJob(SubExchangisJob) try { diff --git a/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/execution/generator/DefaultTaskGeneratorContext.java b/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/execution/generator/DefaultTaskGeneratorContext.java index 138f16ad5..e43a51dc7 100644 --- a/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/execution/generator/DefaultTaskGeneratorContext.java +++ b/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/execution/generator/DefaultTaskGeneratorContext.java @@ -1,5 +1,6 @@ package com.webank.wedatasphere.exchangis.job.server.execution.generator; +import com.webank.wedatasphere.exchangis.datasource.core.service.MetadataInfoService; import com.webank.wedatasphere.exchangis.job.listener.JobLogListener; /** @@ -9,15 +10,24 @@ public class DefaultTaskGeneratorContext implements TaskGeneratorContext { private JobLogListener jobLogListener; + private MetadataInfoService metadataInfoService; + public DefaultTaskGeneratorContext(){ } - public DefaultTaskGeneratorContext(JobLogListener jobLogListener){ + public DefaultTaskGeneratorContext(JobLogListener jobLogListener, + MetadataInfoService metadataInfoService){ this.jobLogListener = jobLogListener; + this.metadataInfoService = metadataInfoService; } @Override public JobLogListener getJobLogListener() { return this.jobLogListener; } + + @Override + public MetadataInfoService getMetadataInfoService() { + return metadataInfoService; + } } diff --git a/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/execution/generator/TaskGeneratorContext.java b/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/execution/generator/TaskGeneratorContext.java index 2831ab7a4..d2a8ee6dc 100644 --- a/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/execution/generator/TaskGeneratorContext.java +++ b/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/execution/generator/TaskGeneratorContext.java @@ -1,5 +1,6 @@ package com.webank.wedatasphere.exchangis.job.server.execution.generator; +import com.webank.wedatasphere.exchangis.datasource.core.service.MetadataInfoService; import com.webank.wedatasphere.exchangis.job.listener.JobLogListener; /** @@ -12,4 +13,10 @@ public interface TaskGeneratorContext { * @return */ JobLogListener getJobLogListener(); + + /** + * Metadata info service + * @return + */ + MetadataInfoService getMetadataInfoService(); } diff --git a/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/mapper/impl/LaunchedJobMapper.xml b/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/mapper/impl/LaunchedJobMapper.xml index a89970eff..17db5dbd9 100644 --- a/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/mapper/impl/LaunchedJobMapper.xml +++ b/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/mapper/impl/LaunchedJobMapper.xml @@ -38,7 +38,7 @@ insert into - (name, create_time, last_update_time, job_id, launchable_task_num, job_execution_id, engine_type, execute_user, job_name, status, progress, error_code, error_msg, retry_num, log_path) + (name, create_time, last_update_time, job_id, launchable_task_num, job_execution_id, engine_type, execute_user, create_user, job_name, status, progress, error_code, error_msg, retry_num, log_path) values( #{name}, #{createTime}, @@ -48,6 +48,7 @@ #{jobExecutionId}, #{engineType}, #{executeUser}, + #{createUser}, #{jobName}, #{status}, #{progress}, diff --git a/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/service/TaskExecuteService.java b/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/service/TaskExecuteService.java index ef72aa90f..7ac422a75 100644 --- a/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/service/TaskExecuteService.java +++ b/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/service/TaskExecuteService.java @@ -17,7 +17,7 @@ public interface TaskExecuteService extends TaskExecutionListener { * @param task task * @param status status */ - void updateTaskStatus(LaunchedExchangisTask task, TaskStatus status) throws ExchangisOnEventException; + void updateTaskStatus(LaunchedExchangisTask task, TaskStatus status, boolean updateJob) throws ExchangisOnEventException; void updateTaskProgress(LaunchedExchangisTask task, float progress) throws ExchangisOnEventException; diff --git a/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/service/impl/DefaultJobExecuteService.java b/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/service/impl/DefaultJobExecuteService.java index 8cee1bf13..3da3e30f5 100644 --- a/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/service/impl/DefaultJobExecuteService.java +++ b/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/service/impl/DefaultJobExecuteService.java @@ -143,14 +143,12 @@ public ExchangisJobProgressVo getJobStatus(String jobExecutionId) throws Exchang boolean allTaskStatus = false; assert jobProgressVo != null; - if(!"Inited".equals(jobProgressVo.getStatus().toString()) || !jobProgressVo.getStatus().toString().equals("Scheduled") || !jobProgressVo.getStatus().toString().equals("Running") || !jobProgressVo.getStatus().toString().equals("WaitForRetry")) { + if(TaskStatus.isCompleted(jobProgressVo.getStatus())) { List taskStatusList = launchedTaskDao.getTaskStatusList(jobExecutionId); - if (taskStatusList.isEmpty() && this.launchedTaskDao.selectTaskListByJobExecutionId(jobExecutionId).isEmpty()) { - allTaskStatus = true; - } else if (!taskStatusList.isEmpty()) { - if (!taskStatusList.contains("Inited") || !taskStatusList.contains("Scheduled") || !taskStatusList.contains("Running") || !taskStatusList.contains("WaitForRetry")) { - allTaskStatus = true; - } + allTaskStatus = taskStatusList.isEmpty(); + if (!allTaskStatus){ + allTaskStatus = taskStatusList.stream().allMatch( status -> + StringUtils.isNotBlank(status) && TaskStatus.isCompleted(TaskStatus.valueOf(status))); } } jobProgressVo.setAllTaskStatus(allTaskStatus); diff --git a/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/service/impl/DefaultTaskExecuteService.java b/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/service/impl/DefaultTaskExecuteService.java index 27f63dd9d..491e3191c 100644 --- a/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/service/impl/DefaultTaskExecuteService.java +++ b/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/service/impl/DefaultTaskExecuteService.java @@ -48,8 +48,9 @@ public void onMetricsUpdate(TaskMetricsUpdateEvent metricsUpdateEvent) { public void onStatusUpdate(TaskStatusUpdateEvent statusUpdateEvent) throws ExchangisOnEventException { LaunchedExchangisTask task = statusUpdateEvent.getLaunchedExchangisTask(); TaskStatus status = statusUpdateEvent.getUpdateStatus(); + LaunchedExchangisJobEntity launchedJob = null; if (!TaskStatus.isCompleted(status)){ - LaunchedExchangisJobEntity launchedJob = launchedJobDao.searchLaunchedJob(task.getJobExecutionId()); + launchedJob = launchedJobDao.searchLaunchedJob(task.getJobExecutionId()); TaskStatus jobStatus = launchedJob.getStatus(); if (TaskStatus.isCompleted(jobStatus) && Objects.nonNull(task.getLauncherTask())){ // Kill the remote task @@ -64,7 +65,9 @@ public void onStatusUpdate(TaskStatusUpdateEvent statusUpdateEvent) throws Excha } // Have different status, then update if (!task.getStatus().equals(status)){ - getSelfService().updateTaskStatus(task, status); + launchedJob = Objects.isNull(launchedJob) ? + launchedJobDao.searchLaunchedJob(task.getJobExecutionId()) : launchedJob; + getSelfService().updateTaskStatus(task, status, !TaskStatus.isCompleted(launchedJob.getStatus())); } } @@ -105,15 +108,17 @@ public void onProgressUpdate(TaskProgressUpdateEvent updateEvent) throws Exchang */ @Override @Transactional(rollbackFor = Exception.class) - public void updateTaskStatus(LaunchedExchangisTask task, TaskStatus status) throws ExchangisOnEventException { + public void updateTaskStatus(LaunchedExchangisTask task, TaskStatus status, boolean updateJob) throws ExchangisOnEventException { JobLogCacheUtils.flush(task.getJobExecutionId(), false); task.setLastUpdateTime(Calendar.getInstance().getTime()); launchedTaskDao.upgradeLaunchedTaskStatus(task.getTaskId(), status.name(), task.getLastUpdateTime()); - if (status == TaskStatus.Failed || status == TaskStatus.Cancelled){ - // Update directly, no open another transaction - launchedJobDao.upgradeLaunchedJobStatus(task.getJobExecutionId(), status.name(), task.getLastUpdateTime()); - } else if (status == TaskStatus.Success){ - getSelfService().updateJobStatus(task.getJobExecutionId(), TaskStatus.Success, task.getLastUpdateTime()); + if (updateJob) { + if (status == TaskStatus.Failed || status == TaskStatus.Cancelled) { + // Update directly, no open another transaction + launchedJobDao.upgradeLaunchedJobStatus(task.getJobExecutionId(), status.name(), task.getLastUpdateTime()); + } else if (status == TaskStatus.Success) { + getSelfService().updateJobStatus(task.getJobExecutionId(), TaskStatus.Success, task.getLastUpdateTime()); + } } } diff --git a/exchangis-plugins/exchangis-appconn/src/main/java/com/webank/wedatasphere/exchangis/appconn/ExchangisStructureIntegrationStandard.java b/exchangis-plugins/exchangis-appconn/src/main/java/com/webank/wedatasphere/exchangis/appconn/ExchangisStructureIntegrationStandard.java deleted file mode 100644 index 7ab10a1b3..000000000 --- a/exchangis-plugins/exchangis-appconn/src/main/java/com/webank/wedatasphere/exchangis/appconn/ExchangisStructureIntegrationStandard.java +++ /dev/null @@ -1,13 +0,0 @@ -package com.webank.wedatasphere.exchangis.appconn; - -import com.webank.wedatasphere.dss.standard.app.structure.AbstractStructureIntegrationStandard; -import com.webank.wedatasphere.dss.standard.app.structure.project.ProjectService; -import com.webank.wedatasphere.exchangis.appconn.service.ExchangisProjectService; - -public class ExchangisStructureIntegrationStandard extends AbstractStructureIntegrationStandard { - @Override - protected ProjectService createProjectService() { - - return new ExchangisProjectService(); - } -} diff --git a/exchangis-plugins/exchangis-appconn/src/main/java/com/webank/wedatasphere/exchangis/appconn/config/ExchangisConfig.java b/exchangis-plugins/exchangis-appconn/src/main/java/com/webank/wedatasphere/exchangis/appconn/config/ExchangisConfig.java deleted file mode 100644 index d34faf9ae..000000000 --- a/exchangis-plugins/exchangis-appconn/src/main/java/com/webank/wedatasphere/exchangis/appconn/config/ExchangisConfig.java +++ /dev/null @@ -1,50 +0,0 @@ -package com.webank.wedatasphere.exchangis.appconn.config; - - -import org.apache.linkis.common.conf.CommonVars; -import org.apache.linkis.server.conf.ServerConfiguration; - -public class ExchangisConfig { - - public final static String BASEURL = "/api/rest_j/" + ServerConfiguration.BDP_SERVER_VERSION() + "/exchangis"; - public final static String SQOOP_JUMP_URL_FORMAT="dss/exchangis/#/childJobManagement"; - public final static String DATAX_JUMP_URL_FORMAT="dss/exchangis/#/childJobManagement"; - - public final static String EXCHANGIS_APPCONN_NAME = CommonVars.apply("wds.dss.appconn.exchangis.name", "Exchangis").getValue(); - public final static String NODE_TYPE_SQOOP = CommonVars.apply("linkis.appconn.exchangis.sqoop", "linkis.appconn.exchangis.sqoop").getValue(); - public final static String NODE_TYPE_DATAX = CommonVars.apply("linkis.appconn.exchangis.datax", "linkis.appconn.exchangis.datax").getValue(); - public final static String ENGINE_TYPE_DATAX_NAME = CommonVars.apply("wds.dss.appconn.exchangis.datax.name", "DATAX").getValue(); - public final static String ENGINE_TYPE_SQOOP_NAME = CommonVars.apply("wds.dss.appconn.exchangis.datax.name", "SQOOP").getValue(); - public final static String JOB_TYPE_OFFLINE = CommonVars.apply("wds.dss.appconn.exchangis.job.type.offline", "OFFLINE").getValue(); - public final static String JOB_TYPE_STREAM = CommonVars.apply("wds.dss.appconn.exchangis.job.type.stream", "STREAM").getValue(); - - public final static String ID = "id"; - public final static String WORKSPACE_NAME = "workspaceName"; - public final static String PROJECT_NAME = "projectName"; - public final static String DSS_PROJECT_ID="dssProjectId"; - public final static String DSS_PROJECT_NAME="dssProjectName"; - public final static String DESCRIPTION = "description"; - public final static String TAGS = "tags"; - public final static String EDIT_USERS = "editUsers"; - public final static String VIEW_USERS = "viewUsers"; - public final static String EXEC_USERS = "execUsers"; - - public final static String NODE_ID="nodeId"; - public final static String NODE_NAME="nodeName"; - public final static String PROJECT_ID = "projectId"; - public final static String ENGINE_TYPE = "engineType"; - public final static String JOB_DESC = "jobDesc"; - public final static String JOB_LABELS = "jobLabels"; - public final static String JOB_NAME = "jobName"; - public final static String JOB_TYPE="jobType"; - - - public static String getUrl(String baseUrl, String format, String entityId){ - return baseUrl + String.format(format, entityId); - } - - public static String getUrl(String baseUrl, String format, String... ids){ - return baseUrl + String.format(format, ids); - } -} - diff --git a/exchangis-plugins/exchangis-appconn/src/main/java/com/webank/wedatasphere/exchangis/appconn/model/ExchangisJobContentDTO.java b/exchangis-plugins/exchangis-appconn/src/main/java/com/webank/wedatasphere/exchangis/appconn/model/ExchangisJobContentDTO.java deleted file mode 100644 index 2fe4e27d2..000000000 --- a/exchangis-plugins/exchangis-appconn/src/main/java/com/webank/wedatasphere/exchangis/appconn/model/ExchangisJobContentDTO.java +++ /dev/null @@ -1,55 +0,0 @@ -package com.webank.wedatasphere.exchangis.appconn.model; - -public class ExchangisJobContentDTO { - - private String content; - - private String proxyUser; - - private String executeNode; - - private String syncType; - - private String jobParams; - - public String getContent() { - return content; - } - - public void setContent(String content) { - this.content = content; - } - - public String getProxyUser() { - return proxyUser; - } - - public void setProxyUser(String proxyUser) { - this.proxyUser = proxyUser; - } - - public String getExecuteNode() { - return executeNode; - } - - public void setExecuteNode(String executeNode) { - this.executeNode = executeNode; - } - - public String getSyncType() { - return syncType; - } - - public void setSyncType(String syncType) { - this.syncType = syncType; - } - - public String getJobParams() { - return jobParams; - } - - public void setJobParams(String jobParams) { - this.jobParams = jobParams; - } - -} diff --git a/exchangis-plugins/exchangis-appconn/src/main/java/com/webank/wedatasphere/exchangis/appconn/operation/ExchangisCreationOperation.java b/exchangis-plugins/exchangis-appconn/src/main/java/com/webank/wedatasphere/exchangis/appconn/operation/ExchangisCreationOperation.java deleted file mode 100644 index 31dd1a21a..000000000 --- a/exchangis-plugins/exchangis-appconn/src/main/java/com/webank/wedatasphere/exchangis/appconn/operation/ExchangisCreationOperation.java +++ /dev/null @@ -1,132 +0,0 @@ -package com.webank.wedatasphere.exchangis.appconn.operation; - -import com.webank.wedatasphere.dss.standard.app.development.operation.RefCreationOperation; -import com.webank.wedatasphere.dss.standard.app.development.ref.CreateRequestRef; -import com.webank.wedatasphere.dss.standard.app.development.ref.NodeRequestRef; -import com.webank.wedatasphere.dss.standard.app.development.service.DevelopmentService; -import com.webank.wedatasphere.dss.standard.app.sso.builder.SSOUrlBuilderOperation; -import com.webank.wedatasphere.dss.standard.app.sso.request.SSORequestOperation; -import com.webank.wedatasphere.dss.standard.common.entity.ref.ResponseRef; -import com.webank.wedatasphere.dss.standard.common.exception.operation.ExternalOperationFailedException; -import com.webank.wedatasphere.exchangis.appconn.config.ExchangisConfig; -import com.webank.wedatasphere.exchangis.appconn.model.ExchangisGetAction; -import com.webank.wedatasphere.exchangis.appconn.model.ExchangisPostAction; -import com.webank.wedatasphere.exchangis.appconn.ref.ExchangisCommonResponseRef; -import com.webank.wedatasphere.exchangis.appconn.utils.AppConnUtils; -import org.apache.linkis.httpclient.response.HttpResult; -import org.apache.linkis.server.BDPJettyServerHelper; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.util.Map; - - -public class ExchangisCreationOperation implements RefCreationOperation { - private final static Logger logger = LoggerFactory.getLogger(ExchangisCreationOperation.class); - - DevelopmentService developmentService; - private SSORequestOperation ssoRequestOperation; - public ExchangisCreationOperation(DevelopmentService developmentService){ - this.developmentService = developmentService; - this.ssoRequestOperation = this.developmentService.getSSORequestService().createSSORequestOperation(ExchangisConfig.EXCHANGIS_APPCONN_NAME); - } - - @Override - public ResponseRef createRef(CreateRequestRef createRequestRef) throws ExternalOperationFailedException { - NodeRequestRef exchangisCreateRequestRef = (NodeRequestRef) createRequestRef; - exchangisCreateRequestRef.getProjectName(); - ResponseRef responseRef = null; - logger.info("create job=>projectId:{},jobName:{},nodeType:{}",exchangisCreateRequestRef.getProjectId(),exchangisCreateRequestRef.getName(),exchangisCreateRequestRef.getNodeType()); - if(ExchangisConfig.NODE_TYPE_SQOOP.equalsIgnoreCase(exchangisCreateRequestRef.getNodeType())){ - responseRef = sendOffLineRequest(exchangisCreateRequestRef, ExchangisConfig.ENGINE_TYPE_SQOOP_NAME); - }else if(ExchangisConfig.NODE_TYPE_DATAX.equalsIgnoreCase(exchangisCreateRequestRef.getNodeType())){ - responseRef = sendOffLineRequest(exchangisCreateRequestRef,ExchangisConfig.ENGINE_TYPE_DATAX_NAME); - } - return responseRef; - } - - private ResponseRef sendOffLineRequest(NodeRequestRef requestRef,String engineType) throws ExternalOperationFailedException { - String url = getBaseUrl()+"/job"; - logger.info("create sqoop job=>jobContent:{} || projectId:{} || projectName:{} || parameters:{} || type:{}",requestRef.getJobContent(),requestRef.getProjectId(),requestRef.getProjectName(),requestRef.getParameters().toString(),requestRef.getType()); - ExchangisPostAction exchangisPostAction = new ExchangisPostAction(); - String projectName = null; - try { - String contextID = requestRef.getJobContent().get("contextID").toString(); - Map contextIDMap = BDPJettyServerHelper.jacksonJson().readValue(contextID, Map.class); - String valueJson = contextIDMap.get("value").toString(); - Map map = BDPJettyServerHelper.jacksonJson().readValue(valueJson, Map.class); - logger.info("map {}",map.toString()); - projectName = map.get("project").toString(); - }catch (Exception e){ - throw new ExternalOperationFailedException(31023, "Get node Id failed!", e); - } - String projectId = this.queryProject(requestRef, projectName); - - exchangisPostAction.setUser(requestRef.getUserName()); - exchangisPostAction.addRequestPayload(ExchangisConfig.PROJECT_ID,projectId); - exchangisPostAction.addRequestPayload(ExchangisConfig.DSS_PROJECT_ID,requestRef.getJobContent().get("projectId").toString()); - exchangisPostAction.addRequestPayload(ExchangisConfig.DSS_PROJECT_NAME,projectName); - exchangisPostAction.addRequestPayload(ExchangisConfig.NODE_ID,requestRef.getJobContent().get("nodeId").toString()); - exchangisPostAction.addRequestPayload(ExchangisConfig.NODE_NAME,requestRef.getName()); - exchangisPostAction.addRequestPayload(ExchangisConfig.JOB_DESC,requestRef.getJobContent().get("desc").toString()); - exchangisPostAction.addRequestPayload(ExchangisConfig.JOB_LABELS, AppConnUtils.changeDssLabelName(requestRef.getDSSLabels())); - exchangisPostAction.addRequestPayload(ExchangisConfig.JOB_NAME,requestRef.getName()); - exchangisPostAction.addRequestPayload(ExchangisConfig.JOB_TYPE,ExchangisConfig.JOB_TYPE_OFFLINE); - exchangisPostAction.addRequestPayload(ExchangisConfig.ENGINE_TYPE,engineType); - - SSOUrlBuilderOperation ssoUrlBuilderOperation = requestRef.getWorkspace().getSSOUrlBuilderOperation().copy(); - ssoUrlBuilderOperation.setAppName(ExchangisConfig.EXCHANGIS_APPCONN_NAME); - ssoUrlBuilderOperation.setReqUrl(url); - ssoUrlBuilderOperation.setWorkspace(requestRef.getWorkspace().getWorkspaceName()); - ExchangisCommonResponseRef responseRef; - try{ - exchangisPostAction.setUrl(ssoUrlBuilderOperation.getBuiltUrl()); - HttpResult httpResult = (HttpResult) this.ssoRequestOperation.requestWithSSO(ssoUrlBuilderOperation, exchangisPostAction); - logger.info("sendSqoop => body:{}",httpResult.getResponseBody()); - responseRef = new ExchangisCommonResponseRef(httpResult.getResponseBody()); - } catch (Exception e){ - throw new ExternalOperationFailedException(31020, "Create sqoop job Exception", e); - } - - return responseRef; - } - - public String queryProject(NodeRequestRef requestRef,String projectName) throws ExternalOperationFailedException{ - String url = getBaseUrl()+"/projects/dss/"+projectName; - ExchangisGetAction exchangisGetAction = new ExchangisGetAction(); - exchangisGetAction.setUser(requestRef.getUserName()); - - SSOUrlBuilderOperation ssoUrlBuilderOperation = requestRef.getWorkspace().getSSOUrlBuilderOperation().copy(); - ssoUrlBuilderOperation.setAppName(ExchangisConfig.EXCHANGIS_APPCONN_NAME); - ssoUrlBuilderOperation.setReqUrl(url); - ssoUrlBuilderOperation.setWorkspace(requestRef.getWorkspace().getWorkspaceName()); - String projectId =""; - try{ - exchangisGetAction.setUrl(ssoUrlBuilderOperation.getBuiltUrl()); - HttpResult httpResult = (HttpResult) this.ssoRequestOperation.requestWithSSO(ssoUrlBuilderOperation, exchangisGetAction); - logger.info("queryProject => body:{} || statusCode:{}",httpResult.getResponseBody(),httpResult.getStatusCode()); - if(httpResult.getStatusCode() == 200){ - Map responseMap = BDPJettyServerHelper.jacksonJson().readValue(httpResult.getResponseBody(), Map.class); - Map item = (Map) ((Map) responseMap.get("data")).get("item"); - projectId = item.get("id").toString(); - }else{ - throw new ExternalOperationFailedException(31024, "queryProject id null"); - } - logger.info("queryProject => projectId:{}",projectId); - } catch (Exception e){ - throw new ExternalOperationFailedException(31020, "queryProject job Exception", e); - } - - - return projectId; - - } - - private String getBaseUrl(){ - return developmentService.getAppInstance().getBaseUrl() + ExchangisConfig.BASEURL; - } - @Override - public void setDevelopmentService(DevelopmentService developmentService) { - this.developmentService = developmentService; - } -} diff --git a/exchangis-plugins/exchangis-appconn/src/main/java/com/webank/wedatasphere/exchangis/appconn/operation/ExchangisDeletionOperation.java b/exchangis-plugins/exchangis-appconn/src/main/java/com/webank/wedatasphere/exchangis/appconn/operation/ExchangisDeletionOperation.java deleted file mode 100644 index b7c3aab4f..000000000 --- a/exchangis-plugins/exchangis-appconn/src/main/java/com/webank/wedatasphere/exchangis/appconn/operation/ExchangisDeletionOperation.java +++ /dev/null @@ -1,76 +0,0 @@ -package com.webank.wedatasphere.exchangis.appconn.operation; - -import com.google.common.collect.Maps; -import com.webank.wedatasphere.dss.standard.app.development.operation.RefDeletionOperation; -import com.webank.wedatasphere.dss.standard.app.development.ref.NodeRequestRef; -import com.webank.wedatasphere.dss.standard.app.development.service.DevelopmentService; -import com.webank.wedatasphere.dss.standard.app.sso.builder.SSOUrlBuilderOperation; -import com.webank.wedatasphere.dss.standard.app.sso.request.SSORequestOperation; -import com.webank.wedatasphere.dss.standard.common.entity.ref.RequestRef; -import com.webank.wedatasphere.dss.standard.common.exception.operation.ExternalOperationFailedException; -import com.webank.wedatasphere.exchangis.appconn.config.ExchangisConfig; -import com.webank.wedatasphere.exchangis.appconn.model.ExchangisDeleteAction; -import org.apache.linkis.httpclient.response.HttpResult; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.util.Map; - -public class ExchangisDeletionOperation implements RefDeletionOperation { - private final static Logger logger = LoggerFactory.getLogger(ExchangisDeletionOperation.class); - - DevelopmentService developmentService; - private SSORequestOperation ssoRequestOperation; - public ExchangisDeletionOperation(DevelopmentService developmentService){ - this.developmentService = developmentService; - this.ssoRequestOperation = this.developmentService.getSSORequestService().createSSORequestOperation(ExchangisConfig.EXCHANGIS_APPCONN_NAME); - } - - @Override - public void deleteRef(RequestRef deleteRequestRef) throws ExternalOperationFailedException { - NodeRequestRef nodeRequestRef = (NodeRequestRef) deleteRequestRef; - logger.info("delete job=>jobContext:{}", nodeRequestRef.getJobContent().toString()); - deleteJob(nodeRequestRef); - } - - private void deleteJob(NodeRequestRef deleteRequestRef) throws ExternalOperationFailedException{ - String url=""; - try { - String jobId = ((Map)((Map) deleteRequestRef.getJobContent().get("data")).get("result")).get("id").toString(); - url = getBaseUrl()+"/"+jobId; - }catch (Exception e){ - throw new ExternalOperationFailedException(31023, "Get job Id failed!", e); - } - - ExchangisDeleteAction exchangisDeleteAction = new ExchangisDeleteAction(); - exchangisDeleteAction.setUser(deleteRequestRef.getUserName()); - SSOUrlBuilderOperation ssoUrlBuilderOperation = deleteRequestRef.getWorkspace().getSSOUrlBuilderOperation().copy(); - ssoUrlBuilderOperation.setAppName(ExchangisConfig.EXCHANGIS_APPCONN_NAME); - ssoUrlBuilderOperation.setReqUrl(url); - logger.info("delete job=>url:{}", url); - ssoUrlBuilderOperation.setWorkspace(deleteRequestRef.getWorkspace().getWorkspaceName()); - String response = ""; - Map resMap = Maps.newHashMap(); - HttpResult httpResult = null; - try { - exchangisDeleteAction.setUrl(ssoUrlBuilderOperation.getBuiltUrl()); - httpResult = (HttpResult) this.ssoRequestOperation.requestWithSSO(ssoUrlBuilderOperation, exchangisDeleteAction); - response = httpResult.getResponseBody(); - if(httpResult.getStatusCode() != 200){ - throw new ExternalOperationFailedException(31024, "Delete Job id error"); - } - logger.info("delete => body:{}",response); - } catch (Exception e) { - throw new ExternalOperationFailedException(31023, "Delete Job Exception", e); - } - } - - @Override - public void setDevelopmentService(DevelopmentService developmentService) { - this.developmentService = developmentService; - } - - private String getBaseUrl(){ - return developmentService.getAppInstance().getBaseUrl() + ExchangisConfig.BASEURL; - } -} diff --git a/exchangis-plugins/exchangis-appconn/src/main/java/com/webank/wedatasphere/exchangis/appconn/operation/ExchangisExecutionOperation.java b/exchangis-plugins/exchangis-appconn/src/main/java/com/webank/wedatasphere/exchangis/appconn/operation/ExchangisExecutionOperation.java deleted file mode 100644 index d96399438..000000000 --- a/exchangis-plugins/exchangis-appconn/src/main/java/com/webank/wedatasphere/exchangis/appconn/operation/ExchangisExecutionOperation.java +++ /dev/null @@ -1,69 +0,0 @@ -package com.webank.wedatasphere.exchangis.appconn.operation; - -import com.webank.wedatasphere.dss.standard.app.development.listener.common.AsyncExecutionRequestRef; -import com.webank.wedatasphere.dss.standard.app.development.operation.RefExecutionOperation; -import com.webank.wedatasphere.dss.standard.app.development.ref.ExecutionRequestRef; -import com.webank.wedatasphere.dss.standard.app.development.service.DevelopmentService; -import com.webank.wedatasphere.dss.standard.app.sso.builder.SSOUrlBuilderOperation; -import com.webank.wedatasphere.dss.standard.app.sso.request.SSORequestOperation; -import com.webank.wedatasphere.dss.standard.common.entity.ref.ResponseRef; -import com.webank.wedatasphere.dss.standard.common.exception.operation.ExternalOperationFailedException; -import com.webank.wedatasphere.exchangis.appconn.config.ExchangisConfig; -import com.webank.wedatasphere.exchangis.appconn.model.ExchangisPostAction; -import com.webank.wedatasphere.exchangis.appconn.ref.ExchangisCommonResponseRef; -import org.apache.linkis.httpclient.request.HttpAction; -import org.apache.linkis.httpclient.response.HttpResult; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.util.Map; - -public class ExchangisExecutionOperation implements RefExecutionOperation { - - private final static Logger logger = LoggerFactory.getLogger(ExchangisExecutionOperation.class); - DevelopmentService developmentService; - private SSORequestOperation ssoRequestOperation; - - public ExchangisExecutionOperation(DevelopmentService service) { - this.developmentService = service; - this.ssoRequestOperation = this.developmentService.getSSORequestService().createSSORequestOperation(ExchangisConfig.EXCHANGIS_APPCONN_NAME); - } - @Override - public ResponseRef execute(ExecutionRequestRef executionRequestRef) throws ExternalOperationFailedException { - AsyncExecutionRequestRef nodeRequestRef = (AsyncExecutionRequestRef) executionRequestRef; - logger.info("execution name:{} || jobContext:{} || RuntimeMap:{} || user:{}",nodeRequestRef.getName(),nodeRequestRef.getJobContent().toString(),nodeRequestRef.getExecutionRequestRefContext().getRuntimeMap().toString(),nodeRequestRef.getExecutionRequestRefContext().getUser()); - String url=""; - try { - String jobId = ((Map)((Map) nodeRequestRef.getJobContent().get("data")).get("result")).get("id").toString(); - url +="/"+jobId+"/action/execute"; - }catch (Exception e){ - throw new ExternalOperationFailedException(31023, "Get job Id failed!", e); - } - - ExchangisPostAction exchangisPostAction = new ExchangisPostAction(); - exchangisPostAction.setUser(getUser(nodeRequestRef)); - SSOUrlBuilderOperation ssoUrlBuilderOperation = nodeRequestRef.getWorkspace().getSSOUrlBuilderOperation().copy(); - ssoUrlBuilderOperation.setAppName(ExchangisConfig.EXCHANGIS_APPCONN_NAME); - ssoUrlBuilderOperation.setReqUrl(url); - ssoUrlBuilderOperation.setWorkspace(nodeRequestRef.getWorkspace().getWorkspaceName()); - ResponseRef responseRef; - try{ - exchangisPostAction.setUrl(ssoUrlBuilderOperation.getBuiltUrl()); - HttpResult httpResult = (HttpResult) this.ssoRequestOperation.requestWithSSO(ssoUrlBuilderOperation, exchangisPostAction); - responseRef = new ExchangisCommonResponseRef(httpResult.getResponseBody()); - logger.info("execute job body:{}",responseRef.getResponseBody()); - } catch (Exception e){ - throw new ExternalOperationFailedException(31025, "import exchangis exception", e); - } - - return responseRef; - } - private String getUser(AsyncExecutionRequestRef requestRef) { - return requestRef.getExecutionRequestRefContext().getRuntimeMap().get("wds.dss.workflow.submit.user").toString(); - } - - @Override - public void setDevelopmentService(DevelopmentService developmentService) { - this.developmentService = developmentService; - } -} diff --git a/exchangis-plugins/exchangis-appconn/src/main/java/com/webank/wedatasphere/exchangis/appconn/operation/ExchangisExportOperation.java b/exchangis-plugins/exchangis-appconn/src/main/java/com/webank/wedatasphere/exchangis/appconn/operation/ExchangisExportOperation.java deleted file mode 100644 index ccaf431e6..000000000 --- a/exchangis-plugins/exchangis-appconn/src/main/java/com/webank/wedatasphere/exchangis/appconn/operation/ExchangisExportOperation.java +++ /dev/null @@ -1,61 +0,0 @@ -package com.webank.wedatasphere.exchangis.appconn.operation; - -import com.webank.wedatasphere.dss.standard.app.development.operation.RefExportOperation; -import com.webank.wedatasphere.dss.standard.app.development.ref.ExportRequestRef; -import com.webank.wedatasphere.dss.standard.app.development.service.DevelopmentService; -import com.webank.wedatasphere.dss.standard.app.sso.builder.SSOUrlBuilderOperation; -import com.webank.wedatasphere.dss.standard.app.sso.request.SSORequestOperation; -import com.webank.wedatasphere.dss.standard.common.entity.ref.ResponseRef; -import com.webank.wedatasphere.dss.standard.common.exception.operation.ExternalOperationFailedException; -import com.webank.wedatasphere.exchangis.appconn.config.ExchangisConfig; -import com.webank.wedatasphere.exchangis.appconn.model.ExchangisGetAction; -import com.webank.wedatasphere.exchangis.appconn.ref.ExchangisCommonResponseRef; -import org.apache.linkis.httpclient.response.HttpResult; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -public class ExchangisExportOperation implements RefExportOperation { - private final static Logger logger = LoggerFactory.getLogger(ExchangisExportOperation.class); - - private DevelopmentService developmentService; - private SSORequestOperation ssoRequestOperation; - - public ExchangisExportOperation(DevelopmentService developmentService){ - this.developmentService = developmentService; - this.ssoRequestOperation = this.developmentService.getSSORequestService().createSSORequestOperation(ExchangisConfig.EXCHANGIS_APPCONN_NAME); - } - - - @Override - public ResponseRef exportRef(ExportRequestRef exportRequestRef) throws ExternalOperationFailedException { - String url = getBaseUrl() + "/export"; - logger.info("importRef job=>parameter:{}",exportRequestRef.getParameters().toString()); - - ExchangisGetAction exchangisGetAction = new ExchangisGetAction(); - exchangisGetAction.setUser(exportRequestRef.getParameter("user").toString()); - exchangisGetAction.setParameter("projectId",exportRequestRef.getParameter("projectId")); - - SSOUrlBuilderOperation ssoUrlBuilderOperation = exportRequestRef.getWorkspace().getSSOUrlBuilderOperation().copy(); - ssoUrlBuilderOperation.setAppName(ExchangisConfig.EXCHANGIS_APPCONN_NAME); - ssoUrlBuilderOperation.setReqUrl(url); - ssoUrlBuilderOperation.setWorkspace(exportRequestRef.getWorkspace().getWorkspaceName()); - ResponseRef responseRef; - try{ - exchangisGetAction.setUrl(ssoUrlBuilderOperation.getBuiltUrl()); - HttpResult httpResult = (HttpResult) this.ssoRequestOperation.requestWithSSO(ssoUrlBuilderOperation, exchangisGetAction); - responseRef = new ExchangisCommonResponseRef(httpResult.getResponseBody()); - } catch (Exception e){ - throw new ExternalOperationFailedException(31022, "export job Exception", e); - } - return responseRef; - } - - @Override - public void setDevelopmentService(DevelopmentService developmentService) { - this.developmentService = developmentService; - } - - private String getBaseUrl(){ - return developmentService.getAppInstance().getBaseUrl() + ExchangisConfig.BASEURL; - } -} diff --git a/exchangis-plugins/exchangis-appconn/src/main/java/com/webank/wedatasphere/exchangis/appconn/operation/ExchangisImportOperation.java b/exchangis-plugins/exchangis-appconn/src/main/java/com/webank/wedatasphere/exchangis/appconn/operation/ExchangisImportOperation.java deleted file mode 100644 index 864843e83..000000000 --- a/exchangis-plugins/exchangis-appconn/src/main/java/com/webank/wedatasphere/exchangis/appconn/operation/ExchangisImportOperation.java +++ /dev/null @@ -1,66 +0,0 @@ -package com.webank.wedatasphere.exchangis.appconn.operation; - -import com.webank.wedatasphere.dss.standard.app.development.operation.RefImportOperation; -import com.webank.wedatasphere.dss.standard.app.development.ref.ImportRequestRef; -import com.webank.wedatasphere.dss.standard.app.development.service.DevelopmentService; -import com.webank.wedatasphere.dss.standard.app.sso.builder.SSOUrlBuilderOperation; -import com.webank.wedatasphere.dss.standard.app.sso.request.SSORequestOperation; -import com.webank.wedatasphere.dss.standard.common.entity.ref.ResponseRef; -import com.webank.wedatasphere.dss.standard.common.exception.operation.ExternalOperationFailedException; -import com.webank.wedatasphere.exchangis.appconn.config.ExchangisConfig; -import com.webank.wedatasphere.exchangis.appconn.model.ExchangisPostAction; -import com.webank.wedatasphere.exchangis.appconn.ref.ExchangisCommonResponseRef; -import org.apache.linkis.httpclient.response.HttpResult; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - - -public class ExchangisImportOperation implements RefImportOperation { - private final static Logger logger = LoggerFactory.getLogger(ExchangisImportOperation.class); - - private DevelopmentService developmentService; - private SSORequestOperation ssoRequestOperation; - - public ExchangisImportOperation(DevelopmentService developmentService){ - this.developmentService = developmentService; - this.ssoRequestOperation = this.developmentService.getSSORequestService().createSSORequestOperation(ExchangisConfig.EXCHANGIS_APPCONN_NAME); - } - - @Override - public ResponseRef importRef(ImportRequestRef importRequestRef) throws ExternalOperationFailedException { - String url = getBaseUrl() + "/relation"; - logger.info("importRef job=>parameter:{} ||name:{}",importRequestRef.getParameters().toString(),importRequestRef.getName()); - - ExchangisPostAction exchangisPostAction = new ExchangisPostAction(); - exchangisPostAction.setUser(importRequestRef.getParameter("user").toString()); - - exchangisPostAction.addRequestPayload("projectId", importRequestRef.getParameter("projectId")); - exchangisPostAction.addRequestPayload("projectVersion", "v1"); - exchangisPostAction.addRequestPayload("flowVersion", importRequestRef.getParameter("orcVersion")); - exchangisPostAction.addRequestPayload("resourceId", importRequestRef.getParameter("resourceId")); - exchangisPostAction.addRequestPayload("version", importRequestRef.getParameter("version")); - SSOUrlBuilderOperation ssoUrlBuilderOperation = importRequestRef.getWorkspace().getSSOUrlBuilderOperation().copy(); - ssoUrlBuilderOperation.setAppName(ExchangisConfig.EXCHANGIS_APPCONN_NAME); - ssoUrlBuilderOperation.setReqUrl(url); - ssoUrlBuilderOperation.setWorkspace(importRequestRef.getWorkspace().getWorkspaceName()); - ResponseRef responseRef; - try{ - exchangisPostAction.setUrl(ssoUrlBuilderOperation.getBuiltUrl()); - HttpResult httpResult = (HttpResult) this.ssoRequestOperation.requestWithSSO(ssoUrlBuilderOperation, exchangisPostAction); - responseRef = new ExchangisCommonResponseRef(httpResult.getResponseBody()); - logger.info("import job body:{}",responseRef.getResponseBody()); - } catch (Exception e){ - throw new ExternalOperationFailedException(31025, "import exchangis exception", e); - } - return responseRef; - } - - @Override - public void setDevelopmentService(DevelopmentService developmentService) { - this.developmentService = developmentService; - } - - private String getBaseUrl(){ - return developmentService.getAppInstance().getBaseUrl() + ExchangisConfig.BASEURL; - } -} diff --git a/exchangis-plugins/exchangis-appconn/src/main/java/com/webank/wedatasphere/exchangis/appconn/operation/ExchangisQueryOperation.java b/exchangis-plugins/exchangis-appconn/src/main/java/com/webank/wedatasphere/exchangis/appconn/operation/ExchangisQueryOperation.java deleted file mode 100644 index 916735c9a..000000000 --- a/exchangis-plugins/exchangis-appconn/src/main/java/com/webank/wedatasphere/exchangis/appconn/operation/ExchangisQueryOperation.java +++ /dev/null @@ -1,59 +0,0 @@ -package com.webank.wedatasphere.exchangis.appconn.operation; - -import com.webank.wedatasphere.dss.common.utils.DSSCommonUtils; -import com.webank.wedatasphere.dss.standard.app.development.operation.RefQueryOperation; -import com.webank.wedatasphere.dss.standard.app.development.ref.OpenRequestRef; -import com.webank.wedatasphere.dss.standard.app.development.service.DevelopmentService; -import com.webank.wedatasphere.dss.standard.app.sso.request.SSORequestOperation; -import com.webank.wedatasphere.dss.standard.common.entity.ref.ResponseRef; -import com.webank.wedatasphere.dss.standard.common.exception.operation.ExternalOperationFailedException; -import com.webank.wedatasphere.exchangis.appconn.config.ExchangisConfig; -import com.webank.wedatasphere.exchangis.appconn.ref.ExchangisOpenRequestRef; -import com.webank.wedatasphere.exchangis.appconn.ref.ExchangisOpenResponseRef; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.lang.reflect.InvocationHandler; -import java.lang.reflect.Method; -import java.lang.reflect.Proxy; -import java.util.HashMap; -import java.util.Map; - -public class ExchangisQueryOperation implements RefQueryOperation { - private final static Logger logger = LoggerFactory.getLogger(ExchangisQueryOperation.class); - - private DevelopmentService developmentService; - private SSORequestOperation ssoRequestOperation; - - public ExchangisQueryOperation(DevelopmentService developmentService){ - this.developmentService = developmentService; - this.ssoRequestOperation = this.developmentService.getSSORequestService().createSSORequestOperation(ExchangisConfig.EXCHANGIS_APPCONN_NAME); - } - @Override - public ResponseRef query(OpenRequestRef openRequestRef) throws ExternalOperationFailedException { - ExchangisOpenRequestRef exchangisOpenRequestRef = (ExchangisOpenRequestRef) openRequestRef; - logger.info("query job=>jobContent:{} ||,projectId:{} ||,projectName:{} ||,parameters:{} ||,type:{}",exchangisOpenRequestRef.getJobContent(),exchangisOpenRequestRef.getProjectId(),exchangisOpenRequestRef.getProjectName(),exchangisOpenRequestRef.getParameters().toString(),exchangisOpenRequestRef.getType()); - try { - String jobId = ((Map)((Map)exchangisOpenRequestRef.getJobContent().get("data")).get("result")).get("id").toString(); - String baseUrl = exchangisOpenRequestRef.getParameter("redirectUrl").toString() + "/"; - String jumpUrl = baseUrl; - if(ExchangisConfig.NODE_TYPE_SQOOP.equalsIgnoreCase(exchangisOpenRequestRef.getType())){ - jumpUrl +=ExchangisConfig.SQOOP_JUMP_URL_FORMAT; - }else if(ExchangisConfig.NODE_TYPE_DATAX.equalsIgnoreCase(exchangisOpenRequestRef.getType())){ - jumpUrl += ExchangisConfig.DATAX_JUMP_URL_FORMAT; - } - jumpUrl +="?id="+jobId; - Map retMap = new HashMap<>(); - logger.info("exchangisCommonResponseRef jumpUrl {}",jumpUrl); - retMap.put("jumpUrl",jumpUrl); - return new ExchangisOpenResponseRef(DSSCommonUtils.COMMON_GSON.toJson(retMap),0); - } catch (Exception e) { - throw new ExternalOperationFailedException(31022, "Failed to parse jobContent ", e); - } - } - - @Override - public void setDevelopmentService(DevelopmentService developmentService) { - this.developmentService = developmentService; - } -} diff --git a/exchangis-plugins/exchangis-appconn/src/main/java/com/webank/wedatasphere/exchangis/appconn/operation/ExchangisUpdateOperation.java b/exchangis-plugins/exchangis-appconn/src/main/java/com/webank/wedatasphere/exchangis/appconn/operation/ExchangisUpdateOperation.java deleted file mode 100644 index 849027ed3..000000000 --- a/exchangis-plugins/exchangis-appconn/src/main/java/com/webank/wedatasphere/exchangis/appconn/operation/ExchangisUpdateOperation.java +++ /dev/null @@ -1,83 +0,0 @@ -package com.webank.wedatasphere.exchangis.appconn.operation; - -import com.webank.wedatasphere.dss.standard.app.development.operation.RefUpdateOperation; -import com.webank.wedatasphere.dss.standard.app.development.ref.NodeRequestRef; -import com.webank.wedatasphere.dss.standard.app.development.ref.UpdateRequestRef; -import com.webank.wedatasphere.dss.standard.app.development.service.DevelopmentService; -import com.webank.wedatasphere.dss.standard.app.sso.builder.SSOUrlBuilderOperation; -import com.webank.wedatasphere.dss.standard.app.sso.request.SSORequestOperation; -import com.webank.wedatasphere.dss.standard.common.entity.ref.ResponseRef; -import com.webank.wedatasphere.dss.standard.common.exception.operation.ExternalOperationFailedException; -import com.webank.wedatasphere.exchangis.appconn.config.ExchangisConfig; -import com.webank.wedatasphere.exchangis.appconn.model.ExchangisPutAction; -import com.webank.wedatasphere.exchangis.appconn.ref.ExchangisCommonResponseRef; -import com.webank.wedatasphere.exchangis.appconn.utils.AppConnUtils; -import org.apache.linkis.httpclient.response.HttpResult; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -public class ExchangisUpdateOperation implements RefUpdateOperation { - private final static Logger logger = LoggerFactory.getLogger(ExchangisUpdateOperation.class); - - DevelopmentService developmentService; - private SSORequestOperation ssoRequestOperation; - public ExchangisUpdateOperation(DevelopmentService developmentService){ - this.developmentService = developmentService; - this.ssoRequestOperation = this.developmentService.getSSORequestService().createSSORequestOperation(ExchangisConfig.EXCHANGIS_APPCONN_NAME); - } - - @Override - public ResponseRef updateRef(UpdateRequestRef updateRequestRef) throws ExternalOperationFailedException { - NodeRequestRef exchangisupdateRequestRef = (NodeRequestRef) updateRequestRef; - logger.info("update job=>projectId:{},jobName:{},nodeType:{}",exchangisupdateRequestRef.getProjectId(),exchangisupdateRequestRef.getName(),exchangisupdateRequestRef.getNodeType()); - ResponseRef responseRef = null; - if(ExchangisConfig.NODE_TYPE_SQOOP.equalsIgnoreCase(exchangisupdateRequestRef.getNodeType())){ - responseRef = updateOffLineRequest(exchangisupdateRequestRef, ExchangisConfig.ENGINE_TYPE_SQOOP_NAME); - }else if(ExchangisConfig.NODE_TYPE_DATAX.equalsIgnoreCase(exchangisupdateRequestRef.getNodeType())){ - responseRef = updateOffLineRequest(exchangisupdateRequestRef,ExchangisConfig.NODE_TYPE_DATAX); - } - return responseRef; - } - - private ResponseRef updateOffLineRequest(NodeRequestRef exchangisupdateRequestRef,String engineType) throws ExternalOperationFailedException{ - logger.info("update sqoop job=>jobContent:{}",exchangisupdateRequestRef.getJobContent()); - ExchangisPutAction exchangisPutAction = new ExchangisPutAction(); - - String nodeId = exchangisupdateRequestRef.getJobContent().get("nodeId").toString(); - String url=getBaseUrl()+"/dss/"+nodeId; - - exchangisPutAction.setUser(exchangisupdateRequestRef.getUserName()); - - exchangisPutAction.setUser(exchangisupdateRequestRef.getUserName()); - exchangisPutAction.addRequestPayload(ExchangisConfig.NODE_NAME,exchangisupdateRequestRef.getName()); - exchangisPutAction.addRequestPayload(ExchangisConfig.JOB_DESC,exchangisupdateRequestRef.getJobContent().get("desc").toString()); - exchangisPutAction.addRequestPayload(ExchangisConfig.JOB_LABELS, AppConnUtils.changeDssLabelName(exchangisupdateRequestRef.getDSSLabels())); - exchangisPutAction.addRequestPayload(ExchangisConfig.JOB_NAME,exchangisupdateRequestRef.getName()); - exchangisPutAction.addRequestPayload(ExchangisConfig.JOB_TYPE,ExchangisConfig.JOB_TYPE_OFFLINE); - exchangisPutAction.addRequestPayload(ExchangisConfig.ENGINE_TYPE,engineType); - - SSOUrlBuilderOperation ssoUrlBuilderOperation = exchangisupdateRequestRef.getWorkspace().getSSOUrlBuilderOperation().copy(); - ssoUrlBuilderOperation.setAppName(ExchangisConfig.EXCHANGIS_APPCONN_NAME); - ssoUrlBuilderOperation.setReqUrl(url); - ssoUrlBuilderOperation.setWorkspace(exchangisupdateRequestRef.getWorkspace().getWorkspaceName()); - ExchangisCommonResponseRef responseRef; - - try{ - exchangisPutAction.setUrl(ssoUrlBuilderOperation.getBuiltUrl()); - HttpResult httpResult = (HttpResult) this.ssoRequestOperation.requestWithSSO(ssoUrlBuilderOperation, exchangisPutAction); - responseRef = new ExchangisCommonResponseRef(httpResult.getResponseBody()); - } catch (Exception e){ - throw new ExternalOperationFailedException(31020, "update sqoop job Exception", e); - } - return responseRef; - } - - - private String getBaseUrl(){ - return developmentService.getAppInstance().getBaseUrl() + ExchangisConfig.BASEURL; - } - @Override - public void setDevelopmentService(DevelopmentService developmentService) { - this.developmentService = developmentService; - } -} diff --git a/exchangis-plugins/exchangis-appconn/src/main/java/com/webank/wedatasphere/exchangis/appconn/project/ExchangisProjectCreationOperation.java b/exchangis-plugins/exchangis-appconn/src/main/java/com/webank/wedatasphere/exchangis/appconn/project/ExchangisProjectCreationOperation.java deleted file mode 100644 index ef2e1191d..000000000 --- a/exchangis-plugins/exchangis-appconn/src/main/java/com/webank/wedatasphere/exchangis/appconn/project/ExchangisProjectCreationOperation.java +++ /dev/null @@ -1,108 +0,0 @@ -package com.webank.wedatasphere.exchangis.appconn.project; - -import com.webank.wedatasphere.dss.standard.app.sso.builder.SSOUrlBuilderOperation; -import com.fasterxml.jackson.core.type.TypeReference; -import com.google.common.collect.Maps; -import com.webank.wedatasphere.dss.standard.app.sso.request.SSORequestOperation; -import com.webank.wedatasphere.dss.standard.app.structure.StructureService; -import com.webank.wedatasphere.dss.standard.app.structure.project.ProjectCreationOperation; -import com.webank.wedatasphere.dss.standard.app.structure.project.ProjectRequestRef; -import com.webank.wedatasphere.dss.standard.app.structure.project.ProjectResponseRef; -import com.webank.wedatasphere.dss.standard.common.exception.operation.ExternalOperationFailedException; - -import com.webank.wedatasphere.exchangis.appconn.config.ExchangisConfig; -import com.webank.wedatasphere.exchangis.appconn.model.ExchangisPostAction; -import com.webank.wedatasphere.exchangis.appconn.ref.ExchangisProjectResponseRef; -import com.webank.wedatasphere.exchangis.appconn.utils.AppConnUtils; -import org.apache.linkis.httpclient.request.HttpAction; -import org.apache.linkis.httpclient.response.HttpResult; -import org.apache.linkis.server.BDPJettyServerHelper; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.util.Map; - -public class ExchangisProjectCreationOperation implements ProjectCreationOperation { - private static Logger logger = LoggerFactory.getLogger(ExchangisProjectCreationOperation.class); - - private SSORequestOperation ssoRequestOperation; - private StructureService structureService; - - public ExchangisProjectCreationOperation(SSORequestOperation ssoRequestOperation, StructureService structureService) { - this.ssoRequestOperation = ssoRequestOperation; - this.structureService = structureService; - } - - private String getAppName() { - return ExchangisConfig.EXCHANGIS_APPCONN_NAME; - } - @Override - public ProjectResponseRef createProject(ProjectRequestRef projectRequestRef) throws ExternalOperationFailedException { - String url = getBaseUrl() +"/createProject"; - - logger.info("create project=>projectId:{},name:{},createName:{},parameters:{},workspaceName:{}",projectRequestRef.getId(),projectRequestRef.getName(),projectRequestRef.getCreateBy(),projectRequestRef.getParameters().toString(),projectRequestRef.getWorkspace().getWorkspaceName()); - - ExchangisPostAction exchangisPostAction = new ExchangisPostAction(); - exchangisPostAction.setUser(projectRequestRef.getCreateBy()); - exchangisPostAction.addRequestPayload(ExchangisConfig.WORKSPACE_NAME,projectRequestRef.getWorkspace().getWorkspaceName()); - exchangisPostAction.addRequestPayload(ExchangisConfig.PROJECT_NAME,projectRequestRef.getName()); - exchangisPostAction.addRequestPayload(ExchangisConfig.DSS_PROJECT_NAME,projectRequestRef.getName()); - exchangisPostAction.addRequestPayload(ExchangisConfig.DESCRIPTION,projectRequestRef.getDescription()); - exchangisPostAction.addRequestPayload(ExchangisConfig.EDIT_USERS,projectRequestRef.getCreateBy()); - exchangisPostAction.addRequestPayload(ExchangisConfig.EXEC_USERS,projectRequestRef.getCreateBy()); - exchangisPostAction.addRequestPayload(ExchangisConfig.VIEW_USERS,projectRequestRef.getCreateBy()); - exchangisPostAction.addRequestPayload(ExchangisConfig.TAGS, AppConnUtils.changeDssLabelName(projectRequestRef.getDSSLabels())); - - SSOUrlBuilderOperation ssoUrlBuilderOperation = projectRequestRef.getWorkspace().getSSOUrlBuilderOperation().copy(); - ssoUrlBuilderOperation.setAppName(getAppName()); - ssoUrlBuilderOperation.setReqUrl(url); - ssoUrlBuilderOperation.setWorkspace(projectRequestRef.getWorkspace().getWorkspaceName()); - - String response = ""; - Map resMap = Maps.newHashMap(); - HttpResult httpResult = null; - try { - exchangisPostAction.setUrl(ssoUrlBuilderOperation.getBuiltUrl()); - httpResult = this.ssoRequestOperation.requestWithSSO(ssoUrlBuilderOperation, exchangisPostAction); - response = httpResult.getResponseBody(); - resMap = BDPJettyServerHelper.jacksonJson().readValue(response, new TypeReference>() {}); - }catch (Exception e){ - logger.error("Create Exchangis Project Exception", e); - throw new ExternalOperationFailedException(31020,e.getMessage()); - } - - logger.info("create project=> status {},response {},resMap {}",httpResult.getStatusCode(),response,resMap.toString()); - Map header = (Map) resMap.get("header"); - int code = (int) header.get("code"); - String errorMsg = ""; - if (code != 200) { - errorMsg = header.toString(); - throw new ExternalOperationFailedException(31020, errorMsg, null); - } - Integer projectId = (Integer) ((Map) resMap.get("payload")).get("projectId"); - ExchangisProjectResponseRef exchangisProjectResponseRef = null; - try { - exchangisProjectResponseRef = new ExchangisProjectResponseRef(response, code); - } catch (Exception e) { - throw new ExternalOperationFailedException(31020, "failed to parse response json", e); - } - exchangisProjectResponseRef.setAppInstance(structureService.getAppInstance()); - exchangisProjectResponseRef.setProjectRefId(projectId.longValue()); - exchangisProjectResponseRef.setErrorMsg(errorMsg); - return exchangisProjectResponseRef; - } - - @Override - public void init() { - - } - - private String getBaseUrl(){ - return structureService.getAppInstance().getBaseUrl() + ExchangisConfig.BASEURL; - } - - @Override - public void setStructureService(StructureService structureService) { - this.structureService=structureService; - } -} diff --git a/exchangis-plugins/exchangis-appconn/src/main/java/com/webank/wedatasphere/exchangis/appconn/project/ExchangisProjectDeletionOperation.java b/exchangis-plugins/exchangis-appconn/src/main/java/com/webank/wedatasphere/exchangis/appconn/project/ExchangisProjectDeletionOperation.java deleted file mode 100644 index 8d5525333..000000000 --- a/exchangis-plugins/exchangis-appconn/src/main/java/com/webank/wedatasphere/exchangis/appconn/project/ExchangisProjectDeletionOperation.java +++ /dev/null @@ -1,96 +0,0 @@ -package com.webank.wedatasphere.exchangis.appconn.project; - -import com.fasterxml.jackson.core.type.TypeReference; -import com.google.common.collect.Maps; -import com.webank.wedatasphere.dss.standard.app.sso.builder.SSOUrlBuilderOperation; -import com.webank.wedatasphere.dss.standard.app.sso.request.SSORequestOperation; -import com.webank.wedatasphere.dss.standard.app.structure.StructureService; -import com.webank.wedatasphere.dss.standard.app.structure.project.ProjectDeletionOperation; -import com.webank.wedatasphere.dss.standard.app.structure.project.ProjectRequestRef; -import com.webank.wedatasphere.dss.standard.app.structure.project.ProjectResponseRef; -import com.webank.wedatasphere.dss.standard.common.exception.operation.ExternalOperationFailedException; -import com.webank.wedatasphere.exchangis.appconn.config.ExchangisConfig; -import com.webank.wedatasphere.exchangis.appconn.model.ExchangisDeleteAction; -import com.webank.wedatasphere.exchangis.appconn.ref.ExchangisProjectResponseRef; -import org.apache.linkis.httpclient.request.HttpAction; -import org.apache.linkis.httpclient.response.HttpResult; -import org.apache.linkis.server.BDPJettyServerHelper; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.util.Map; - - -public class ExchangisProjectDeletionOperation implements ProjectDeletionOperation { - private static Logger logger = LoggerFactory.getLogger(ExchangisProjectDeletionOperation.class); - - private SSORequestOperation ssoRequestOperation; - private StructureService structureService; - - public ExchangisProjectDeletionOperation(SSORequestOperation ssoRequestOperation, StructureService structureService) { - this.ssoRequestOperation = ssoRequestOperation; - this.structureService = structureService; - } - - private String getAppName() { - return ExchangisConfig.EXCHANGIS_APPCONN_NAME; - } - @Override - public ProjectResponseRef deleteProject(ProjectRequestRef projectRequestRef) throws ExternalOperationFailedException { - Long projectId = projectRequestRef.getId(); - logger.info("delete project=>projectId:{},name:{},createName:{}",projectRequestRef.getId(),projectRequestRef.getName(),projectRequestRef.getCreateBy()); - - String url = getBaseUrl() +"/projects/dss/"+projectRequestRef.getName(); - ExchangisDeleteAction exchangisPostAction = new ExchangisDeleteAction(); - exchangisPostAction.setUser(projectRequestRef.getCreateBy()); - - SSOUrlBuilderOperation ssoUrlBuilderOperation = projectRequestRef.getWorkspace().getSSOUrlBuilderOperation().copy(); - ssoUrlBuilderOperation.setAppName(getAppName()); - ssoUrlBuilderOperation.setReqUrl(url); - ssoUrlBuilderOperation.setWorkspace(projectRequestRef.getWorkspace().getWorkspaceName()); - - String response = ""; - Map resMap = Maps.newHashMap(); - HttpResult httpResult = null; - try { - exchangisPostAction.setUrl(ssoUrlBuilderOperation.getBuiltUrl()); - httpResult = this.ssoRequestOperation.requestWithSSO(ssoUrlBuilderOperation, exchangisPostAction); - response = httpResult.getResponseBody(); - resMap = BDPJettyServerHelper.jacksonJson().readValue(response, new TypeReference>() {}); - }catch (Exception e){ - logger.error("delete Exchangis Project Exception", e); - throw new ExternalOperationFailedException(31020,e.getMessage()); - } - - Map header = (Map) resMap.get("header"); - int code = (int) header.get("code"); - String errorMsg = ""; - if (code != 200) { - errorMsg = header.toString(); - throw new ExternalOperationFailedException(31020, errorMsg, null); - } - ExchangisProjectResponseRef exchangisProjectResponseRef = null; - try { - exchangisProjectResponseRef = new ExchangisProjectResponseRef(response, code); - } catch (Exception e) { - throw new ExternalOperationFailedException(31020, "failed to parse response json", e); - } - exchangisProjectResponseRef.setAppInstance(structureService.getAppInstance()); - exchangisProjectResponseRef.setProjectRefId(projectId); - exchangisProjectResponseRef.setErrorMsg(errorMsg); - return exchangisProjectResponseRef; - } - - @Override - public void init() { - - } - - private String getBaseUrl(){ - return structureService.getAppInstance().getBaseUrl() + ExchangisConfig.BASEURL; - } - @Override - public void setStructureService(StructureService structureService) { - this.structureService=structureService; - } -} diff --git a/exchangis-plugins/exchangis-appconn/src/main/java/com/webank/wedatasphere/exchangis/appconn/project/ExchangisProjectUpdateOperation.java b/exchangis-plugins/exchangis-appconn/src/main/java/com/webank/wedatasphere/exchangis/appconn/project/ExchangisProjectUpdateOperation.java deleted file mode 100644 index 0cfbd41fe..000000000 --- a/exchangis-plugins/exchangis-appconn/src/main/java/com/webank/wedatasphere/exchangis/appconn/project/ExchangisProjectUpdateOperation.java +++ /dev/null @@ -1,103 +0,0 @@ -package com.webank.wedatasphere.exchangis.appconn.project; - -import com.fasterxml.jackson.core.type.TypeReference; -import com.google.common.collect.Maps; -import com.webank.wedatasphere.dss.standard.app.sso.builder.SSOUrlBuilderOperation; -import com.webank.wedatasphere.dss.standard.app.sso.request.SSORequestOperation; -import com.webank.wedatasphere.dss.standard.app.structure.StructureService; -import com.webank.wedatasphere.dss.standard.app.structure.project.ProjectRequestRef; -import com.webank.wedatasphere.dss.standard.app.structure.project.ProjectResponseRef; -import com.webank.wedatasphere.dss.standard.app.structure.project.ProjectUpdateOperation; -import com.webank.wedatasphere.dss.standard.common.exception.operation.ExternalOperationFailedException; -import com.webank.wedatasphere.exchangis.appconn.config.ExchangisConfig; -import com.webank.wedatasphere.exchangis.appconn.model.ExchangisPutAction; -import com.webank.wedatasphere.exchangis.appconn.ref.ExchangisProjectResponseRef; -import com.webank.wedatasphere.exchangis.appconn.utils.AppConnUtils; -import org.apache.linkis.httpclient.request.HttpAction; -import org.apache.linkis.httpclient.response.HttpResult; -import org.apache.linkis.server.BDPJettyServerHelper; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.util.Map; - -public class ExchangisProjectUpdateOperation implements ProjectUpdateOperation { - private static final Logger logger = LoggerFactory.getLogger(ExchangisProjectUpdateOperation.class); - private SSORequestOperation ssoRequestOperation; - private StructureService structureService; - - public ExchangisProjectUpdateOperation(SSORequestOperation ssoRequestOperation, StructureService structureService) { - this.ssoRequestOperation = ssoRequestOperation; - this.structureService = structureService; - } - - private String getAppName() { - return ExchangisConfig.EXCHANGIS_APPCONN_NAME; - } - @Override - public ProjectResponseRef updateProject(ProjectRequestRef projectRequestRef) throws ExternalOperationFailedException { - String url = getBaseUrl() +"/updateProject"; - logger.info("update project=>projectId:{},name:{},createName:{}",projectRequestRef.getId(),projectRequestRef.getName(),projectRequestRef.getCreateBy()); - - ExchangisPutAction exchangisPutAction = new ExchangisPutAction(); - exchangisPutAction.setUser(projectRequestRef.getCreateBy()); - exchangisPutAction.addRequestPayload(ExchangisConfig.WORKSPACE_NAME,projectRequestRef.getWorkspaceName()); - exchangisPutAction.addRequestPayload(ExchangisConfig.DSS_PROJECT_ID,projectRequestRef.getId().toString()); - exchangisPutAction.addRequestPayload(ExchangisConfig.DSS_PROJECT_NAME,projectRequestRef.getName()); - exchangisPutAction.addRequestPayload(ExchangisConfig.PROJECT_NAME,projectRequestRef.getName()); - exchangisPutAction.addRequestPayload(ExchangisConfig.DESCRIPTION,projectRequestRef.getDescription()); - exchangisPutAction.addRequestPayload(ExchangisConfig.EDIT_USERS,projectRequestRef.getUpdateBy()); - exchangisPutAction.addRequestPayload(ExchangisConfig.EXEC_USERS,projectRequestRef.getUpdateBy()); - exchangisPutAction.addRequestPayload(ExchangisConfig.VIEW_USERS,projectRequestRef.getUpdateBy()); - exchangisPutAction.addRequestPayload(ExchangisConfig.TAGS, AppConnUtils.changeDssLabelName(projectRequestRef.getDSSLabels())); - - SSOUrlBuilderOperation ssoUrlBuilderOperation = projectRequestRef.getWorkspace().getSSOUrlBuilderOperation().copy(); - ssoUrlBuilderOperation.setAppName(getAppName()); - ssoUrlBuilderOperation.setReqUrl(url); - ssoUrlBuilderOperation.setWorkspace(projectRequestRef.getWorkspace().getWorkspaceName()); - - String response = ""; - Map resMap = Maps.newHashMap(); - HttpResult httpResult = null; - try { - exchangisPutAction.setUrl(ssoUrlBuilderOperation.getBuiltUrl()); - httpResult = this.ssoRequestOperation.requestWithSSO(ssoUrlBuilderOperation, exchangisPutAction); - response = httpResult.getResponseBody(); - resMap = BDPJettyServerHelper.jacksonJson().readValue(response, new TypeReference>() {}); - }catch (Exception e){ - logger.error("Create Exchangis Project Exception", e); - throw new ExternalOperationFailedException(31020,e.getMessage()); - } - Map header = (Map) resMap.get("header"); - int code = (int) header.get("code"); - String errorMsg = ""; - if (code != 200) { - errorMsg = header.toString(); - throw new ExternalOperationFailedException(31020, errorMsg, null); - } - Integer projectId = (Integer) ((Map) resMap.get("payload")).get("projectId"); - ExchangisProjectResponseRef exchangisProjectResponseRef = null; - try { - exchangisProjectResponseRef = new ExchangisProjectResponseRef(response, code); - } catch (Exception e) { - throw new ExternalOperationFailedException(31020, "failed to parse response json", e); - } - exchangisProjectResponseRef.setAppInstance(structureService.getAppInstance()); - exchangisProjectResponseRef.setProjectRefId(projectId.longValue()); - exchangisProjectResponseRef.setErrorMsg(errorMsg); - return exchangisProjectResponseRef; - } - - @Override - public void init() { - - } - - private String getBaseUrl(){ - return structureService.getAppInstance().getBaseUrl() + ExchangisConfig.BASEURL; - } - @Override - public void setStructureService(StructureService structureService) { - this.structureService=structureService; - } -} diff --git a/exchangis-plugins/exchangis-appconn/src/main/java/com/webank/wedatasphere/exchangis/appconn/ref/ExchangisCommonResponseRef.java b/exchangis-plugins/exchangis-appconn/src/main/java/com/webank/wedatasphere/exchangis/appconn/ref/ExchangisCommonResponseRef.java deleted file mode 100644 index 459103d56..000000000 --- a/exchangis-plugins/exchangis-appconn/src/main/java/com/webank/wedatasphere/exchangis/appconn/ref/ExchangisCommonResponseRef.java +++ /dev/null @@ -1,10 +0,0 @@ -package com.webank.wedatasphere.exchangis.appconn.ref; - -import com.webank.wedatasphere.dss.standard.app.development.ref.DSSCommonResponseRef; - -public class ExchangisCommonResponseRef extends DSSCommonResponseRef { - - public ExchangisCommonResponseRef(String responseBody) throws Exception { - super(responseBody); - } -} diff --git a/exchangis-plugins/exchangis-appconn/src/main/java/com/webank/wedatasphere/exchangis/appconn/service/ExchangisCRUDService.java b/exchangis-plugins/exchangis-appconn/src/main/java/com/webank/wedatasphere/exchangis/appconn/service/ExchangisCRUDService.java deleted file mode 100644 index 0f66de433..000000000 --- a/exchangis-plugins/exchangis-appconn/src/main/java/com/webank/wedatasphere/exchangis/appconn/service/ExchangisCRUDService.java +++ /dev/null @@ -1,35 +0,0 @@ -package com.webank.wedatasphere.exchangis.appconn.service; - -import com.webank.wedatasphere.dss.standard.app.development.operation.RefCopyOperation; -import com.webank.wedatasphere.dss.standard.app.development.operation.RefCreationOperation; -import com.webank.wedatasphere.dss.standard.app.development.operation.RefDeletionOperation; -import com.webank.wedatasphere.dss.standard.app.development.operation.RefUpdateOperation; -import com.webank.wedatasphere.dss.standard.app.development.service.AbstractRefCRUDService; -import com.webank.wedatasphere.exchangis.appconn.operation.ExchangisCreationOperation; -import com.webank.wedatasphere.exchangis.appconn.operation.ExchangisDeletionOperation; -import com.webank.wedatasphere.exchangis.appconn.operation.ExchangisUpdateOperation; - -public class ExchangisCRUDService extends AbstractRefCRUDService { - - @Override - protected RefCreationOperation createRefCreationOperation() { - return new ExchangisCreationOperation(this); - } - - - @Override - protected RefCopyOperation createRefCopyOperation() { - return null; - } - - @Override - protected RefUpdateOperation createRefUpdateOperation() { - return new ExchangisUpdateOperation(this); - } - - @Override - protected RefDeletionOperation createRefDeletionOperation() { - return new ExchangisDeletionOperation(this); - - } -} diff --git a/exchangis-plugins/exchangis-appconn/src/main/java/com/webank/wedatasphere/exchangis/appconn/service/ExchangisExecutionService.java b/exchangis-plugins/exchangis-appconn/src/main/java/com/webank/wedatasphere/exchangis/appconn/service/ExchangisExecutionService.java deleted file mode 100644 index 668dd2799..000000000 --- a/exchangis-plugins/exchangis-appconn/src/main/java/com/webank/wedatasphere/exchangis/appconn/service/ExchangisExecutionService.java +++ /dev/null @@ -1,13 +0,0 @@ -package com.webank.wedatasphere.exchangis.appconn.service; - -import com.webank.wedatasphere.dss.standard.app.development.operation.RefExecutionOperation; -import com.webank.wedatasphere.dss.standard.app.development.service.AbstractRefExecutionService; -import com.webank.wedatasphere.exchangis.appconn.operation.ExchangisExecutionOperation; - -public class ExchangisExecutionService extends AbstractRefExecutionService { - @Override - public RefExecutionOperation createRefExecutionOperation() { - ExchangisExecutionOperation exchangisExecutionOperation = new ExchangisExecutionOperation(this); - return exchangisExecutionOperation; - } -} diff --git a/exchangis-plugins/exchangis-appconn/src/main/java/com/webank/wedatasphere/exchangis/appconn/service/ExchangisProjectService.java b/exchangis-plugins/exchangis-appconn/src/main/java/com/webank/wedatasphere/exchangis/appconn/service/ExchangisProjectService.java deleted file mode 100644 index 058edba41..000000000 --- a/exchangis-plugins/exchangis-appconn/src/main/java/com/webank/wedatasphere/exchangis/appconn/service/ExchangisProjectService.java +++ /dev/null @@ -1,54 +0,0 @@ -package com.webank.wedatasphere.exchangis.appconn.service; - -import com.webank.wedatasphere.dss.standard.app.sso.request.SSORequestOperation; -import com.webank.wedatasphere.dss.standard.app.structure.project.*; -import com.webank.wedatasphere.exchangis.appconn.config.ExchangisConfig; -import com.webank.wedatasphere.exchangis.appconn.project.ExchangisProjectCreationOperation; -import com.webank.wedatasphere.exchangis.appconn.project.ExchangisProjectDeletionOperation; -import com.webank.wedatasphere.exchangis.appconn.project.ExchangisProjectUpdateOperation; -import org.apache.linkis.httpclient.request.HttpAction; -import org.apache.linkis.httpclient.response.HttpResult; - -public class ExchangisProjectService extends ProjectService { - @Override - public boolean isCooperationSupported() { - return true; - } - - @Override - public boolean isProjectNameUnique() { - return false; - } - - @Override - protected ProjectCreationOperation createProjectCreationOperation() { - - SSORequestOperation ssoRequestOperation = getSSORequestService().createSSORequestOperation(ExchangisConfig.EXCHANGIS_APPCONN_NAME); - ExchangisProjectCreationOperation exchangisProjectCreationOperation = new ExchangisProjectCreationOperation(ssoRequestOperation,this); - exchangisProjectCreationOperation.setStructureService(this); - return exchangisProjectCreationOperation; - } - - @Override - protected ProjectUpdateOperation createProjectUpdateOperation() { - - SSORequestOperation ssoRequestOperation = getSSORequestService().createSSORequestOperation(ExchangisConfig.EXCHANGIS_APPCONN_NAME); - ExchangisProjectUpdateOperation exchangisProjectUpdateOperation = new ExchangisProjectUpdateOperation(ssoRequestOperation,this); - exchangisProjectUpdateOperation.setStructureService(this); - return exchangisProjectUpdateOperation; - } - - @Override - protected ProjectDeletionOperation createProjectDeletionOperation() { - - SSORequestOperation ssoRequestOperation = getSSORequestService().createSSORequestOperation(ExchangisConfig.EXCHANGIS_APPCONN_NAME); - ExchangisProjectDeletionOperation exchangisProjectDeletionOperation = new ExchangisProjectDeletionOperation(ssoRequestOperation,this); - exchangisProjectDeletionOperation.setStructureService(this); - return exchangisProjectDeletionOperation; - } - - @Override - protected ProjectUrlOperation createProjectUrlOperation() { - return null; - } -} diff --git a/exchangis-plugins/exchangis-appconn/src/main/java/com/webank/wedatasphere/exchangis/appconn/service/ExchangisQueryService.java b/exchangis-plugins/exchangis-appconn/src/main/java/com/webank/wedatasphere/exchangis/appconn/service/ExchangisQueryService.java deleted file mode 100644 index 1e8613a57..000000000 --- a/exchangis-plugins/exchangis-appconn/src/main/java/com/webank/wedatasphere/exchangis/appconn/service/ExchangisQueryService.java +++ /dev/null @@ -1,12 +0,0 @@ -package com.webank.wedatasphere.exchangis.appconn.service; - -import com.webank.wedatasphere.dss.standard.app.development.operation.RefQueryOperation; -import com.webank.wedatasphere.dss.standard.app.development.service.AbstractRefQueryService; -import com.webank.wedatasphere.exchangis.appconn.operation.ExchangisQueryOperation; - -public class ExchangisQueryService extends AbstractRefQueryService { - @Override - protected RefQueryOperation createRefQueryOperation() { - return new ExchangisQueryOperation(this); - } -} diff --git a/exchangis-plugins/exchangis-appconn/src/main/java/com/webank/wedatasphere/exchangis/appconn/service/ExchangisRefExportService.java b/exchangis-plugins/exchangis-appconn/src/main/java/com/webank/wedatasphere/exchangis/appconn/service/ExchangisRefExportService.java deleted file mode 100644 index e94eab6ee..000000000 --- a/exchangis-plugins/exchangis-appconn/src/main/java/com/webank/wedatasphere/exchangis/appconn/service/ExchangisRefExportService.java +++ /dev/null @@ -1,12 +0,0 @@ -package com.webank.wedatasphere.exchangis.appconn.service; - -import com.webank.wedatasphere.dss.standard.app.development.operation.RefExportOperation; -import com.webank.wedatasphere.dss.standard.app.development.service.AbstractRefExportService; -import com.webank.wedatasphere.exchangis.appconn.operation.ExchangisExportOperation; - -public class ExchangisRefExportService extends AbstractRefExportService { - @Override - protected RefExportOperation createRefExportOperation() { - return new ExchangisExportOperation(this); - } -} diff --git a/exchangis-plugins/exchangis-appconn/src/main/java/com/webank/wedatasphere/exchangis/appconn/service/ExchangisRefImportService.java b/exchangis-plugins/exchangis-appconn/src/main/java/com/webank/wedatasphere/exchangis/appconn/service/ExchangisRefImportService.java deleted file mode 100644 index 31d06c6f2..000000000 --- a/exchangis-plugins/exchangis-appconn/src/main/java/com/webank/wedatasphere/exchangis/appconn/service/ExchangisRefImportService.java +++ /dev/null @@ -1,14 +0,0 @@ -package com.webank.wedatasphere.exchangis.appconn.service; - - -import com.webank.wedatasphere.dss.standard.app.development.operation.RefImportOperation; -import com.webank.wedatasphere.dss.standard.app.development.service.AbstractRefImportService; -import com.webank.wedatasphere.exchangis.appconn.operation.ExchangisImportOperation; - -public class ExchangisRefImportService extends AbstractRefImportService { - - @Override - protected RefImportOperation createRefImportOperation() { - return new ExchangisImportOperation(this); - } -} diff --git a/exchangis-plugins/exchangis-appconn/src/main/java/com/webank/wedatasphere/exchangis/appconn/utils/AppConnUtils.java b/exchangis-plugins/exchangis-appconn/src/main/java/com/webank/wedatasphere/exchangis/appconn/utils/AppConnUtils.java deleted file mode 100644 index 3c041d513..000000000 --- a/exchangis-plugins/exchangis-appconn/src/main/java/com/webank/wedatasphere/exchangis/appconn/utils/AppConnUtils.java +++ /dev/null @@ -1,42 +0,0 @@ -package com.webank.wedatasphere.exchangis.appconn.utils; - -import com.webank.wedatasphere.dss.common.label.DSSLabel; -import com.webank.wedatasphere.dss.standard.app.development.ref.NodeRequestRef; -import com.webank.wedatasphere.dss.standard.common.exception.operation.ExternalOperationFailedException; -import org.apache.linkis.manager.label.entity.SerializableLabel; -import org.apache.linkis.server.BDPJettyServerHelper; - -import java.util.List; -import java.util.Map; -import java.util.stream.Collectors; - -public class AppConnUtils { - public static String changeDssLabelName(List list){ - String dssLabelStr=""; - if(list != null && !list.isEmpty()){ - dssLabelStr=list.stream().map(SerializableLabel::getStringValue).collect(Collectors.joining(",")); - } - return dssLabelStr; - } - - public static String getId(NodeRequestRef nodeRequestRef) throws Exception { - String externalContent = BDPJettyServerHelper.jacksonJson().writeValueAsString(nodeRequestRef.getJobContent()); - return NumberUtils.parseDoubleString(getNodeId(externalContent)); - } - - public static String getJobContent(Map jobContent) throws Exception { - String externalContent = BDPJettyServerHelper.jacksonJson().writeValueAsString(jobContent); - return NumberUtils.parseDoubleString(getNodeId(externalContent)); - } - - private static String getNodeId(String responseBody) throws ExternalOperationFailedException { - String nodeId=""; - try { - Map responseMap = BDPJettyServerHelper.jacksonJson().readValue(responseBody, Map.class); - nodeId = ((Map) responseMap.get("payload")).get("id").toString(); - }catch (Exception e){ - throw new ExternalOperationFailedException(31022, "Get node Id failed!", e); - } - return nodeId; - } -} diff --git a/exchangis-plugins/exchangis-appconn/src/main/java/com/webank/wedatasphere/exchangis/appconn/utils/NumberUtils.java b/exchangis-plugins/exchangis-appconn/src/main/java/com/webank/wedatasphere/exchangis/appconn/utils/NumberUtils.java deleted file mode 100644 index 1a010b3d9..000000000 --- a/exchangis-plugins/exchangis-appconn/src/main/java/com/webank/wedatasphere/exchangis/appconn/utils/NumberUtils.java +++ /dev/null @@ -1,16 +0,0 @@ -package com.webank.wedatasphere.exchangis.appconn.utils; - -public class NumberUtils { - public static Integer getInt(Object original){ - if(original instanceof Double){ - return ((Double) original).intValue(); - } - return (Integer) original; - } - - public static String parseDoubleString(String doubleString) { - Double doubleValue = Double.parseDouble(doubleString); - Integer intValue = doubleValue.intValue(); - return intValue.toString(); - } -} diff --git a/exchangis-plugins/exchangis-appconn/src/main/java/com/webank/wedatasphere/exchangis/appconn/ExchangisAppConn.java b/exchangis-plugins/exchangis-appconn/src/main/java/com/webank/wedatasphere/exchangis/dss/appconn/ExchangisAppConn.java similarity index 70% rename from exchangis-plugins/exchangis-appconn/src/main/java/com/webank/wedatasphere/exchangis/appconn/ExchangisAppConn.java rename to exchangis-plugins/exchangis-appconn/src/main/java/com/webank/wedatasphere/exchangis/dss/appconn/ExchangisAppConn.java index c59be242e..7240d7864 100644 --- a/exchangis-plugins/exchangis-appconn/src/main/java/com/webank/wedatasphere/exchangis/appconn/ExchangisAppConn.java +++ b/exchangis-plugins/exchangis-appconn/src/main/java/com/webank/wedatasphere/exchangis/dss/appconn/ExchangisAppConn.java @@ -1,14 +1,23 @@ -package com.webank.wedatasphere.exchangis.appconn; +package com.webank.wedatasphere.exchangis.dss.appconn; import com.webank.wedatasphere.dss.appconn.core.ext.ThirdlyAppConn; import com.webank.wedatasphere.dss.appconn.core.impl.AbstractOnlySSOAppConn; import com.webank.wedatasphere.dss.standard.app.development.standard.DevelopmentIntegrationStandard; import com.webank.wedatasphere.dss.standard.app.structure.StructureIntegrationStandard; +/** + * Exchangis AppConn top implement + */ public class ExchangisAppConn extends AbstractOnlySSOAppConn implements ThirdlyAppConn { - + /** + * Project service operation + */ private ExchangisStructureIntegrationStandard exchangisStructureIntegrationStandard; + + /** + * Operation for flow node + */ private ExchangisDevelopmentIntegrationStandard exchangisDevelopmentIntegrationStandard; @Override public DevelopmentIntegrationStandard getOrCreateDevelopmentStandard() { @@ -22,7 +31,7 @@ public StructureIntegrationStandard getOrCreateStructureStandard() { @Override protected void initialize() { - exchangisStructureIntegrationStandard = new ExchangisStructureIntegrationStandard(); - exchangisDevelopmentIntegrationStandard = new ExchangisDevelopmentIntegrationStandard(); + exchangisStructureIntegrationStandard = ExchangisStructureIntegrationStandard.getInstance(); + exchangisDevelopmentIntegrationStandard = ExchangisDevelopmentIntegrationStandard.getInstance(); } } diff --git a/exchangis-plugins/exchangis-appconn/src/main/java/com/webank/wedatasphere/exchangis/appconn/ExchangisDevelopmentIntegrationStandard.java b/exchangis-plugins/exchangis-appconn/src/main/java/com/webank/wedatasphere/exchangis/dss/appconn/ExchangisDevelopmentIntegrationStandard.java similarity index 52% rename from exchangis-plugins/exchangis-appconn/src/main/java/com/webank/wedatasphere/exchangis/appconn/ExchangisDevelopmentIntegrationStandard.java rename to exchangis-plugins/exchangis-appconn/src/main/java/com/webank/wedatasphere/exchangis/dss/appconn/ExchangisDevelopmentIntegrationStandard.java index 074ea8445..b1dd91dcf 100644 --- a/exchangis-plugins/exchangis-appconn/src/main/java/com/webank/wedatasphere/exchangis/appconn/ExchangisDevelopmentIntegrationStandard.java +++ b/exchangis-plugins/exchangis-appconn/src/main/java/com/webank/wedatasphere/exchangis/dss/appconn/ExchangisDevelopmentIntegrationStandard.java @@ -1,18 +1,40 @@ -package com.webank.wedatasphere.exchangis.appconn; +package com.webank.wedatasphere.exchangis.dss.appconn; import com.webank.wedatasphere.dss.standard.app.development.service.*; import com.webank.wedatasphere.dss.standard.app.development.standard.AbstractDevelopmentIntegrationStandard; -import com.webank.wedatasphere.exchangis.appconn.service.*; +import com.webank.wedatasphere.exchangis.dss.appconn.service.*; +/** + * Develop integration + */ public class ExchangisDevelopmentIntegrationStandard extends AbstractDevelopmentIntegrationStandard { + + private static ExchangisDevelopmentIntegrationStandard instance; + + static { + instance = new ExchangisDevelopmentIntegrationStandard(); + } + + /** + * Get singleton instance + * @return development integration + */ + public static ExchangisDevelopmentIntegrationStandard getInstance(){ + return instance; + } + + private ExchangisDevelopmentIntegrationStandard(){ + + } + @Override protected RefCRUDService createRefCRUDService() { - return new ExchangisCRUDService(); + return new ExchangisRefCRUDService(); } @Override protected RefExecutionService createRefExecutionService() { - return new ExchangisExecutionService(); + return new ExchangisRefExecutionService(); } @Override @@ -27,6 +49,6 @@ protected RefImportService createRefImportService() { @Override protected RefQueryService createRefQueryService() { - return new ExchangisQueryService(); + return new ExchangisRefQueryService(); } } diff --git a/exchangis-plugins/exchangis-appconn/src/main/java/com/webank/wedatasphere/exchangis/dss/appconn/ExchangisStructureIntegrationStandard.java b/exchangis-plugins/exchangis-appconn/src/main/java/com/webank/wedatasphere/exchangis/dss/appconn/ExchangisStructureIntegrationStandard.java new file mode 100644 index 000000000..1a68b928e --- /dev/null +++ b/exchangis-plugins/exchangis-appconn/src/main/java/com/webank/wedatasphere/exchangis/dss/appconn/ExchangisStructureIntegrationStandard.java @@ -0,0 +1,29 @@ +package com.webank.wedatasphere.exchangis.dss.appconn; + +import com.webank.wedatasphere.dss.standard.app.structure.AbstractStructureIntegrationStandard; +import com.webank.wedatasphere.dss.standard.app.structure.project.ProjectService; +import com.webank.wedatasphere.exchangis.dss.appconn.service.ExchangisProjectService; + +/** + * Structure(Project) service implement + */ +public class ExchangisStructureIntegrationStandard extends AbstractStructureIntegrationStandard { + + private static ExchangisStructureIntegrationStandard instance; + + static{ + instance = new ExchangisStructureIntegrationStandard(); + } + + /** + * Get the singleton instance + * @return structure integration + */ + public static ExchangisStructureIntegrationStandard getInstance(){ + return instance; + } + @Override + protected ProjectService createProjectService() { + return new ExchangisProjectService(); + } +} diff --git a/exchangis-plugins/exchangis-appconn/src/main/java/com/webank/wedatasphere/exchangis/dss/appconn/constraints/Constraints.java b/exchangis-plugins/exchangis-appconn/src/main/java/com/webank/wedatasphere/exchangis/dss/appconn/constraints/Constraints.java new file mode 100644 index 000000000..b17e6aaf6 --- /dev/null +++ b/exchangis-plugins/exchangis-appconn/src/main/java/com/webank/wedatasphere/exchangis/dss/appconn/constraints/Constraints.java @@ -0,0 +1,37 @@ +package com.webank.wedatasphere.exchangis.dss.appconn.constraints; + + +import org.apache.linkis.common.conf.CommonVars; + +/** + * Constraints + */ +public class Constraints { + + // AppConn name + public final static String EXCHANGIS_APPCONN_NAME = CommonVars.apply("wds.dss.appconn.exchangis.name", "Exchangis").getValue(); + + public final static String DOMAIN_NAME = CommonVars.apply("wds.dss.appconn.exchangis.domain.name", "DSS").getValue(); + + // Constraint in Project operation + public final static String PROJECT_ID = "projectId"; + + // Node type + public final static String NODE_TYPE_SQOOP = CommonVars.apply("wds.dss.appconn.exchangis.node-type.sqoop", "linkis.appconn.exchangis.sqoop").getValue(); + public final static String NODE_TYPE_DATAX = CommonVars.apply("wds.dss.appconn.exchangis.node-type.datax", "linkis.appconn.exchangis.datax").getValue(); + + // Engine type + public final static String ENGINE_TYPE_DATAX_NAME = CommonVars.apply("wds.dss.appconn.exchangis.engine.datax.name", "DATAX").getValue(); + public final static String ENGINE_TYPE_SQOOP_NAME = CommonVars.apply("wds.dss.appconn.exchangis.engine.sqoop.name", "SQOOP").getValue(); + + // Job type + public final static String JOB_TYPE_OFFLINE = CommonVars.apply("wds.dss.appconn.exchangis.job-type.offline", "OFFLINE").getValue(); + + // Constraint in Ref operation + public final static String REF_JOB_DESC = "desc"; + public final static String REF_JOB_ID = "id"; + public final static String REF_JUMP_URL_FORMAT = "dss/exchangis/#/childJobManagement"; + + +} + diff --git a/exchangis-plugins/exchangis-appconn/src/main/java/com/webank/wedatasphere/exchangis/dss/appconn/operation/AbstractExchangisOperation.java b/exchangis-plugins/exchangis-appconn/src/main/java/com/webank/wedatasphere/exchangis/dss/appconn/operation/AbstractExchangisOperation.java new file mode 100644 index 000000000..d607cc131 --- /dev/null +++ b/exchangis-plugins/exchangis-appconn/src/main/java/com/webank/wedatasphere/exchangis/dss/appconn/operation/AbstractExchangisOperation.java @@ -0,0 +1,171 @@ +package com.webank.wedatasphere.exchangis.dss.appconn.operation; + +import com.webank.wedatasphere.dss.standard.app.sso.Workspace; +import com.webank.wedatasphere.dss.standard.app.sso.builder.SSOUrlBuilderOperation; +import com.webank.wedatasphere.dss.standard.app.sso.request.SSORequestOperation; +import com.webank.wedatasphere.dss.standard.app.sso.request.SSORequestService; +import com.webank.wedatasphere.dss.standard.common.app.AppIntegrationService; +import com.webank.wedatasphere.dss.standard.common.entity.ref.RequestRef; +import com.webank.wedatasphere.dss.standard.common.exception.operation.ExternalOperationFailedException; +import com.webank.wedatasphere.exchangis.dss.appconn.constraints.Constraints; +import com.webank.wedatasphere.exchangis.dss.appconn.request.action.HttpExtAction; +import com.webank.wedatasphere.exchangis.dss.appconn.response.result.ExchangisEntityRespResult; +import org.apache.commons.io.IOUtils; +import org.apache.commons.lang.StringUtils; +import org.apache.linkis.httpclient.request.HttpAction; +import org.apache.linkis.httpclient.request.UserAction; +import org.apache.linkis.httpclient.response.HttpResult; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.Map; +import java.util.Objects; +import java.util.function.Function; +import java.util.function.Supplier; + +/** + * Abstract implement, contains the method to create sso request + */ +public abstract class AbstractExchangisOperation { + + private SSORequestService ssoRequestService; + + private String uri; + + /** + * Refer to the url in table 'dss_appconn_instance' + */ + private String baseURL; + + public AbstractExchangisOperation(){ + + } + + protected abstract Logger getLogger(); + + public AbstractExchangisOperation(String[] uriParts){ + if (Objects.nonNull(uriParts)){ + uri = StringUtils.join(uriParts, IOUtils.DIR_SEPARATOR_UNIX); + } + } + public AbstractExchangisOperation(AppIntegrationService appIntegrationService){ + setSSORequestService(appIntegrationService); + } + + /** + * Set sso request service + * @param appIntegrationService integrated sso request service + */ + protected void setSSORequestService(AppIntegrationService appIntegrationService){ + if (Objects.nonNull(appIntegrationService)){ + this.ssoRequestService = appIntegrationService.getSSORequestService(); + // Also upgrade the base url + this.baseURL = appIntegrationService.getAppInstance().getBaseUrl(); + } + } + + @SuppressWarnings("unchecked") + protected SSORequestOperation getOrCreateSSORequestOperation(String name) throws ExternalOperationFailedException { + if (Objects.nonNull(this.ssoRequestService)){ + return ssoRequestService.createSSORequestOperation(name); + } + throw new ExternalOperationFailedException(-1, "The ssoRequestService is empty in operation: [" + + this.getClass().getSimpleName() + "]"); + } + + protected SSORequestOperation getOrCreateSSORequestOperation() throws ExternalOperationFailedException{ + return getOrCreateSSORequestOperation(getAppName()); + } + + /** + * Get sso url builder operation + * @param workspace work space + * @param appName app name + * @param url url + * @return + */ + protected SSOUrlBuilderOperation getSSOUrlBuilderOperation(Workspace workspace, + String appName, String url){ + return workspace.getSSOUrlBuilderOperation().copy().setAppName(appName).setReqUrl(requestURL()) + .setWorkspace(workspace.getWorkspaceName()); + } + + protected SSOUrlBuilderOperation getSSOUrlBuilderOperation(Workspace workspace){ + return getSSOUrlBuilderOperation(workspace, getAppName(), requestURL()); + } + + /** + * AppConn name + * @return name value + */ + protected String getAppName() { + return Constraints.EXCHANGIS_APPCONN_NAME; + } + + /** + * Get request url of operation + * @return url value + */ + public String requestURL(){ + return requestURL(uri); + } + + public String requestURL(String customUri){ + return baseURL.endsWith(IOUtils.DIR_SEPARATOR_UNIX + "") ? + baseURL + customUri : baseURL + IOUtils.DIR_SEPARATOR_UNIX + customUri; + } + + /** + * Send request and get response entity + * @param url url + * @param workspace workspace + * @param requestRef request ref + * @param httpActionBuilder action builder + * @param entityClass response entity class + * @param entityParameters parametric types of response entity class + * @param response entity + * @param request ref + * @return + * @throws ExternalOperationFailedException + */ + protected ExchangisEntityRespResult.BasicMessageEntity requestToGetEntity(String url, + Workspace workspace, + T requestRef, HttpActionBuilder httpActionBuilder, Class entityClass, Class... entityParameters) throws ExternalOperationFailedException { + HttpExtAction action = httpActionBuilder.build(requestRef); + if (Objects.nonNull(action)){ + SSOUrlBuilderOperation ssoUrlBuilderOperation = getSSOUrlBuilderOperation(workspace, getAppName(), url); + ExchangisEntityRespResult.BasicMessageEntity entity; + try { + action.setUrl(ssoUrlBuilderOperation.getBuiltUrl()); + SSORequestOperation ssoRequestOperation = getOrCreateSSORequestOperation(); + ExchangisEntityRespResult httpResult = new ExchangisEntityRespResult(ssoRequestOperation.requestWithSSO(ssoUrlBuilderOperation, action)); + entity = httpResult.getEntity(entityClass, entityParameters); + return entity; + } catch (Exception e){ + if (e instanceof ExternalOperationFailedException){ + getLogger().error("Response process exception: url: [" + url + "], message: [" + e.getMessage(), e); + throw (ExternalOperationFailedException)e; + } + getLogger().error("Request to url: [" + url + "] exception", e); + throw new ExternalOperationFailedException(-1, e.getMessage()); + } + } + + return null; + } + + protected ExchangisEntityRespResult.BasicMessageEntity requestToGetEntity(Workspace workspace, + T requestRef, HttpActionBuilder httpActionBuilder, Class entityClass, Class... entityParameters) throws ExternalOperationFailedException { + return requestToGetEntity(requestURL(), workspace, requestRef, httpActionBuilder, entityClass, entityParameters); + } + @FunctionalInterface + public interface HttpActionBuilder{ + + /** + * Build main entrance + * @return http action + */ + HttpExtAction build(T input) throws ExternalOperationFailedException; + } + +} diff --git a/exchangis-plugins/exchangis-appconn/src/main/java/com/webank/wedatasphere/exchangis/dss/appconn/operation/project/AbstractExchangisProjectOperation.java b/exchangis-plugins/exchangis-appconn/src/main/java/com/webank/wedatasphere/exchangis/dss/appconn/operation/project/AbstractExchangisProjectOperation.java new file mode 100644 index 000000000..567ee804b --- /dev/null +++ b/exchangis-plugins/exchangis-appconn/src/main/java/com/webank/wedatasphere/exchangis/dss/appconn/operation/project/AbstractExchangisProjectOperation.java @@ -0,0 +1,46 @@ +package com.webank.wedatasphere.exchangis.dss.appconn.operation.project; + +import com.webank.wedatasphere.dss.standard.app.structure.project.ProjectRequestRef; +import com.webank.wedatasphere.dss.standard.common.exception.operation.ExternalOperationFailedException; +import com.webank.wedatasphere.exchangis.dss.appconn.operation.AbstractExchangisOperation; +import com.webank.wedatasphere.exchangis.dss.appconn.request.action.ExchangisEntityPostAction; +import com.webank.wedatasphere.exchangis.dss.appconn.request.entity.ProjectReqEntity; +import com.webank.wedatasphere.exchangis.dss.appconn.utils.AppConnUtils; +import com.webank.wedatasphere.exchangis.dss.appconn.utils.JsonExtension; + +import java.util.HashMap; +import java.util.Map; + +/** + * Abstract implement of operation related by project + */ +public abstract class AbstractExchangisProjectOperation extends AbstractExchangisOperation { + + public AbstractExchangisProjectOperation(String[] uriParts) { + super(uriParts); + } + + public AbstractExchangisProjectOperation(){ + + } + /** + * Get Project entity + * @return postEntity + */ + protected ProjectReqEntity getProjectEntity(ProjectRequestRef projectRequestRef){ + // Build project request entity + Map source = new HashMap<>(); + String owner = projectRequestRef.getCreateBy(); + ProjectReqEntity projectReqEntity = new ProjectReqEntity(owner, + projectRequestRef.getName(), projectRequestRef.getDescription(), source); + projectReqEntity.setTags(AppConnUtils.serializeDssLabel(projectRequestRef.getDSSLabels())); + // Try to set the project request ref into the source map + try { + Map requestRefMap = JsonExtension.convert(projectReqEntity, Map.class, String.class, Object.class); + source.putAll(requestRefMap); + } catch (ExternalOperationFailedException e){ + getLogger().warn("Unable to serialize the project requestRef into the source map", e); + } + return projectReqEntity; + } +} diff --git a/exchangis-plugins/exchangis-appconn/src/main/java/com/webank/wedatasphere/exchangis/dss/appconn/operation/project/ExchangisProjectCreationOperation.java b/exchangis-plugins/exchangis-appconn/src/main/java/com/webank/wedatasphere/exchangis/dss/appconn/operation/project/ExchangisProjectCreationOperation.java new file mode 100644 index 000000000..bd6b77dc3 --- /dev/null +++ b/exchangis-plugins/exchangis-appconn/src/main/java/com/webank/wedatasphere/exchangis/dss/appconn/operation/project/ExchangisProjectCreationOperation.java @@ -0,0 +1,84 @@ +package com.webank.wedatasphere.exchangis.dss.appconn.operation.project; + +import com.webank.wedatasphere.dss.standard.app.sso.builder.SSOUrlBuilderOperation; +import com.webank.wedatasphere.dss.standard.app.sso.request.SSORequestOperation; +import com.webank.wedatasphere.dss.standard.app.structure.StructureService; +import com.webank.wedatasphere.dss.standard.app.structure.project.ProjectCreationOperation; +import com.webank.wedatasphere.dss.standard.app.structure.project.ProjectRequestRef; +import com.webank.wedatasphere.dss.standard.app.structure.project.ProjectResponseRef; +import com.webank.wedatasphere.dss.standard.common.exception.operation.ExternalOperationFailedException; + +import com.webank.wedatasphere.exchangis.dss.appconn.constraints.Constraints; +import com.webank.wedatasphere.exchangis.dss.appconn.request.action.ExchangisEntityPostAction; +import com.webank.wedatasphere.exchangis.dss.appconn.operation.AbstractExchangisOperation; +import com.webank.wedatasphere.exchangis.dss.appconn.ref.ExchangisProjectResponseRef; +import com.webank.wedatasphere.exchangis.dss.appconn.request.entity.ProjectReqEntity; +import com.webank.wedatasphere.exchangis.dss.appconn.response.result.ExchangisEntityRespResult; +import com.webank.wedatasphere.exchangis.dss.appconn.utils.AppConnUtils; +import com.webank.wedatasphere.exchangis.dss.appconn.utils.JsonExtension; +import org.apache.linkis.httpclient.request.HttpAction; +import org.apache.linkis.httpclient.response.HttpResult; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.HashMap; +import java.util.Map; +import java.util.Objects; + +/** + * Project create operation + */ +public class ExchangisProjectCreationOperation extends AbstractExchangisProjectOperation implements ProjectCreationOperation { + private final static Logger LOG = LoggerFactory.getLogger(ExchangisProjectCreationOperation.class); + + private StructureService structureService; + + public ExchangisProjectCreationOperation(StructureService structureService) { + super(new String[]{"project"}); + setStructureService(structureService); + } + + @Override + public ProjectResponseRef createProject(ProjectRequestRef projectRequestRef) throws ExternalOperationFailedException { + LOG.info("create project request => dss_projectId:{}, name:{}, createUser:{}, parameters:{}, workspaceName:{}", + projectRequestRef.getId(), projectRequestRef.getName(), projectRequestRef.getCreateBy(), projectRequestRef.getParameters().toString(), + projectRequestRef.getWorkspace().getWorkspaceName()); + ExchangisEntityRespResult.BasicMessageEntity> entity = requestToGetEntity(projectRequestRef.getWorkspace(), projectRequestRef, + (requestRef) -> { + // Build project post(add) action + ExchangisEntityPostAction postAction = new ExchangisEntityPostAction<>(getProjectEntity(requestRef)); + postAction.setUser(requestRef.getCreateBy()); + return postAction; + }, Map.class); + if (Objects.isNull(entity)){ + throw new ExternalOperationFailedException(31020, "The response entity cannot be empty", null); + } + ExchangisEntityRespResult httpResult = entity.getResult(); + LOG.info("create project response => status {}, response {}", httpResult.getStatusCode(), httpResult.getResponseBody()); + long projectId; + try { + projectId = Long.parseLong(String.valueOf(entity.getData().get(Constraints.PROJECT_ID))); + } catch (Exception e){ + throw new ExternalOperationFailedException(31020, "Fail to resolve the project id from response entity", e); + } + ExchangisProjectResponseRef responseRef = new ExchangisProjectResponseRef(httpResult, projectId); + responseRef.setAppInstance(structureService.getAppInstance()); + return responseRef; + } + + @Override + public void init() { + + } + + @Override + public void setStructureService(StructureService structureService) { + this.structureService = structureService; + setSSORequestService(this.structureService); + } + + @Override + protected Logger getLogger() { + return LOG; + } +} diff --git a/exchangis-plugins/exchangis-appconn/src/main/java/com/webank/wedatasphere/exchangis/dss/appconn/operation/project/ExchangisProjectDeletionOperation.java b/exchangis-plugins/exchangis-appconn/src/main/java/com/webank/wedatasphere/exchangis/dss/appconn/operation/project/ExchangisProjectDeletionOperation.java new file mode 100644 index 000000000..4a4cd2bb3 --- /dev/null +++ b/exchangis-plugins/exchangis-appconn/src/main/java/com/webank/wedatasphere/exchangis/dss/appconn/operation/project/ExchangisProjectDeletionOperation.java @@ -0,0 +1,72 @@ +package com.webank.wedatasphere.exchangis.dss.appconn.operation.project; + +import com.webank.wedatasphere.dss.standard.app.sso.request.SSORequestOperation; +import com.webank.wedatasphere.dss.standard.app.structure.StructureService; +import com.webank.wedatasphere.dss.standard.app.structure.project.ProjectDeletionOperation; +import com.webank.wedatasphere.dss.standard.app.structure.project.ProjectRequestRef; +import com.webank.wedatasphere.dss.standard.app.structure.project.ProjectResponseRef; +import com.webank.wedatasphere.dss.standard.common.exception.operation.ExternalOperationFailedException; +import com.webank.wedatasphere.exchangis.dss.appconn.constraints.Constraints; +import com.webank.wedatasphere.exchangis.dss.appconn.request.action.ExchangisDeleteAction; +import com.webank.wedatasphere.exchangis.dss.appconn.ref.ExchangisProjectResponseRef; +import com.webank.wedatasphere.exchangis.dss.appconn.response.result.ExchangisEntityRespResult; +import org.apache.linkis.httpclient.request.HttpAction; +import org.apache.linkis.httpclient.response.HttpResult; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.Map; +import java.util.Objects; + + +/** + * Project delete operation + */ +public class ExchangisProjectDeletionOperation extends AbstractExchangisProjectOperation implements ProjectDeletionOperation { + private static Logger LOG = LoggerFactory.getLogger(ExchangisProjectDeletionOperation.class); + + private SSORequestOperation ssoRequestOperation; + private StructureService structureService; + + public ExchangisProjectDeletionOperation(StructureService structureService) { + setStructureService(structureService); + } + + @Override + protected Logger getLogger() { + return LOG; + } + + @Override + public ProjectResponseRef deleteProject(ProjectRequestRef projectRequestRef) throws ExternalOperationFailedException { + // TODO Get the project id + long projectId = 0l; + String url = requestURL("/project/" + projectId); + LOG.info("delete project request => dss_projectId:{}, name:{}, createName:{}", projectRequestRef.getId(), + projectRequestRef.getName(), projectRequestRef.getCreateBy()); + ExchangisEntityRespResult.BasicMessageEntity> entity = requestToGetEntity(url, projectRequestRef.getWorkspace(), projectRequestRef, + (requestRef) -> { + // Build project delete action + return new ExchangisDeleteAction(requestRef.getCreateBy()); + }, Map.class); + if (Objects.isNull(entity)){ + throw new ExternalOperationFailedException(31020, "The response entity cannot be empty", null); + } + ExchangisEntityRespResult httpResult = entity.getResult(); + LOG.info("delete project response => status {}, response {}", httpResult.getStatusCode(), httpResult.getResponseBody()); + ExchangisProjectResponseRef responseRef = new ExchangisProjectResponseRef(httpResult, null); + responseRef.setAppInstance(structureService.getAppInstance()); + return responseRef; + } + + @Override + public void init() { + + } + + @Override + public void setStructureService(StructureService structureService) { + this.structureService = structureService; + setSSORequestService(this.structureService); + } +} diff --git a/exchangis-plugins/exchangis-appconn/src/main/java/com/webank/wedatasphere/exchangis/dss/appconn/operation/project/ExchangisProjectUpdateOperation.java b/exchangis-plugins/exchangis-appconn/src/main/java/com/webank/wedatasphere/exchangis/dss/appconn/operation/project/ExchangisProjectUpdateOperation.java new file mode 100644 index 000000000..7a504431f --- /dev/null +++ b/exchangis-plugins/exchangis-appconn/src/main/java/com/webank/wedatasphere/exchangis/dss/appconn/operation/project/ExchangisProjectUpdateOperation.java @@ -0,0 +1,77 @@ +package com.webank.wedatasphere.exchangis.dss.appconn.operation.project; + +import com.webank.wedatasphere.dss.standard.app.structure.StructureService; +import com.webank.wedatasphere.dss.standard.app.structure.project.ProjectRequestRef; +import com.webank.wedatasphere.dss.standard.app.structure.project.ProjectResponseRef; +import com.webank.wedatasphere.dss.standard.app.structure.project.ProjectUpdateOperation; +import com.webank.wedatasphere.dss.standard.common.exception.operation.ExternalOperationFailedException; +import com.webank.wedatasphere.exchangis.dss.appconn.constraints.Constraints; +import com.webank.wedatasphere.exchangis.dss.appconn.ref.ExchangisProjectResponseRef; +import com.webank.wedatasphere.exchangis.dss.appconn.request.action.ExchangisEntityPutAction; +import com.webank.wedatasphere.exchangis.dss.appconn.request.entity.ProjectReqEntity; +import com.webank.wedatasphere.exchangis.dss.appconn.response.result.ExchangisEntityRespResult; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.Map; +import java.util.Objects; +import java.util.Optional; +import java.util.concurrent.atomic.AtomicLong; + +/** + * Project update operation + */ +public class ExchangisProjectUpdateOperation extends AbstractExchangisProjectOperation implements ProjectUpdateOperation { + private static final Logger LOG = LoggerFactory.getLogger(ExchangisProjectUpdateOperation.class); + + private StructureService structureService; + + public ExchangisProjectUpdateOperation(StructureService structureService) { + setStructureService(structureService); + } + + @Override + public ProjectResponseRef updateProject(ProjectRequestRef projectRequestRef) throws ExternalOperationFailedException { + // TODO Get the project id + long projectId = 0l; + String url = requestURL("/project/" + projectId); + LOG.info("update project request => dss_projectId:{}, name:{}, createName:{}", + projectRequestRef.getId(), projectRequestRef.getName(),projectRequestRef.getCreateBy()); + ExchangisEntityRespResult.BasicMessageEntity> entity = requestToGetEntity(url, projectRequestRef.getWorkspace(), projectRequestRef, + (requestRef) -> { + // Build project put(update) action + return new ExchangisEntityPutAction<>(getProjectEntity(requestRef), requestRef.getCreateBy()); + }, Map.class); + if (Objects.isNull(entity)){ + throw new ExternalOperationFailedException(31020, "The response entity cannot be empty", null); + } + ExchangisEntityRespResult httpResult = entity.getResult(); + LOG.info("update project response => status {}, response {}", httpResult.getStatusCode(), httpResult.getResponseBody()); + AtomicLong newProjectId = new AtomicLong(projectId); + try { + Optional.ofNullable(entity.getData()).ifPresent( data -> newProjectId + .set(Long.parseLong(String.valueOf(data.getOrDefault(Constraints.PROJECT_ID, newProjectId.get()))))); + } catch (Exception e){ + throw new ExternalOperationFailedException(31020, "Fail to resolve the project id from response entity", e); + } + ExchangisProjectResponseRef responseRef = new ExchangisProjectResponseRef(httpResult, newProjectId.get()); + responseRef.setAppInstance(structureService.getAppInstance()); + return responseRef; + } + + @Override + public void init() { + + } + + @Override + public void setStructureService(StructureService structureService) { + this.structureService = structureService; + setSSORequestService(this.structureService); + } + + @Override + protected Logger getLogger() { + return LOG; + } +} diff --git a/exchangis-plugins/exchangis-appconn/src/main/java/com/webank/wedatasphere/exchangis/dss/appconn/operation/ref/AbstractExchangisRefOperation.java b/exchangis-plugins/exchangis-appconn/src/main/java/com/webank/wedatasphere/exchangis/dss/appconn/operation/ref/AbstractExchangisRefOperation.java new file mode 100644 index 000000000..7d0260bf4 --- /dev/null +++ b/exchangis-plugins/exchangis-appconn/src/main/java/com/webank/wedatasphere/exchangis/dss/appconn/operation/ref/AbstractExchangisRefOperation.java @@ -0,0 +1,40 @@ +package com.webank.wedatasphere.exchangis.dss.appconn.operation.ref; + +import com.webank.wedatasphere.dss.standard.app.development.ref.NodeRequestRef; +import com.webank.wedatasphere.exchangis.dss.appconn.constraints.Constraints; +import com.webank.wedatasphere.exchangis.dss.appconn.operation.AbstractExchangisOperation; +import com.webank.wedatasphere.exchangis.dss.appconn.request.entity.RefJobReqEntity; +import com.webank.wedatasphere.exchangis.dss.appconn.utils.AppConnUtils; + +import java.util.Collections; +import java.util.Map; +import java.util.Optional; + +/** + * Abstract implement of operation related by ref + */ +public abstract class AbstractExchangisRefOperation extends AbstractExchangisOperation { + public AbstractExchangisRefOperation() { + } + + public AbstractExchangisRefOperation(String[] uriParts) { + super(uriParts); + } + + /** + * Job request entity + * @param nodeRequestRef node request ref + * @param engineType engine type + * @return + */ + public RefJobReqEntity getRefJobReqEntity(NodeRequestRef nodeRequestRef, String engineType){ + Map jobContent = Optional.ofNullable(nodeRequestRef.getJobContent()).orElse(Collections.emptyMap()); + String desc = String.valueOf(jobContent.getOrDefault(Constraints.REF_JOB_DESC, "")); + RefJobReqEntity jobReqEntity = new RefJobReqEntity(nodeRequestRef.getName(), desc, + Constraints.JOB_TYPE_OFFLINE, engineType, nodeRequestRef.getProjectId()); + jobReqEntity.setJobLabels(AppConnUtils.serializeDssLabel(nodeRequestRef.getDSSLabels())); + //TODO store the job content + jobReqEntity.setSource(nodeRequestRef.getJobContent()); + return jobReqEntity; + } +} diff --git a/exchangis-plugins/exchangis-appconn/src/main/java/com/webank/wedatasphere/exchangis/dss/appconn/operation/ref/ExchangisExportOperation.java b/exchangis-plugins/exchangis-appconn/src/main/java/com/webank/wedatasphere/exchangis/dss/appconn/operation/ref/ExchangisExportOperation.java new file mode 100644 index 000000000..9768b9714 --- /dev/null +++ b/exchangis-plugins/exchangis-appconn/src/main/java/com/webank/wedatasphere/exchangis/dss/appconn/operation/ref/ExchangisExportOperation.java @@ -0,0 +1,40 @@ +package com.webank.wedatasphere.exchangis.dss.appconn.operation.ref; + +import com.webank.wedatasphere.dss.standard.app.development.operation.RefExportOperation; +import com.webank.wedatasphere.dss.standard.app.development.ref.ExportRequestRef; +import com.webank.wedatasphere.dss.standard.app.development.service.DevelopmentService; +import com.webank.wedatasphere.dss.standard.common.entity.ref.ResponseRef; +import com.webank.wedatasphere.dss.standard.common.exception.operation.ExternalOperationFailedException; +import com.webank.wedatasphere.exchangis.dss.appconn.constraints.Constraints; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Ref export operation + */ +public class ExchangisExportOperation extends AbstractExchangisRefOperation implements RefExportOperation { + private final static Logger LOG = LoggerFactory.getLogger(ExchangisExportOperation.class); + + private DevelopmentService developmentService; + + public ExchangisExportOperation(DevelopmentService developmentService){ + this.developmentService = developmentService; + setSSORequestService(this.developmentService); + } + + + @Override + public ResponseRef exportRef(ExportRequestRef exportRequestRef) throws ExternalOperationFailedException { + return null; + } + + @Override + public void setDevelopmentService(DevelopmentService developmentService) { + this.developmentService = developmentService; + } + + @Override + protected Logger getLogger() { + return LOG; + } +} diff --git a/exchangis-plugins/exchangis-appconn/src/main/java/com/webank/wedatasphere/exchangis/dss/appconn/operation/ref/ExchangisImportOperation.java b/exchangis-plugins/exchangis-appconn/src/main/java/com/webank/wedatasphere/exchangis/dss/appconn/operation/ref/ExchangisImportOperation.java new file mode 100644 index 000000000..a0bae1b41 --- /dev/null +++ b/exchangis-plugins/exchangis-appconn/src/main/java/com/webank/wedatasphere/exchangis/dss/appconn/operation/ref/ExchangisImportOperation.java @@ -0,0 +1,45 @@ +package com.webank.wedatasphere.exchangis.dss.appconn.operation.ref; + +import com.webank.wedatasphere.dss.standard.app.development.operation.RefImportOperation; +import com.webank.wedatasphere.dss.standard.app.development.ref.ImportRequestRef; +import com.webank.wedatasphere.dss.standard.app.development.service.DevelopmentService; +import com.webank.wedatasphere.dss.standard.app.sso.builder.SSOUrlBuilderOperation; +import com.webank.wedatasphere.dss.standard.app.sso.request.SSORequestOperation; +import com.webank.wedatasphere.dss.standard.common.entity.ref.ResponseRef; +import com.webank.wedatasphere.dss.standard.common.exception.operation.ExternalOperationFailedException; +import com.webank.wedatasphere.exchangis.dss.appconn.constraints.Constraints; +import com.webank.wedatasphere.exchangis.dss.appconn.request.action.ExchangisPostAction; +import com.webank.wedatasphere.exchangis.dss.appconn.ref.AbstractExchangisResponseRef; +import org.apache.linkis.httpclient.response.HttpResult; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + + +/** + * Ref import operation + */ +public class ExchangisImportOperation extends AbstractExchangisRefOperation implements RefImportOperation { + private final static Logger LOG = LoggerFactory.getLogger(ExchangisImportOperation.class); + + private DevelopmentService developmentService; + + public ExchangisImportOperation(DevelopmentService developmentService){ + this.developmentService = developmentService; + setSSORequestService(this.developmentService); + } + + @Override + public ResponseRef importRef(ImportRequestRef importRequestRef) throws ExternalOperationFailedException { + return null; + } + + @Override + public void setDevelopmentService(DevelopmentService developmentService) { + this.developmentService = developmentService; + } + + @Override + protected Logger getLogger() { + return LOG; + } +} diff --git a/exchangis-plugins/exchangis-appconn/src/main/java/com/webank/wedatasphere/exchangis/dss/appconn/operation/ref/ExchangisRefCreationOperation.java b/exchangis-plugins/exchangis-appconn/src/main/java/com/webank/wedatasphere/exchangis/dss/appconn/operation/ref/ExchangisRefCreationOperation.java new file mode 100644 index 000000000..9f1cc5a84 --- /dev/null +++ b/exchangis-plugins/exchangis-appconn/src/main/java/com/webank/wedatasphere/exchangis/dss/appconn/operation/ref/ExchangisRefCreationOperation.java @@ -0,0 +1,73 @@ +package com.webank.wedatasphere.exchangis.dss.appconn.operation.ref; + +import com.webank.wedatasphere.dss.standard.app.development.operation.RefCreationOperation; +import com.webank.wedatasphere.dss.standard.app.development.ref.CreateRequestRef; +import com.webank.wedatasphere.dss.standard.app.development.ref.NodeRequestRef; +import com.webank.wedatasphere.dss.standard.app.development.service.DevelopmentService; +import com.webank.wedatasphere.dss.standard.common.entity.ref.ResponseRef; +import com.webank.wedatasphere.dss.standard.common.exception.operation.ExternalOperationFailedException; +import com.webank.wedatasphere.exchangis.dss.appconn.constraints.Constraints; +import com.webank.wedatasphere.exchangis.dss.appconn.ref.ExchangisCommonResponseDef; +import com.webank.wedatasphere.exchangis.dss.appconn.request.action.ExchangisEntityPostAction; +import com.webank.wedatasphere.exchangis.dss.appconn.response.result.ExchangisEntityRespResult; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.Map; +import java.util.Objects; + + +/** + * Ref creation operation + */ +public class ExchangisRefCreationOperation extends AbstractExchangisRefOperation implements RefCreationOperation { + private final static Logger LOG = LoggerFactory.getLogger(ExchangisRefCreationOperation.class); + + DevelopmentService developmentService; + + public ExchangisRefCreationOperation(DevelopmentService developmentService){ + super(new String[]{"job"}); + this.developmentService = developmentService; + setSSORequestService(developmentService); + } + + @Override + public ResponseRef createRef(CreateRequestRef createRequestRef) throws ExternalOperationFailedException { + NodeRequestRef exchangisCreateRequestRef = (NodeRequestRef) createRequestRef; + exchangisCreateRequestRef.getProjectName(); + ResponseRef responseRef = null; + if(Constraints.NODE_TYPE_SQOOP.equalsIgnoreCase(exchangisCreateRequestRef.getNodeType())){ + responseRef = sendOffLineRequest(exchangisCreateRequestRef, Constraints.ENGINE_TYPE_SQOOP_NAME); + }else if(Constraints.NODE_TYPE_DATAX.equalsIgnoreCase(exchangisCreateRequestRef.getNodeType())){ + responseRef = sendOffLineRequest(exchangisCreateRequestRef, Constraints.ENGINE_TYPE_DATAX_NAME); + } + return responseRef; + } + + private ResponseRef sendOffLineRequest(NodeRequestRef nodeRequestRef, String engineType) throws ExternalOperationFailedException { + LOG.info("create {} job request => jobContent:{}, projectId:{}, projectName:{}, parameters:{}, type:{}", + engineType, nodeRequestRef.getJobContent(), nodeRequestRef.getProjectId(), nodeRequestRef.getProjectName(), + nodeRequestRef.getParameters().toString(), nodeRequestRef.getType()); + ExchangisEntityRespResult.BasicMessageEntity> entity = requestToGetEntity(nodeRequestRef.getWorkspace(), nodeRequestRef, + (requestRef) -> { + // Build ref creation action + return new ExchangisEntityPostAction<>(getRefJobReqEntity(requestRef, engineType), requestRef.getUserName()); + }, Map.class); + if (Objects.isNull(entity)){ + throw new ExternalOperationFailedException(31020, "The response entity cannot be empty", null); + } + ExchangisEntityRespResult httpResult = entity.getResult(); + LOG.info("create {} job response => status: {}, response: {}", engineType, httpResult.getStatusCode(), httpResult.getResponseBody()); + return new ExchangisCommonResponseDef(httpResult); + } + + @Override + public void setDevelopmentService(DevelopmentService developmentService) { + this.developmentService = developmentService; + } + + @Override + protected Logger getLogger() { + return LOG; + } +} diff --git a/exchangis-plugins/exchangis-appconn/src/main/java/com/webank/wedatasphere/exchangis/dss/appconn/operation/ref/ExchangisRefDeletionOperation.java b/exchangis-plugins/exchangis-appconn/src/main/java/com/webank/wedatasphere/exchangis/dss/appconn/operation/ref/ExchangisRefDeletionOperation.java new file mode 100644 index 000000000..d06c9279f --- /dev/null +++ b/exchangis-plugins/exchangis-appconn/src/main/java/com/webank/wedatasphere/exchangis/dss/appconn/operation/ref/ExchangisRefDeletionOperation.java @@ -0,0 +1,66 @@ +package com.webank.wedatasphere.exchangis.dss.appconn.operation.ref; + +import com.google.common.collect.Maps; +import com.webank.wedatasphere.dss.standard.app.development.operation.RefDeletionOperation; +import com.webank.wedatasphere.dss.standard.app.development.ref.NodeRequestRef; +import com.webank.wedatasphere.dss.standard.app.development.service.DevelopmentService; +import com.webank.wedatasphere.dss.standard.app.sso.builder.SSOUrlBuilderOperation; +import com.webank.wedatasphere.dss.standard.app.sso.request.SSORequestOperation; +import com.webank.wedatasphere.dss.standard.common.entity.ref.RequestRef; +import com.webank.wedatasphere.dss.standard.common.exception.operation.ExternalOperationFailedException; +import com.webank.wedatasphere.exchangis.dss.appconn.constraints.Constraints; +import com.webank.wedatasphere.exchangis.dss.appconn.request.action.ExchangisDeleteAction; +import com.webank.wedatasphere.exchangis.dss.appconn.response.result.ExchangisEntityRespResult; +import com.webank.wedatasphere.exchangis.dss.appconn.utils.AppConnUtils; +import org.apache.linkis.httpclient.response.HttpResult; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.Map; +import java.util.Objects; + +/** + * Ref delete operation + */ +public class ExchangisRefDeletionOperation extends AbstractExchangisRefOperation implements RefDeletionOperation { + private final static Logger LOG = LoggerFactory.getLogger(ExchangisRefDeletionOperation.class); + + DevelopmentService developmentService; + + public ExchangisRefDeletionOperation(DevelopmentService developmentService){ + this.developmentService = developmentService; + setSSORequestService(developmentService); + } + + @Override + public void deleteRef(RequestRef deleteRequestRef) throws ExternalOperationFailedException { + NodeRequestRef nodeRequestRef = (NodeRequestRef) deleteRequestRef; + deleteJob(nodeRequestRef); + } + + private void deleteJob(NodeRequestRef nodeRequestRef) throws ExternalOperationFailedException{ + Long id = AppConnUtils.resolveParam(nodeRequestRef.getJobContent(), Constraints.REF_JOB_ID, Long.class); + LOG.info("delete job request => id: {}, jobContext:{}", + id, nodeRequestRef.getJobContent().toString()); + String url = requestURL("/job/" + id); + ExchangisEntityRespResult.BasicMessageEntity> entity = requestToGetEntity(url, nodeRequestRef.getWorkspace(), nodeRequestRef, + (requestRef) ->{ + // Build ref delete action + return new ExchangisDeleteAction(requestRef.getUserName()); + }, Map.class); + if (Objects.isNull(entity)){ + throw new ExternalOperationFailedException(31020, "The response entity cannot be empty", null); + } + LOG.info("delete job response => status: {}, url:{}", entity.getResult().getStatusCode(), url); + } + + @Override + public void setDevelopmentService(DevelopmentService developmentService) { + this.developmentService = developmentService; + } + + @Override + protected Logger getLogger() { + return LOG; + } +} diff --git a/exchangis-plugins/exchangis-appconn/src/main/java/com/webank/wedatasphere/exchangis/dss/appconn/operation/ref/ExchangisRefExecutionOperation.java b/exchangis-plugins/exchangis-appconn/src/main/java/com/webank/wedatasphere/exchangis/dss/appconn/operation/ref/ExchangisRefExecutionOperation.java new file mode 100644 index 000000000..1bf442df7 --- /dev/null +++ b/exchangis-plugins/exchangis-appconn/src/main/java/com/webank/wedatasphere/exchangis/dss/appconn/operation/ref/ExchangisRefExecutionOperation.java @@ -0,0 +1,70 @@ +package com.webank.wedatasphere.exchangis.dss.appconn.operation.ref; + +import com.webank.wedatasphere.dss.standard.app.development.listener.common.AsyncExecutionRequestRef; +import com.webank.wedatasphere.dss.standard.app.development.operation.RefExecutionOperation; +import com.webank.wedatasphere.dss.standard.app.development.ref.ExecutionRequestRef; +import com.webank.wedatasphere.dss.standard.app.development.service.DevelopmentService; +import com.webank.wedatasphere.dss.standard.app.sso.builder.SSOUrlBuilderOperation; +import com.webank.wedatasphere.dss.standard.app.sso.request.SSORequestOperation; +import com.webank.wedatasphere.dss.standard.common.entity.ref.ResponseRef; +import com.webank.wedatasphere.dss.standard.common.exception.operation.ExternalOperationFailedException; +import com.webank.wedatasphere.exchangis.dss.appconn.constraints.Constraints; +import com.webank.wedatasphere.exchangis.dss.appconn.ref.ExchangisCommonResponseDef; +import com.webank.wedatasphere.exchangis.dss.appconn.request.action.ExchangisEntityPostAction; +import com.webank.wedatasphere.exchangis.dss.appconn.request.action.ExchangisPostAction; +import com.webank.wedatasphere.exchangis.dss.appconn.ref.AbstractExchangisResponseRef; +import com.webank.wedatasphere.exchangis.dss.appconn.response.result.ExchangisEntityRespResult; +import com.webank.wedatasphere.exchangis.dss.appconn.utils.AppConnUtils; +import org.apache.linkis.httpclient.request.HttpAction; +import org.apache.linkis.httpclient.response.HttpResult; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.Map; +import java.util.Objects; + +/** + * Ref execute operation + */ +public class ExchangisRefExecutionOperation extends AbstractExchangisRefOperation implements RefExecutionOperation { + + private final static Logger LOG = LoggerFactory.getLogger(ExchangisRefExecutionOperation.class); + + DevelopmentService developmentService; + + public ExchangisRefExecutionOperation(DevelopmentService service) { + this.developmentService = service; + setSSORequestService(service); + } + @Override + public ResponseRef execute(ExecutionRequestRef executionRequestRef) throws ExternalOperationFailedException { + AsyncExecutionRequestRef nodeRequestRef = (AsyncExecutionRequestRef) executionRequestRef; + Long id = AppConnUtils.resolveParam(nodeRequestRef.getJobContent(), Constraints.REF_JOB_ID, Long.class); + LOG.info("execute job request => id: {}, name: {}, user: {}, jobContent: {}", + id, nodeRequestRef.getName(), nodeRequestRef.getExecutionRequestRefContext().getUser(), + nodeRequestRef.getJobContent().toString()); + String url = requestURL("/job/execute/" + id); + ExchangisEntityRespResult.BasicMessageEntity> entity = requestToGetEntity(nodeRequestRef.getWorkspace(), nodeRequestRef, + (requestRef) -> { + // Build ref execution action + return new ExchangisEntityPostAction<>(null, + requestRef.getExecutionRequestRefContext().getUser()); + }, Map.class); + if (Objects.isNull(entity)){ + throw new ExternalOperationFailedException(31020, "The response entity cannot be empty", null); + } + ExchangisEntityRespResult httpResult = entity.getResult(); + LOG.info("execute job response => status: {}, response: {}", httpResult.getStatusCode(), httpResult.getResponseBody()); + return new ExchangisCommonResponseDef(httpResult); + } + + @Override + public void setDevelopmentService(DevelopmentService developmentService) { + this.developmentService = developmentService; + } + + @Override + protected Logger getLogger() { + return LOG; + } +} diff --git a/exchangis-plugins/exchangis-appconn/src/main/java/com/webank/wedatasphere/exchangis/dss/appconn/operation/ref/ExchangisRefQueryOperation.java b/exchangis-plugins/exchangis-appconn/src/main/java/com/webank/wedatasphere/exchangis/dss/appconn/operation/ref/ExchangisRefQueryOperation.java new file mode 100644 index 000000000..9049d6449 --- /dev/null +++ b/exchangis-plugins/exchangis-appconn/src/main/java/com/webank/wedatasphere/exchangis/dss/appconn/operation/ref/ExchangisRefQueryOperation.java @@ -0,0 +1,56 @@ +package com.webank.wedatasphere.exchangis.dss.appconn.operation.ref; + +import com.webank.wedatasphere.dss.common.utils.DSSCommonUtils; +import com.webank.wedatasphere.dss.standard.app.development.operation.RefQueryOperation; +import com.webank.wedatasphere.dss.standard.app.development.ref.OpenRequestRef; +import com.webank.wedatasphere.dss.standard.app.development.service.DevelopmentService; +import com.webank.wedatasphere.dss.standard.common.entity.ref.ResponseRef; +import com.webank.wedatasphere.dss.standard.common.exception.operation.ExternalOperationFailedException; +import com.webank.wedatasphere.exchangis.dss.appconn.constraints.Constraints; +import com.webank.wedatasphere.exchangis.dss.appconn.ref.ExchangisOpenRequestRef; +import com.webank.wedatasphere.exchangis.dss.appconn.ref.ExchangisOpenResponseRef; +import com.webank.wedatasphere.exchangis.dss.appconn.utils.AppConnUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.HashMap; +import java.util.Map; + +/** + * Ref query operation + */ +public class ExchangisRefQueryOperation extends AbstractExchangisRefOperation implements RefQueryOperation { + private final static Logger LOG = LoggerFactory.getLogger(ExchangisRefQueryOperation.class); + + private DevelopmentService developmentService; + + public ExchangisRefQueryOperation(DevelopmentService developmentService){ + this.developmentService = developmentService; + setSSORequestService(developmentService); + } + @Override + public ResponseRef query(OpenRequestRef openRequestRef) throws ExternalOperationFailedException { + // Note: dss will scan the AppConn package to new the ExchangisOpenRequestRef + ExchangisOpenRequestRef exchangisOpenRequestRef = (ExchangisOpenRequestRef) openRequestRef; + try { + Long id = AppConnUtils.resolveParam(exchangisOpenRequestRef.getJobContent(), Constraints.REF_JOB_ID, Long.class); + String jumpUrl = requestURL(Constraints.REF_JUMP_URL_FORMAT + "?id=" + id); + Map retMap = new HashMap<>(); + LOG.info("ExchangisOpenResponseRef jump url: {}", jumpUrl); + retMap.put("jumpUrl",jumpUrl); + return new ExchangisOpenResponseRef(DSSCommonUtils.COMMON_GSON.toJson(retMap),0); + } catch (Exception e) { + throw new ExternalOperationFailedException(31022, "Failed to parse jobContent ", e); + } + } + + @Override + public void setDevelopmentService(DevelopmentService developmentService) { + this.developmentService = developmentService; + } + + @Override + protected Logger getLogger() { + return LOG; + } +} diff --git a/exchangis-plugins/exchangis-appconn/src/main/java/com/webank/wedatasphere/exchangis/dss/appconn/operation/ref/ExchangisRefUpdateOperation.java b/exchangis-plugins/exchangis-appconn/src/main/java/com/webank/wedatasphere/exchangis/dss/appconn/operation/ref/ExchangisRefUpdateOperation.java new file mode 100644 index 000000000..e2ea49858 --- /dev/null +++ b/exchangis-plugins/exchangis-appconn/src/main/java/com/webank/wedatasphere/exchangis/dss/appconn/operation/ref/ExchangisRefUpdateOperation.java @@ -0,0 +1,80 @@ +package com.webank.wedatasphere.exchangis.dss.appconn.operation.ref; + +import com.webank.wedatasphere.dss.standard.app.development.operation.RefUpdateOperation; +import com.webank.wedatasphere.dss.standard.app.development.ref.NodeRequestRef; +import com.webank.wedatasphere.dss.standard.app.development.ref.UpdateRequestRef; +import com.webank.wedatasphere.dss.standard.app.development.service.DevelopmentService; +import com.webank.wedatasphere.dss.standard.app.sso.builder.SSOUrlBuilderOperation; +import com.webank.wedatasphere.dss.standard.app.sso.request.SSORequestOperation; +import com.webank.wedatasphere.dss.standard.common.entity.ref.ResponseRef; +import com.webank.wedatasphere.dss.standard.common.exception.operation.ExternalOperationFailedException; +import com.webank.wedatasphere.exchangis.dss.appconn.constraints.Constraints; +import com.webank.wedatasphere.exchangis.dss.appconn.ref.ExchangisCommonResponseDef; +import com.webank.wedatasphere.exchangis.dss.appconn.request.action.ExchangisEntityPutAction; +import com.webank.wedatasphere.exchangis.dss.appconn.request.action.ExchangisPutAction; +import com.webank.wedatasphere.exchangis.dss.appconn.ref.AbstractExchangisResponseRef; +import com.webank.wedatasphere.exchangis.dss.appconn.request.entity.RefJobReqEntity; +import com.webank.wedatasphere.exchangis.dss.appconn.response.result.ExchangisEntityRespResult; +import com.webank.wedatasphere.exchangis.dss.appconn.utils.AppConnUtils; +import org.apache.linkis.httpclient.response.HttpResult; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.Map; +import java.util.Objects; + +/** + * Ref update operation + */ +public class ExchangisRefUpdateOperation extends AbstractExchangisRefOperation implements RefUpdateOperation { + private final static Logger LOG = LoggerFactory.getLogger(ExchangisRefUpdateOperation.class); + + DevelopmentService developmentService; + + public ExchangisRefUpdateOperation(DevelopmentService developmentService){ + this.developmentService = developmentService; + setSSORequestService(developmentService); + } + + @Override + public ResponseRef updateRef(UpdateRequestRef updateRequestRef) throws ExternalOperationFailedException { + NodeRequestRef nodeRequestRef = (NodeRequestRef) updateRequestRef; + ResponseRef responseRef = null; + if(Constraints.NODE_TYPE_SQOOP.equalsIgnoreCase(nodeRequestRef.getNodeType())){ + responseRef = updateOffLineRequest(nodeRequestRef, Constraints.ENGINE_TYPE_SQOOP_NAME); + }else if(Constraints.NODE_TYPE_DATAX.equalsIgnoreCase(nodeRequestRef.getNodeType())){ + responseRef = updateOffLineRequest(nodeRequestRef, Constraints.NODE_TYPE_DATAX); + } + return responseRef; + } + + private ResponseRef updateOffLineRequest(NodeRequestRef nodeRequestRef, String engineType) throws ExternalOperationFailedException{ + Long id = AppConnUtils.resolveParam(nodeRequestRef.getJobContent(), Constraints.REF_JOB_ID, Long.class); + LOG.info("update {} job request => id: {}, jobContent:{}", id, engineType, nodeRequestRef.getJobContent()); + String url = requestURL("/job/" + id); + ExchangisEntityRespResult.BasicMessageEntity> entity = requestToGetEntity(url, nodeRequestRef.getWorkspace(), nodeRequestRef, + (requestRef) -> { + // Build ref update action + RefJobReqEntity jobReqEntity = getRefJobReqEntity(requestRef, engineType); + jobReqEntity.setId(id); + return new ExchangisEntityPutAction<>(jobReqEntity, requestRef.getUserName()); + }, Map.class); + if (Objects.isNull(entity)){ + throw new ExternalOperationFailedException(31020, "The response entity cannot be empty", null); + } + ExchangisEntityRespResult httpResult = entity.getResult(); + LOG.info("update {} job response => id: {}, status: {}, response: {}", engineType, id, httpResult.getStatusCode(), + httpResult.getResponseBody()); + return new ExchangisCommonResponseDef(httpResult); + } + + @Override + public void setDevelopmentService(DevelopmentService developmentService) { + this.developmentService = developmentService; + } + + @Override + protected Logger getLogger() { + return LOG; + } +} diff --git a/exchangis-plugins/exchangis-appconn/src/main/java/com/webank/wedatasphere/exchangis/dss/appconn/ref/AbstractExchangisResponseRef.java b/exchangis-plugins/exchangis-appconn/src/main/java/com/webank/wedatasphere/exchangis/dss/appconn/ref/AbstractExchangisResponseRef.java new file mode 100644 index 000000000..3c2da6031 --- /dev/null +++ b/exchangis-plugins/exchangis-appconn/src/main/java/com/webank/wedatasphere/exchangis/dss/appconn/ref/AbstractExchangisResponseRef.java @@ -0,0 +1,42 @@ +package com.webank.wedatasphere.exchangis.dss.appconn.ref; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.webank.wedatasphere.dss.standard.common.entity.ref.AbstractResponseRef; +import com.webank.wedatasphere.exchangis.dss.appconn.response.result.ExchangisEntityRespResult; +import org.apache.linkis.server.BDPJettyServerHelper; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.Collections; +import java.util.Map; + +/** + * Abstract response ref + */ +public abstract class AbstractExchangisResponseRef extends AbstractResponseRef { + private static final Logger LOG = LoggerFactory.getLogger(AbstractExchangisResponseRef.class); + protected AbstractExchangisResponseRef(String responseBody, int status) { + super(responseBody, status); + } + + public AbstractExchangisResponseRef(ExchangisEntityRespResult result){ + super(result.getResponseBody(), result.getStatusCode()); + } + @Override + @SuppressWarnings("unchecked") + public Map toMap() { + try { + responseMap = BDPJettyServerHelper.jacksonJson().readValue(responseBody, Map.class); + } catch (JsonProcessingException e) { + LOG.warn("Fail to convert the response body {} to map", responseBody); + return Collections.emptyMap(); + } + return responseMap; + } + + @Override + public String getErrorMsg() { + return errorMsg; + } + +} diff --git a/exchangis-plugins/exchangis-appconn/src/main/java/com/webank/wedatasphere/exchangis/dss/appconn/ref/ExchangisCommonResponseDef.java b/exchangis-plugins/exchangis-appconn/src/main/java/com/webank/wedatasphere/exchangis/dss/appconn/ref/ExchangisCommonResponseDef.java new file mode 100644 index 000000000..120d0d860 --- /dev/null +++ b/exchangis-plugins/exchangis-appconn/src/main/java/com/webank/wedatasphere/exchangis/dss/appconn/ref/ExchangisCommonResponseDef.java @@ -0,0 +1,17 @@ +package com.webank.wedatasphere.exchangis.dss.appconn.ref; + +import com.webank.wedatasphere.exchangis.dss.appconn.response.result.ExchangisEntityRespResult; + +/** + * Common response ref + */ +public class ExchangisCommonResponseDef extends AbstractExchangisResponseRef{ + + protected ExchangisCommonResponseDef(String responseBody, int status) { + super(responseBody, status); + } + + public ExchangisCommonResponseDef(ExchangisEntityRespResult result) { + super(result); + } +} diff --git a/exchangis-plugins/exchangis-appconn/src/main/java/com/webank/wedatasphere/exchangis/appconn/ref/ExchangisCompletedExecutionResponseRef.java b/exchangis-plugins/exchangis-appconn/src/main/java/com/webank/wedatasphere/exchangis/dss/appconn/ref/ExchangisCompletedExecutionResponseRef.java similarity index 87% rename from exchangis-plugins/exchangis-appconn/src/main/java/com/webank/wedatasphere/exchangis/appconn/ref/ExchangisCompletedExecutionResponseRef.java rename to exchangis-plugins/exchangis-appconn/src/main/java/com/webank/wedatasphere/exchangis/dss/appconn/ref/ExchangisCompletedExecutionResponseRef.java index 3b783dd39..07ab8de79 100644 --- a/exchangis-plugins/exchangis-appconn/src/main/java/com/webank/wedatasphere/exchangis/appconn/ref/ExchangisCompletedExecutionResponseRef.java +++ b/exchangis-plugins/exchangis-appconn/src/main/java/com/webank/wedatasphere/exchangis/dss/appconn/ref/ExchangisCompletedExecutionResponseRef.java @@ -1,4 +1,4 @@ -package com.webank.wedatasphere.exchangis.appconn.ref; +package com.webank.wedatasphere.exchangis.dss.appconn.ref; import com.webank.wedatasphere.dss.standard.app.development.listener.common.CompletedExecutionResponseRef; diff --git a/exchangis-plugins/exchangis-appconn/src/main/java/com/webank/wedatasphere/exchangis/appconn/ref/ExchangisOpenRequestRef.java b/exchangis-plugins/exchangis-appconn/src/main/java/com/webank/wedatasphere/exchangis/dss/appconn/ref/ExchangisOpenRequestRef.java similarity index 78% rename from exchangis-plugins/exchangis-appconn/src/main/java/com/webank/wedatasphere/exchangis/appconn/ref/ExchangisOpenRequestRef.java rename to exchangis-plugins/exchangis-appconn/src/main/java/com/webank/wedatasphere/exchangis/dss/appconn/ref/ExchangisOpenRequestRef.java index 865a47cda..31916f9d6 100644 --- a/exchangis-plugins/exchangis-appconn/src/main/java/com/webank/wedatasphere/exchangis/appconn/ref/ExchangisOpenRequestRef.java +++ b/exchangis-plugins/exchangis-appconn/src/main/java/com/webank/wedatasphere/exchangis/dss/appconn/ref/ExchangisOpenRequestRef.java @@ -1,24 +1,28 @@ -package com.webank.wedatasphere.exchangis.appconn.ref; +package com.webank.wedatasphere.exchangis.dss.appconn.ref; import com.webank.wedatasphere.dss.standard.app.development.ref.OpenRequestRef; import com.webank.wedatasphere.dss.standard.app.development.ref.impl.CommonRequestRefImpl; import java.util.Map; +/** + * Open request ref + */ public class ExchangisOpenRequestRef extends CommonRequestRefImpl implements OpenRequestRef { - @Override + @SuppressWarnings("unchecked") public String getName() { return ((Map)this.getParameters().get("params")).get("title").toString(); } @Override + @SuppressWarnings("unchecked") public String getType() { return ((Map)this.getParameters().get("node")).get("nodeType").toString(); } + @SuppressWarnings("unchecked") public Map getJobContent() { return ((Map)this.getParameters().get("params")); } - } diff --git a/exchangis-plugins/exchangis-appconn/src/main/java/com/webank/wedatasphere/exchangis/appconn/ref/ExchangisOpenResponseRef.java b/exchangis-plugins/exchangis-appconn/src/main/java/com/webank/wedatasphere/exchangis/dss/appconn/ref/ExchangisOpenResponseRef.java similarity index 75% rename from exchangis-plugins/exchangis-appconn/src/main/java/com/webank/wedatasphere/exchangis/appconn/ref/ExchangisOpenResponseRef.java rename to exchangis-plugins/exchangis-appconn/src/main/java/com/webank/wedatasphere/exchangis/dss/appconn/ref/ExchangisOpenResponseRef.java index 37bb75f35..7e23b682d 100644 --- a/exchangis-plugins/exchangis-appconn/src/main/java/com/webank/wedatasphere/exchangis/appconn/ref/ExchangisOpenResponseRef.java +++ b/exchangis-plugins/exchangis-appconn/src/main/java/com/webank/wedatasphere/exchangis/dss/appconn/ref/ExchangisOpenResponseRef.java @@ -1,7 +1,10 @@ -package com.webank.wedatasphere.exchangis.appconn.ref; +package com.webank.wedatasphere.exchangis.dss.appconn.ref; import com.webank.wedatasphere.dss.standard.app.development.ref.CommonResponseRef; +/** + * Ref open response + */ public class ExchangisOpenResponseRef extends CommonResponseRef { public ExchangisOpenResponseRef(String responseBody, int status) { super(responseBody, status); diff --git a/exchangis-plugins/exchangis-appconn/src/main/java/com/webank/wedatasphere/exchangis/appconn/ref/ExchangisProjectResponseRef.java b/exchangis-plugins/exchangis-appconn/src/main/java/com/webank/wedatasphere/exchangis/dss/appconn/ref/ExchangisProjectResponseRef.java similarity index 56% rename from exchangis-plugins/exchangis-appconn/src/main/java/com/webank/wedatasphere/exchangis/appconn/ref/ExchangisProjectResponseRef.java rename to exchangis-plugins/exchangis-appconn/src/main/java/com/webank/wedatasphere/exchangis/dss/appconn/ref/ExchangisProjectResponseRef.java index fae785c4d..19336b9ba 100644 --- a/exchangis-plugins/exchangis-appconn/src/main/java/com/webank/wedatasphere/exchangis/appconn/ref/ExchangisProjectResponseRef.java +++ b/exchangis-plugins/exchangis-appconn/src/main/java/com/webank/wedatasphere/exchangis/dss/appconn/ref/ExchangisProjectResponseRef.java @@ -1,24 +1,26 @@ -package com.webank.wedatasphere.exchangis.appconn.ref; +package com.webank.wedatasphere.exchangis.dss.appconn.ref; import com.google.common.collect.Maps; import com.webank.wedatasphere.dss.standard.app.structure.project.ProjectResponseRef; import com.webank.wedatasphere.dss.standard.common.desc.AppInstance; -import com.webank.wedatasphere.dss.standard.common.entity.ref.AbstractResponseRef; -import org.apache.linkis.server.BDPJettyServerHelper; +import com.webank.wedatasphere.exchangis.dss.appconn.response.result.ExchangisEntityRespResult; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.util.Map; -public class ExchangisProjectResponseRef extends AbstractResponseRef implements ProjectResponseRef { - private static final Logger LOGGER = LoggerFactory.getLogger(ExchangisProjectResponseRef.class); +/** + * Response ref + */ +public class ExchangisProjectResponseRef extends AbstractExchangisResponseRef implements ProjectResponseRef { + private static final Logger LOG = LoggerFactory.getLogger(ExchangisProjectResponseRef.class); private Long projectRefId; private AppInstance appInstance; - private String errorMsg; - public ExchangisProjectResponseRef(String responseBody, int status) throws Exception { - super(responseBody, status); - responseMap = BDPJettyServerHelper.jacksonJson().readValue(responseBody, Map.class); + public ExchangisProjectResponseRef(ExchangisEntityRespResult result, + Long projectId){ + super(result.getResponseBody(), result.getStatusCode()); + this.projectRefId = projectId; } @Override @@ -33,15 +35,6 @@ public Map getProjectRefIds() { return projectRefIdsMap; } - @Override - public Map toMap() { - return responseMap; - } - - @Override - public String getErrorMsg() { - return errorMsg; - } public void setProjectRefId(Long projectRefId) { this.projectRefId = projectRefId; diff --git a/exchangis-plugins/exchangis-appconn/src/main/java/com/webank/wedatasphere/exchangis/appconn/model/ExchangisDeleteAction.java b/exchangis-plugins/exchangis-appconn/src/main/java/com/webank/wedatasphere/exchangis/dss/appconn/request/action/ExchangisDeleteAction.java similarity index 63% rename from exchangis-plugins/exchangis-appconn/src/main/java/com/webank/wedatasphere/exchangis/appconn/model/ExchangisDeleteAction.java rename to exchangis-plugins/exchangis-appconn/src/main/java/com/webank/wedatasphere/exchangis/dss/appconn/request/action/ExchangisDeleteAction.java index 820c547a8..8688d84fd 100644 --- a/exchangis-plugins/exchangis-appconn/src/main/java/com/webank/wedatasphere/exchangis/appconn/model/ExchangisDeleteAction.java +++ b/exchangis-plugins/exchangis-appconn/src/main/java/com/webank/wedatasphere/exchangis/dss/appconn/request/action/ExchangisDeleteAction.java @@ -1,19 +1,34 @@ -package com.webank.wedatasphere.exchangis.appconn.model; +package com.webank.wedatasphere.exchangis.dss.appconn.request.action; import org.apache.linkis.httpclient.request.DeleteAction; import org.apache.linkis.httpclient.request.UserAction; -public class ExchangisDeleteAction extends DeleteAction implements UserAction { +/** + * Delete action + */ +public class ExchangisDeleteAction extends DeleteAction implements HttpExtAction, UserAction { + /** + * URL + */ String url; + String user; + public ExchangisDeleteAction(){ + + } + + public ExchangisDeleteAction(String user){ + this.user = user; + } @Override public String getURL() { return url; } + @Override public void setUrl(String url) { this.url = url; } diff --git a/exchangis-plugins/exchangis-appconn/src/main/java/com/webank/wedatasphere/exchangis/dss/appconn/request/action/ExchangisEntityPostAction.java b/exchangis-plugins/exchangis-appconn/src/main/java/com/webank/wedatasphere/exchangis/dss/appconn/request/action/ExchangisEntityPostAction.java new file mode 100644 index 000000000..36cd12805 --- /dev/null +++ b/exchangis-plugins/exchangis-appconn/src/main/java/com/webank/wedatasphere/exchangis/dss/appconn/request/action/ExchangisEntityPostAction.java @@ -0,0 +1,87 @@ +package com.webank.wedatasphere.exchangis.dss.appconn.request.action; + +import com.fasterxml.jackson.core.JsonProcessingException; +import org.apache.linkis.httpclient.request.POSTAction; +import org.apache.linkis.httpclient.request.UserAction; +import org.apache.linkis.server.BDPJettyServerHelper; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.Objects; + +/** + * Entity post action + * @param + */ +public class ExchangisEntityPostAction extends POSTAction implements HttpExtAction, UserAction { + + private static final Logger LOG = LoggerFactory.getLogger(ExchangisEntityPostAction.class); + + /** + * URL + */ + private String url; + + private String user; + /** + * Entity to post request + */ + private T postEntity; + + public ExchangisEntityPostAction(){ + + } + + public ExchangisEntityPostAction(T postEntity){ + this.postEntity = postEntity; + } + + public ExchangisEntityPostAction(T postEntity, String user){ + this.postEntity = postEntity; + this.user = user; + } + + @Override + public String getURL() { + return url; + } + + @Override + public String getRequestPayload() { + String requestPayLoad = ""; + try { + if (Objects.nonNull(postEntity)) { + requestPayLoad = BDPJettyServerHelper.jacksonJson().writeValueAsString(postEntity); + } else { + requestPayLoad = BDPJettyServerHelper.jacksonJson().writeValueAsString(getRequestPayloads()); + } + }catch (JsonProcessingException e) { + LOG.error("Failed to covert entity/request payload to a string", e); + } + return requestPayLoad; + } + + @Override + public void setUser(String user) { + this.user = user; + } + + @Override + public String getUser() { + return this.user; + } + + @Override + public void setUrl(String url) { + this.url = url; + } + + public T getPostEntity() { + return postEntity; + } + + public void setPostEntity(T postEntity) { + this.postEntity = postEntity; + } + +} diff --git a/exchangis-plugins/exchangis-appconn/src/main/java/com/webank/wedatasphere/exchangis/dss/appconn/request/action/ExchangisEntityPutAction.java b/exchangis-plugins/exchangis-appconn/src/main/java/com/webank/wedatasphere/exchangis/dss/appconn/request/action/ExchangisEntityPutAction.java new file mode 100644 index 000000000..d8c1f9c8f --- /dev/null +++ b/exchangis-plugins/exchangis-appconn/src/main/java/com/webank/wedatasphere/exchangis/dss/appconn/request/action/ExchangisEntityPutAction.java @@ -0,0 +1,50 @@ +package com.webank.wedatasphere.exchangis.dss.appconn.request.action; + +import org.apache.linkis.httpclient.request.PutAction; +import org.apache.linkis.httpclient.request.UserAction; + +/** + * Actual contains a post action + * @param + */ +public class ExchangisEntityPutAction extends PutAction implements HttpExtAction, UserAction { + /** + * Inner action + */ + private ExchangisEntityPostAction postAction; + + public ExchangisEntityPutAction(){ + postAction = new ExchangisEntityPostAction<>(); + } + + public ExchangisEntityPutAction(T entity){ + postAction = new ExchangisEntityPostAction<>(entity); + } + public ExchangisEntityPutAction(T entity, String user){ + postAction = new ExchangisEntityPostAction<>(entity, user); + } + @Override + public String getURL() { + return postAction.getURL(); + } + + @Override + public String getRequestPayload() { + return postAction.getRequestPayload(); + } + + @Override + public void setUser(String user) { + postAction.setUser(user); + } + + @Override + public String getUser() { + return postAction.getUser(); + } + + @Override + public void setUrl(String url) { + postAction.setUrl(url); + } +} diff --git a/exchangis-plugins/exchangis-appconn/src/main/java/com/webank/wedatasphere/exchangis/appconn/model/ExchangisGetAction.java b/exchangis-plugins/exchangis-appconn/src/main/java/com/webank/wedatasphere/exchangis/dss/appconn/request/action/ExchangisGetAction.java similarity index 88% rename from exchangis-plugins/exchangis-appconn/src/main/java/com/webank/wedatasphere/exchangis/appconn/model/ExchangisGetAction.java rename to exchangis-plugins/exchangis-appconn/src/main/java/com/webank/wedatasphere/exchangis/dss/appconn/request/action/ExchangisGetAction.java index 5d4eb8fc1..b6f4c5420 100644 --- a/exchangis-plugins/exchangis-appconn/src/main/java/com/webank/wedatasphere/exchangis/appconn/model/ExchangisGetAction.java +++ b/exchangis-plugins/exchangis-appconn/src/main/java/com/webank/wedatasphere/exchangis/dss/appconn/request/action/ExchangisGetAction.java @@ -1,4 +1,4 @@ -package com.webank.wedatasphere.exchangis.appconn.model; +package com.webank.wedatasphere.exchangis.dss.appconn.request.action; import org.apache.linkis.httpclient.request.GetAction; diff --git a/exchangis-plugins/exchangis-appconn/src/main/java/com/webank/wedatasphere/exchangis/appconn/model/ExchangisPostAction.java b/exchangis-plugins/exchangis-appconn/src/main/java/com/webank/wedatasphere/exchangis/dss/appconn/request/action/ExchangisPostAction.java similarity index 94% rename from exchangis-plugins/exchangis-appconn/src/main/java/com/webank/wedatasphere/exchangis/appconn/model/ExchangisPostAction.java rename to exchangis-plugins/exchangis-appconn/src/main/java/com/webank/wedatasphere/exchangis/dss/appconn/request/action/ExchangisPostAction.java index f9ff3f882..326248719 100644 --- a/exchangis-plugins/exchangis-appconn/src/main/java/com/webank/wedatasphere/exchangis/appconn/model/ExchangisPostAction.java +++ b/exchangis-plugins/exchangis-appconn/src/main/java/com/webank/wedatasphere/exchangis/dss/appconn/request/action/ExchangisPostAction.java @@ -1,4 +1,4 @@ -package com.webank.wedatasphere.exchangis.appconn.model; +package com.webank.wedatasphere.exchangis.dss.appconn.request.action; import com.fasterxml.jackson.core.JsonProcessingException; import org.apache.linkis.httpclient.request.POSTAction; diff --git a/exchangis-plugins/exchangis-appconn/src/main/java/com/webank/wedatasphere/exchangis/appconn/model/ExchangisPutAction.java b/exchangis-plugins/exchangis-appconn/src/main/java/com/webank/wedatasphere/exchangis/dss/appconn/request/action/ExchangisPutAction.java similarity index 94% rename from exchangis-plugins/exchangis-appconn/src/main/java/com/webank/wedatasphere/exchangis/appconn/model/ExchangisPutAction.java rename to exchangis-plugins/exchangis-appconn/src/main/java/com/webank/wedatasphere/exchangis/dss/appconn/request/action/ExchangisPutAction.java index adf19ab57..94d956886 100644 --- a/exchangis-plugins/exchangis-appconn/src/main/java/com/webank/wedatasphere/exchangis/appconn/model/ExchangisPutAction.java +++ b/exchangis-plugins/exchangis-appconn/src/main/java/com/webank/wedatasphere/exchangis/dss/appconn/request/action/ExchangisPutAction.java @@ -1,4 +1,4 @@ -package com.webank.wedatasphere.exchangis.appconn.model; +package com.webank.wedatasphere.exchangis.dss.appconn.request.action; import com.fasterxml.jackson.core.JsonProcessingException; import org.apache.linkis.httpclient.request.PutAction; diff --git a/exchangis-plugins/exchangis-appconn/src/main/java/com/webank/wedatasphere/exchangis/dss/appconn/request/action/HttpExtAction.java b/exchangis-plugins/exchangis-appconn/src/main/java/com/webank/wedatasphere/exchangis/dss/appconn/request/action/HttpExtAction.java new file mode 100644 index 000000000..2eb2cb0d9 --- /dev/null +++ b/exchangis-plugins/exchangis-appconn/src/main/java/com/webank/wedatasphere/exchangis/dss/appconn/request/action/HttpExtAction.java @@ -0,0 +1,15 @@ +package com.webank.wedatasphere.exchangis.dss.appconn.request.action; + +import org.apache.linkis.httpclient.request.HttpAction; + +/** + * Extension http action + */ +public interface HttpExtAction extends HttpAction { + + /** + * Set url + * @param url url path + */ + void setUrl(String url); +} diff --git a/exchangis-plugins/exchangis-appconn/src/main/java/com/webank/wedatasphere/exchangis/dss/appconn/request/entity/ProjectReqEntity.java b/exchangis-plugins/exchangis-appconn/src/main/java/com/webank/wedatasphere/exchangis/dss/appconn/request/entity/ProjectReqEntity.java new file mode 100644 index 000000000..ef7647bfe --- /dev/null +++ b/exchangis-plugins/exchangis-appconn/src/main/java/com/webank/wedatasphere/exchangis/dss/appconn/request/entity/ProjectReqEntity.java @@ -0,0 +1,112 @@ +package com.webank.wedatasphere.exchangis.dss.appconn.request.entity; + +import com.webank.wedatasphere.exchangis.dss.appconn.constraints.Constraints; + +import java.util.HashMap; +import java.util.Map; + +/** + * Request entity of project + */ +public class ProjectReqEntity { + private String projectName; + + private String description; + + /** + * Request domain + */ + private String domain = Constraints.DOMAIN_NAME; + /** + * Information from the dss + */ + private Map source = new HashMap<>(); + + /** + * User has the edit permission + */ + private String editUsers; + + /** + * User has the view permission + */ + private String viewUsers; + + /** + * User has the execute permission + */ + private String execUsers; + + /** + * Tags + */ + private String tags; + + public ProjectReqEntity(){ + + } + + public ProjectReqEntity(String owner, String projectName, String description, Map source){ + this.projectName = projectName; + this.description = description; + this.source = source; + setEditUsers(owner); + setViewUsers(owner); + setExecUsers(owner); + } + public String getProjectName() { + return projectName; + } + + public void setProjectName(String projectName) { + this.projectName = projectName; + } + + public String getDescription() { + return description; + } + + public void setDescription(String description) { + this.description = description; + } + + public Map getSource() { + return source; + } + + public void setSource(Map source) { + this.source = source; + } + + public String getEditUsers() { + return editUsers; + } + + public void setEditUsers(String editUsers) { + this.editUsers = editUsers; + } + + public String getViewUsers() { + return viewUsers; + } + + public void setViewUsers(String viewUsers) { + this.viewUsers = viewUsers; + } + + public String getExecUsers() { + return execUsers; + } + + public void setExecUsers(String execUsers) { + this.execUsers = execUsers; + } + + public String getTags() { + return tags; + } + + public void setTags(String tags) { + this.tags = tags; + } +} diff --git a/exchangis-plugins/exchangis-appconn/src/main/java/com/webank/wedatasphere/exchangis/dss/appconn/request/entity/RefJobReqEntity.java b/exchangis-plugins/exchangis-appconn/src/main/java/com/webank/wedatasphere/exchangis/dss/appconn/request/entity/RefJobReqEntity.java new file mode 100644 index 000000000..a2f7df877 --- /dev/null +++ b/exchangis-plugins/exchangis-appconn/src/main/java/com/webank/wedatasphere/exchangis/dss/appconn/request/entity/RefJobReqEntity.java @@ -0,0 +1,118 @@ +package com.webank.wedatasphere.exchangis.dss.appconn.request.entity; + +import java.util.HashMap; +import java.util.Map; + +/** + * Ref job entity + */ +public class RefJobReqEntity { + + private Long id; + /** + * Project id in thirty system + */ + private Long projectId; + + /** + * Job type + */ + private String jobType; + + /** + * Engine Type + */ + private String engineType; + + /** + * Job labels + */ + private String jobLabels; + + /** + * Job description + */ + private String jobDesc; + + /** + * Job name + */ + private String jobName; + + private Map source = new HashMap<>();; + public RefJobReqEntity(String jobName, String jobDesc, + String jobType, String engineType, + Long projectId){ + this.jobName = jobName; + this.jobDesc = jobDesc; + this.jobType = jobType; + this.engineType = engineType; + this.projectId = projectId; + } + public RefJobReqEntity(){ + + } + public Long getProjectId() { + return projectId; + } + + public void setProjectId(Long projectId) { + this.projectId = projectId; + } + + public String getJobType() { + return jobType; + } + + public void setJobType(String jobType) { + this.jobType = jobType; + } + + public String getEngineType() { + return engineType; + } + + public void setEngineType(String engineType) { + this.engineType = engineType; + } + + public String getJobLabels() { + return jobLabels; + } + + public void setJobLabels(String jobLabels) { + this.jobLabels = jobLabels; + } + + public String getJobDesc() { + return jobDesc; + } + + public void setJobDesc(String jobDesc) { + this.jobDesc = jobDesc; + } + + public String getJobName() { + return jobName; + } + + public void setJobName(String jobName) { + this.jobName = jobName; + } + + public Map getSource() { + return source; + } + + public void setSource(Map source) { + this.source = source; + } + + public Long getId() { + return id; + } + + public void setId(Long id) { + this.id = id; + } +} diff --git a/exchangis-plugins/exchangis-appconn/src/main/java/com/webank/wedatasphere/exchangis/dss/appconn/response/entity/ProjectRespEntity.java b/exchangis-plugins/exchangis-appconn/src/main/java/com/webank/wedatasphere/exchangis/dss/appconn/response/entity/ProjectRespEntity.java new file mode 100644 index 000000000..0a2c2f864 --- /dev/null +++ b/exchangis-plugins/exchangis-appconn/src/main/java/com/webank/wedatasphere/exchangis/dss/appconn/response/entity/ProjectRespEntity.java @@ -0,0 +1,4 @@ +package com.webank.wedatasphere.exchangis.dss.appconn.response.entity; + +public class ProjectRespEntity { +} diff --git a/exchangis-plugins/exchangis-appconn/src/main/java/com/webank/wedatasphere/exchangis/dss/appconn/response/result/ExchangisEntityRespResult.java b/exchangis-plugins/exchangis-appconn/src/main/java/com/webank/wedatasphere/exchangis/dss/appconn/response/result/ExchangisEntityRespResult.java new file mode 100644 index 000000000..fdbb23c73 --- /dev/null +++ b/exchangis-plugins/exchangis-appconn/src/main/java/com/webank/wedatasphere/exchangis/dss/appconn/response/result/ExchangisEntityRespResult.java @@ -0,0 +1,151 @@ +package com.webank.wedatasphere.exchangis.dss.appconn.response.result; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.webank.wedatasphere.dss.standard.common.exception.operation.ExternalOperationFailedException; +import com.webank.wedatasphere.exchangis.dss.appconn.utils.JsonExtension; +import org.apache.commons.lang.StringUtils; +import org.apache.linkis.httpclient.response.HttpResult; + +import java.util.Objects; + +/** + * Convert the response body of http result to entity + */ + +public class ExchangisEntityRespResult implements HttpResult { + + /** + * Result held inner + */ + private HttpResult result; + + public ExchangisEntityRespResult(HttpResult result) throws ExternalOperationFailedException { + this.result = result; + int statusCode = result.getStatusCode(); + if (statusCode != 200){ + throw new ExternalOperationFailedException(-1, "The response result has wrong status code: [" + + result.getStatusCode() + "], response body: [" + result.getResponseBody() + "]", null); + } + } + + @Override + public String getContentType() { + return result.getContentType(); + } + + @Override + public String getUri() { + return result.getUri(); + } + + @Override + public int getStatusCode() { + return result.getStatusCode(); + } + + @Override + public void set(String responseBody, int statusCode, String url, String contentType) { + this.set(responseBody, statusCode, url, contentType); + } + + @Override + public String getResponseBody() { + return this.result.getResponseBody(); + } + + /** + * Get the entity from the response body + * @return entity + */ + public BasicMessageEntity getEntity(Class mainClass, Class... parameters) throws ExternalOperationFailedException { + String responseBody = this.result.getResponseBody(); + ObjectMapper mapper = JsonExtension.getMapper(); + Class[] parametricClass = new Class[parameters.length + 1]; + parametricClass[0] = mainClass; + if (parameters.length > 0){ + System.arraycopy(parameters, 0, parametricClass, 1, parameters.length); + } + if (StringUtils.isNotBlank(responseBody)){ + try { + BasicMessageEntity messageEntity = mapper.readValue(responseBody, mapper.getTypeFactory().constructParametricType(BasicMessageEntity.class, parametricClass)); + if (messageEntity.getStatus() != 0){ + throw new ExternalOperationFailedException(-1, "The status in Response message entity is " + + "" + messageEntity.getStatus() + ", message: [" + messageEntity.getMessage() +"]", null); + } + messageEntity.result = this; + return messageEntity; + } catch (JsonProcessingException e) { + throw new ExternalOperationFailedException(3130, "Fail to convert the response body: [" + responseBody + + "] to message entity, entity: [" + mainClass.getSimpleName() + "]", e); + } + } + return null; + + } + + public T getEntityValue(Class mainClass, Class... parameters) throws ExternalOperationFailedException{ + BasicMessageEntity entity = getEntity(mainClass, parameters); + return Objects.nonNull(entity)? entity.getData() : null; + } + public static class BasicMessageEntity{ + /** + * Result status + */ + private Integer status = 0; + + /** + * Refer request method(uri) + */ + private String method; + + /** + * Error message + */ + private String message; + + /** + * Actual entity + */ + private T data; + + private ExchangisEntityRespResult result; + + public ExchangisEntityRespResult getResult() { + return result; + } + + public Integer getStatus() { + return status; + } + + public void setStatus(Integer status) { + this.status = status; + } + + public String getMethod() { + return method; + } + + public void setMethod(String method) { + this.method = method; + } + + public String getMessage() { + return message; + } + + public void setMessage(String message) { + this.message = message; + } + + public T getData() { + return data; + } + + public void setData(T data) { + this.data = data; + } + } + +} diff --git a/exchangis-plugins/exchangis-appconn/src/main/java/com/webank/wedatasphere/exchangis/dss/appconn/service/ExchangisProjectService.java b/exchangis-plugins/exchangis-appconn/src/main/java/com/webank/wedatasphere/exchangis/dss/appconn/service/ExchangisProjectService.java new file mode 100644 index 000000000..a26fa1ae9 --- /dev/null +++ b/exchangis-plugins/exchangis-appconn/src/main/java/com/webank/wedatasphere/exchangis/dss/appconn/service/ExchangisProjectService.java @@ -0,0 +1,40 @@ +package com.webank.wedatasphere.exchangis.dss.appconn.service; + +import com.webank.wedatasphere.dss.standard.app.structure.project.*; +import com.webank.wedatasphere.exchangis.dss.appconn.operation.project.ExchangisProjectCreationOperation; + +/** + * Project service implement + */ +public class ExchangisProjectService extends ProjectService { + + @Override + public boolean isCooperationSupported() { + return true; + } + + @Override + public boolean isProjectNameUnique() { + return false; + } + + @Override + protected ProjectCreationOperation createProjectCreationOperation() { + return new ExchangisProjectCreationOperation(this); + } + + @Override + protected ProjectUpdateOperation createProjectUpdateOperation() { + return null; + } + + @Override + protected ProjectDeletionOperation createProjectDeletionOperation() { + return null; + } + + @Override + protected ProjectUrlOperation createProjectUrlOperation() { + return null; + } +} diff --git a/exchangis-plugins/exchangis-appconn/src/main/java/com/webank/wedatasphere/exchangis/dss/appconn/service/ExchangisRefCRUDService.java b/exchangis-plugins/exchangis-appconn/src/main/java/com/webank/wedatasphere/exchangis/dss/appconn/service/ExchangisRefCRUDService.java new file mode 100644 index 000000000..b9587bab4 --- /dev/null +++ b/exchangis-plugins/exchangis-appconn/src/main/java/com/webank/wedatasphere/exchangis/dss/appconn/service/ExchangisRefCRUDService.java @@ -0,0 +1,43 @@ +package com.webank.wedatasphere.exchangis.dss.appconn.service; + +import com.webank.wedatasphere.dss.standard.app.development.operation.RefCopyOperation; +import com.webank.wedatasphere.dss.standard.app.development.operation.RefCreationOperation; +import com.webank.wedatasphere.dss.standard.app.development.operation.RefDeletionOperation; +import com.webank.wedatasphere.dss.standard.app.development.operation.RefUpdateOperation; +import com.webank.wedatasphere.dss.standard.app.development.ref.CopyRequestRef; +import com.webank.wedatasphere.dss.standard.app.development.ref.CreateRequestRef; +import com.webank.wedatasphere.dss.standard.app.development.ref.DeleteRequestRef; +import com.webank.wedatasphere.dss.standard.app.development.service.AbstractRefCRUDService; +import com.webank.wedatasphere.dss.standard.common.entity.ref.RequestRef; +import com.webank.wedatasphere.exchangis.dss.appconn.operation.ref.ExchangisRefCreationOperation; +import com.webank.wedatasphere.exchangis.dss.appconn.operation.ref.ExchangisRefDeletionOperation; +import com.webank.wedatasphere.exchangis.dss.appconn.operation.ref.ExchangisRefUpdateOperation; + +/** + * Ref CRUD service + */ +public class ExchangisRefCRUDService extends AbstractRefCRUDService { + + @Override + @SuppressWarnings("unchecked") + protected RefCreationOperation createRefCreationOperation() { + return (RefCreationOperation) new ExchangisRefCreationOperation(this); + } + + @Override + protected RefCopyOperation createRefCopyOperation() { + return null; + } + + @Override + @SuppressWarnings("unchecked") + protected RefUpdateOperation createRefUpdateOperation() { + return (RefUpdateOperation) new ExchangisRefUpdateOperation(this); + } + + @Override + @SuppressWarnings("unchecked") + protected RefDeletionOperation createRefDeletionOperation() { + return new ExchangisRefDeletionOperation(this); + } +} diff --git a/exchangis-plugins/exchangis-appconn/src/main/java/com/webank/wedatasphere/exchangis/dss/appconn/service/ExchangisRefExecutionService.java b/exchangis-plugins/exchangis-appconn/src/main/java/com/webank/wedatasphere/exchangis/dss/appconn/service/ExchangisRefExecutionService.java new file mode 100644 index 000000000..ed0aabd51 --- /dev/null +++ b/exchangis-plugins/exchangis-appconn/src/main/java/com/webank/wedatasphere/exchangis/dss/appconn/service/ExchangisRefExecutionService.java @@ -0,0 +1,13 @@ +package com.webank.wedatasphere.exchangis.dss.appconn.service; + +import com.webank.wedatasphere.dss.standard.app.development.operation.RefExecutionOperation; +import com.webank.wedatasphere.dss.standard.app.development.service.AbstractRefExecutionService; +import com.webank.wedatasphere.exchangis.dss.appconn.operation.ref.ExchangisRefExecutionOperation; + +public class ExchangisRefExecutionService extends AbstractRefExecutionService { + @Override + public RefExecutionOperation createRefExecutionOperation() { + ExchangisRefExecutionOperation exchangisExecutionOperation = new ExchangisRefExecutionOperation(this); + return exchangisExecutionOperation; + } +} diff --git a/exchangis-plugins/exchangis-appconn/src/main/java/com/webank/wedatasphere/exchangis/dss/appconn/service/ExchangisRefExportService.java b/exchangis-plugins/exchangis-appconn/src/main/java/com/webank/wedatasphere/exchangis/dss/appconn/service/ExchangisRefExportService.java new file mode 100644 index 000000000..54e084a8a --- /dev/null +++ b/exchangis-plugins/exchangis-appconn/src/main/java/com/webank/wedatasphere/exchangis/dss/appconn/service/ExchangisRefExportService.java @@ -0,0 +1,18 @@ +package com.webank.wedatasphere.exchangis.dss.appconn.service; + +import com.webank.wedatasphere.dss.standard.app.development.operation.RefExportOperation; +import com.webank.wedatasphere.dss.standard.app.development.service.AbstractRefExportService; +import com.webank.wedatasphere.dss.standard.common.entity.ref.RequestRef; +import com.webank.wedatasphere.exchangis.dss.appconn.operation.ref.ExchangisExportOperation; + +/** + * Ref export service + */ +public class ExchangisRefExportService extends AbstractRefExportService { + @Override + @SuppressWarnings("unchecked") + protected RefExportOperation createRefExportOperation() { + return (RefExportOperation) new ExchangisExportOperation(this); + } + +} diff --git a/exchangis-plugins/exchangis-appconn/src/main/java/com/webank/wedatasphere/exchangis/dss/appconn/service/ExchangisRefImportService.java b/exchangis-plugins/exchangis-appconn/src/main/java/com/webank/wedatasphere/exchangis/dss/appconn/service/ExchangisRefImportService.java new file mode 100644 index 000000000..12d968fc4 --- /dev/null +++ b/exchangis-plugins/exchangis-appconn/src/main/java/com/webank/wedatasphere/exchangis/dss/appconn/service/ExchangisRefImportService.java @@ -0,0 +1,19 @@ +package com.webank.wedatasphere.exchangis.dss.appconn.service; + + +import com.webank.wedatasphere.dss.standard.app.development.operation.RefImportOperation; +import com.webank.wedatasphere.dss.standard.app.development.service.AbstractRefImportService; +import com.webank.wedatasphere.dss.standard.common.entity.ref.RequestRef; +import com.webank.wedatasphere.exchangis.dss.appconn.operation.ref.ExchangisImportOperation; + +/** + * Ref import service + */ +public class ExchangisRefImportService extends AbstractRefImportService { + @Override + @SuppressWarnings("unchecked") + protected RefImportOperation createRefImportOperation() { + return (RefImportOperation) new ExchangisImportOperation(this); + } + +} diff --git a/exchangis-plugins/exchangis-appconn/src/main/java/com/webank/wedatasphere/exchangis/dss/appconn/service/ExchangisRefQueryService.java b/exchangis-plugins/exchangis-appconn/src/main/java/com/webank/wedatasphere/exchangis/dss/appconn/service/ExchangisRefQueryService.java new file mode 100644 index 000000000..123d93975 --- /dev/null +++ b/exchangis-plugins/exchangis-appconn/src/main/java/com/webank/wedatasphere/exchangis/dss/appconn/service/ExchangisRefQueryService.java @@ -0,0 +1,16 @@ +package com.webank.wedatasphere.exchangis.dss.appconn.service; + +import com.webank.wedatasphere.dss.standard.app.development.operation.RefQueryOperation; +import com.webank.wedatasphere.dss.standard.app.development.service.AbstractRefQueryService; +import com.webank.wedatasphere.exchangis.dss.appconn.operation.ref.ExchangisRefQueryOperation; + +/** + * Ref query service + */ +public class ExchangisRefQueryService extends AbstractRefQueryService { + @Override + @SuppressWarnings("rawtypes") + protected RefQueryOperation createRefQueryOperation() { + return new ExchangisRefQueryOperation(this); + } +} diff --git a/exchangis-plugins/exchangis-appconn/src/main/java/com/webank/wedatasphere/exchangis/dss/appconn/utils/AppConnUtils.java b/exchangis-plugins/exchangis-appconn/src/main/java/com/webank/wedatasphere/exchangis/dss/appconn/utils/AppConnUtils.java new file mode 100644 index 000000000..2755ede81 --- /dev/null +++ b/exchangis-plugins/exchangis-appconn/src/main/java/com/webank/wedatasphere/exchangis/dss/appconn/utils/AppConnUtils.java @@ -0,0 +1,50 @@ +package com.webank.wedatasphere.exchangis.dss.appconn.utils; + +import com.webank.wedatasphere.dss.common.label.DSSLabel; +import com.webank.wedatasphere.dss.standard.common.exception.operation.ExternalOperationFailedException; +import com.webank.wedatasphere.exchangis.dss.appconn.response.result.ExchangisEntityRespResult; +import org.apache.linkis.manager.label.entity.SerializableLabel; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.stream.Collectors; + +/** + * Appconn utils for exchangis + */ +public class AppConnUtils { + + private static final Logger LOG = LoggerFactory.getLogger(AppConnUtils.class); + /** + * Invoke the "getStringValue" method in label entity and then concat each one + * @param list label list + * @return serialized string value + */ + public static String serializeDssLabel(List list){ + String dssLabelStr = ""; + if(list != null && !list.isEmpty()){ + dssLabelStr = list.stream().map(SerializableLabel::getStringValue).collect(Collectors.joining(",")); + } + return dssLabelStr; + } + + @SuppressWarnings("unchecked") + public static T resolveParam(Map responseMap, String key, Class type){ + try { + ExchangisEntityRespResult.BasicMessageEntity entity = JsonExtension.convert(responseMap, ExchangisEntityRespResult.BasicMessageEntity.class, Object.class); + Object data = entity.getData(); + if (Objects.nonNull(data) && data instanceof Map){ + Map dataMap = (Map)data; + // TODO convert different type + return (T)dataMap.get(key); + } + } catch (ExternalOperationFailedException e) { + LOG.warn("Exception in resolving params: " + key, e); + } + return null; + } + +} diff --git a/exchangis-plugins/exchangis-appconn/src/main/java/com/webank/wedatasphere/exchangis/dss/appconn/utils/JsonExtension.java b/exchangis-plugins/exchangis-appconn/src/main/java/com/webank/wedatasphere/exchangis/dss/appconn/utils/JsonExtension.java new file mode 100644 index 000000000..0bde26625 --- /dev/null +++ b/exchangis-plugins/exchangis-appconn/src/main/java/com/webank/wedatasphere/exchangis/dss/appconn/utils/JsonExtension.java @@ -0,0 +1,36 @@ +package com.webank.wedatasphere.exchangis.dss.appconn.utils; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.webank.wedatasphere.dss.standard.common.exception.operation.ExternalOperationFailedException; + +/** + * Extends the function from the mapper "BDPJettyServerHelper.jacksonJson()" + */ +public class JsonExtension { + + /** + * Convert object using serialization and deserialization + * + * @param simpleObj simpleObj + * @param tClass type class + * @param T + * @return result + */ + @SuppressWarnings("unchecked") + public static T convert(Object simpleObj, Class tClass, Class... parameters) throws ExternalOperationFailedException{ + ObjectMapper mapper = getMapper(); + try { + if (parameters.length > 0) { + return mapper.convertValue(simpleObj, mapper.getTypeFactory().constructParametricType(tClass, parameters)); + } + return (T) mapper.convertValue(simpleObj, tClass); + } catch (Exception e) { + throw new ExternalOperationFailedException(3130, "Fail to process method 'convert(" + simpleObj + ": " + simpleObj.getClass().getSimpleName() + + ", " + tClass.getSimpleName() + ": "+ Class.class + ", ...: " + Class.class + ")", e); + } + } + + public static ObjectMapper getMapper(){ + return new ObjectMapper(); + } +} diff --git a/exchangis-project/exchangis-project-server/src/main/java/com/webank/wedatasphere/exchangis/project/server/vo/ExchangisProjectVo.java b/exchangis-project/exchangis-project-server/src/main/java/com/webank/wedatasphere/exchangis/project/server/vo/ExchangisProjectVo.java new file mode 100644 index 000000000..ef074cfbf --- /dev/null +++ b/exchangis-project/exchangis-project-server/src/main/java/com/webank/wedatasphere/exchangis/project/server/vo/ExchangisProjectVo.java @@ -0,0 +1,4 @@ +package com.webank.wedatasphere.exchangis.project.server.vo; + +public class ExchangisProjectVo { +}