diff --git a/dss-apps/dss-apiservice-server/pom.xml b/dss-apps/dss-apiservice-server/pom.xml new file mode 100644 index 0000000000..97f69346ba --- /dev/null +++ b/dss-apps/dss-apiservice-server/pom.xml @@ -0,0 +1,420 @@ + + + + + + dss + com.webank.wedatasphere.dss + 1.0.0 + ../../pom.xml + + 4.0.0 + + dss-apiservice-server + + + + UTF-8 + 2.16 + 2.22.2 + 2.22.2 + + + + + com.webank.wedatasphere.dss + dss-common + ${dss.version} + provided + + + com.webank.wedatasphere.linkis + linkis-module + ${linkis.version} + provided + + + asm + org.ow2.asm + + + hk2-api + org.glassfish.hk2 + + + jersey-common + org.glassfish.jersey.core + + + linkis-common + com.webank.wedatasphere.linkis + + + + + com.alibaba + fastjson + 1.2.70 + + + + org.glassfish.jersey.ext + jersey-bean-validation + ${jersey.version} + provided + + + javax.ws.rs-api + javax.ws.rs + + + hk2-locator + org.glassfish.hk2 + + + hk2-api + org.glassfish.hk2 + + + hibernate-validator + org.hibernate + + + jersey-server + org.glassfish.jersey.core + + + + + + com.webank.wedatasphere.linkis + linkis-bml-client + ${linkis.version} + + + commons-beanutils + commons-beanutils + + + linkis-common + com.webank.wedatasphere.linkis + + + json4s-jackson_2.11 + org.json4s + + + + + com.webank.wedatasphere.linkis + linkis-computation-client + ${linkis.version} + + + commons-beanutils + commons-beanutils + + + linkis-common + com.webank.wedatasphere.linkis + + + + + org.postgresql + postgresql + 42.2.12 + + + com.webank.wedatasphere.linkis + linkis-mybatis + ${linkis.version} + provided + + + com.webank.wedatasphere.linkis + linkis-storage + ${linkis.version} + provided + + + linkis-common + com.webank.wedatasphere.linkis + + + + + com.webank.wedatasphere.linkis + linkis-common + ${linkis.version} + provided + + + com.webank.wedatasphere.linkis + linkis-rpc + ${linkis.version} + provided + + + archaius-core + com.netflix.archaius + + + slf4j-api + org.slf4j + + + spring-cloud-starter + org.springframework.cloud + + + spring-web + org.springframework + + + + + com.zaxxer + HikariCP + 3.4.1 + + + slf4j-api + org.slf4j + + + + + + + + + + + hk2-api + org.glassfish.hk2 + 2.4.0-b34 + + + + + + + + + org.modelmapper + modelmapper + 0.7.5 + + + + io.jsonwebtoken + jjwt + 0.6.0 + + + jackson-databind + com.fasterxml.jackson.core + + + + + + org.hamcrest + hamcrest-all + 1.3 + test + + + + + com.h2database + h2 + 1.4.196 + test + + + + com.ninja-squad + DbSetup + 2.1.0 + test + + + org.springframework.boot + spring-boot-starter-test + ${spring.boot.version} + test + + + spring-boot-autoconfigure + org.springframework.boot + + + + + + junit + junit + 4.12 + test + + + org.junit.platform + junit-platform-launcher + 1.5.2 + test + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + xstream + com.thoughtworks.xstream + 1.4.11.1 + + + + + + + + maven-surefire-plugin + ${maven-surefire-plugin.version} + + + maven-failsafe-plugin + ${maven-failsafe-plugin.version} + + + org.apache.maven.plugins + maven-deploy-plugin + + + + net.alchim31.maven + scala-maven-plugin + + + org.apache.maven.plugins + maven-jar-plugin + + + org.apache.maven.plugins + maven-assembly-plugin + 2.3 + false + + + make-assembly + package + + single + + + + src/main/assembly/distribution.xml + + + + + + false + out + false + false + + src/main/assembly/distribution.xml + + + + + + + src/main/java + + **/*.xml + + + + + + + + + + + + + ${project.artifactId}-${project.version} + + diff --git a/dss-apps/dss-apiservice-server/src/main/assembly/distribution.xml b/dss-apps/dss-apiservice-server/src/main/assembly/distribution.xml new file mode 100644 index 0000000000..9882c22ece --- /dev/null +++ b/dss-apps/dss-apiservice-server/src/main/assembly/distribution.xml @@ -0,0 +1,44 @@ + + + + dss-apiService-server + + dir + + true + dss-apiservice-server + + + + + + lib + true + true + false + true + true + + + + + + + diff --git a/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/action/ApiServiceGetAction.scala b/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/action/ApiServiceGetAction.scala new file mode 100644 index 0000000000..62353e7225 --- /dev/null +++ b/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/action/ApiServiceGetAction.scala @@ -0,0 +1,29 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.apiservice.core.action + +import com.webank.wedatasphere.linkis.httpclient.request.GetAction +import com.webank.wedatasphere.linkis.ujes.client.request.UJESJobAction + +/** + * @author allenlliu + * @date 2020/09/14 10:32 AM + * @version 2.0.0 + */ +class ApiServiceGetAction extends GetAction with UJESJobAction { + override def suffixURLs: Array[String] = Array("dss","framework","workspace", "getWorkspaceIdByUserName") +} diff --git a/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/action/ResultSetDownloadAction.scala b/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/action/ResultSetDownloadAction.scala new file mode 100644 index 0000000000..25c9b75673 --- /dev/null +++ b/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/action/ResultSetDownloadAction.scala @@ -0,0 +1,38 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.apiservice.core.action + +import com.webank.wedatasphere.linkis.httpclient.request.{DownloadAction, GetAction} +import com.webank.wedatasphere.linkis.ujes.client.request.UJESJobAction + +import scala.tools.nsc.interpreter.InputStream + +/** + * @author allenlliu + * @version 2.0.0 + */ + +class ResultSetDownloadAction extends GetAction with DownloadAction with UJESJobAction { + + private var inputStream: InputStream = _ + + override def write(inputStream: InputStream): Unit = this.inputStream = inputStream + + def getInputStream: InputStream = inputStream + + override def suffixURLs: Array[String] = Array("filesystem", "resultsetToExcel") +} diff --git a/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/action/ResultWorkspaceIds.scala b/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/action/ResultWorkspaceIds.scala new file mode 100644 index 0000000000..8ae4fa95f8 --- /dev/null +++ b/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/action/ResultWorkspaceIds.scala @@ -0,0 +1,33 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.apiservice.core.action + +import com.webank.wedatasphere.linkis.httpclient.dws.annotation.DWSHttpMessageResult +import com.webank.wedatasphere.linkis.httpclient.dws.response.DWSResult + +/** + * @author allenlliu + * @date 2020/09/14 11:44 AM + * @version 2.0.0 + */ + +@DWSHttpMessageResult("/api/rest_j/v\\d+/dss/framework/workspace/getWorkspaceIdByUserName") +class ResultWorkspaceIds extends DWSResult{ + var userWorkspaceIds:String = _ + def getUserWorkspaceIds:String = this.userWorkspaceIds + def setUserWorkspaceIds(userWorkspaceIds:String):Unit = this.userWorkspaceIds = userWorkspaceIds +} \ No newline at end of file diff --git a/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/bo/ApiServiceJob.java b/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/bo/ApiServiceJob.java new file mode 100644 index 0000000000..95ce75af59 --- /dev/null +++ b/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/bo/ApiServiceJob.java @@ -0,0 +1,55 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.apiservice.core.bo; + +import com.webank.wedatasphere.linkis.ujes.client.response.JobExecuteResult; + +/** + * @author allenlliu + * @version 2.0.0 + * @date 2020/09/03 07:28 PM + */ +public class ApiServiceJob { + + private String submitUser; + private String proxyUser; + private JobExecuteResult jobExecuteResult; + public String getSubmitUser() { + return submitUser; + } + + public void setSubmitUser(String submitUser) { + this.submitUser = submitUser; + } + + + + public String getProxyUser() { + return proxyUser; + } + + public void setProxyUser(String proxyUser) { + this.proxyUser = proxyUser; + } + public JobExecuteResult getJobExecuteResult() { + return jobExecuteResult; + } + + public void setJobExecuteResult(JobExecuteResult jobExecuteResult) { + this.jobExecuteResult = jobExecuteResult; + } +} diff --git a/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/bo/ApiServiceQuery.java b/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/bo/ApiServiceQuery.java new file mode 100644 index 0000000000..ff5148bb46 --- /dev/null +++ b/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/bo/ApiServiceQuery.java @@ -0,0 +1,126 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.apiservice.core.bo; + + +/** + * query + * + * @author zhulixin + */ +public class ApiServiceQuery { + + /** + * 服务名称 + */ + private String name; + + /** + * 标签 + */ + private String tag; + + /** + * 状态 + */ + private Integer status; + + /** + * api创建者 + */ + private String creator; + + private int currentPage; + + private int pageSize; + + public int getWorkspaceId() { + return workspaceId; + } + + public void setWorkspaceId(int workspaceId) { + this.workspaceId = workspaceId; + } + + private int workspaceId; + + public String getUserName() { + return userName; + } + + public void setUserName(String userName) { + this.userName = userName; + } + + private String userName; + + public ApiServiceQuery(String userName,String name, String tag, Integer status, String creator) { + this.userName= userName; + this.name = name; + this.tag = tag; + this.status = status; + this.creator = creator; + } + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public String getTag() { + return tag; + } + + public void setTag(String tag) { + this.tag = tag; + } + + public Integer getStatus() { + return status; + } + + public void setStatus(Integer status) { + this.status = status; + } + + public String getCreator() { + return creator; + } + + public void setCreator(String creator) { + this.creator = creator; + } + + public int getCurrentPage() { + return currentPage; + } + + public void setCurrentPage(int currentPage) { + this.currentPage = currentPage; + } + + public int getPageSize() { + return pageSize; + } + + public void setPageSize(int pageSize) { + this.pageSize = pageSize; + } +} diff --git a/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/bo/ApiServiceToken.java b/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/bo/ApiServiceToken.java new file mode 100644 index 0000000000..c4b66f9c9c --- /dev/null +++ b/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/bo/ApiServiceToken.java @@ -0,0 +1,70 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.apiservice.core.bo; + + + +import java.util.Date; + +/** + * @author allenlliu + * @version 2.0.0 + * @date 2020/08/11 08:38 PM + */ + +public class ApiServiceToken { + + private String applyUser; + private Date applyTime; + private Long apiServiceId; + private String publisher; + + public String getPublisher() { + return publisher; + } + + public void setPublisher(String publisher) { + this.publisher = publisher; + } + + public Long getApiServiceId() { + return apiServiceId; + } + + public void setApiServiceId(Long apiServiceId) { + this.apiServiceId = apiServiceId; + } + + public Date getApplyTime() { + return applyTime; + } + + public void setApplyTime(Date applyTime) { + this.applyTime = applyTime; + } + + public String getApplyUser() { + return applyUser; + } + + public void setApplyUser(String applyUser) { + this.applyUser = applyUser; + } + + + +} diff --git a/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/bo/DirFileTree.java b/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/bo/DirFileTree.java new file mode 100644 index 0000000000..33b891234d --- /dev/null +++ b/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/bo/DirFileTree.java @@ -0,0 +1,83 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.apiservice.core.bo; + +import java.util.HashMap; +import java.util.List; + +/** + * @author allenlliu + * @version 2.0.0 + * @date 2020/09/03 05:29 PM + */ +public class DirFileTree { + + private String name; + private String path; + private HashMap properties; + private List children; + private Boolean isLeaf = false; + private String parentPath; + + public Boolean getIsLeaf() { + return isLeaf; + } + + public void setIsLeaf(Boolean isLeaf) { + this.isLeaf = isLeaf; + } + + public String getParentPath() { + return parentPath; + } + + public void setParentPath(String parentPath) { + this.parentPath = parentPath; + } + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public String getPath() { + return path; + } + + public void setPath(String path) { + this.path = path; + } + + public HashMap getProperties() { + return properties; + } + + public void setProperties(HashMap properties) { + this.properties = properties; + } + + public List getChildren() { + return children; + } + + public void setChildren(List children) { + this.children = children; + } +} \ No newline at end of file diff --git a/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/bo/LinkisExecuteResult.java b/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/bo/LinkisExecuteResult.java new file mode 100644 index 0000000000..910d4bc9f3 --- /dev/null +++ b/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/bo/LinkisExecuteResult.java @@ -0,0 +1,50 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.apiservice.core.bo; + +/** + * @author allenlliu + * @version 2.0.0 + * @date 2020/08/13 11:39 AM + */ +public class LinkisExecuteResult { + + public LinkisExecuteResult(String taskId,String execId){ + this.taskId = taskId; + this.execId = execId; + } + + public String getTaskId() { + return taskId; + } + + public void setTaskId(String taskId) { + this.taskId = taskId; + } + + private String taskId; + + public String getExecId() { + return execId; + } + + public void setExecId(String execId) { + this.execId = execId; + } + + private String execId; +} diff --git a/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/bo/QueryRequest.java b/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/bo/QueryRequest.java new file mode 100644 index 0000000000..46acb97d90 --- /dev/null +++ b/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/bo/QueryRequest.java @@ -0,0 +1,51 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package com.webank.wedatasphere.dss.apiservice.core.bo; + +import java.util.Map; + +/** + * 接口调用request + * + * @author lidongzhang + */ +public class QueryRequest { + /** + * 模块名，也是登录用户名、密码 + */ + private String moduleName; + + /** + * 调用参数 + */ + private Map params; + + public Map getParams() { + return params; + } + + public void setParams(Map params) { + this.params = params; + } + + public String getModuleName() { + return moduleName; + } + + public void setModuleName(String moduleName) { + this.moduleName = moduleName; + } +} diff --git a/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/bo/TokenQuery.java b/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/bo/TokenQuery.java new file mode 100644 index 0000000000..6cede52470 --- /dev/null +++ b/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/bo/TokenQuery.java @@ -0,0 +1,117 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.apiservice.core.bo; + + +import java.util.Date; + +public class TokenQuery { + private Long apiId; + private String user; + private Integer status; + + private String startDate; + private String endDate; + + private int currentPage; + + private int pageSize; + + public String getCreator() { + return creator; + } + + public void setCreator(String creator) { + this.creator = creator; + } + + private String creator; + + + public TokenQuery(Long apiId, String user, Integer status) { + this.apiId = apiId; + this.user = user; + this.status = status; + } + + public TokenQuery(Long apiId, String user, Integer status, String startDate, String endDate) { + this.apiId = apiId; + this.user = user; + this.status = status; + this.startDate = startDate; + this.endDate = endDate; + } + + public TokenQuery() { + } + + public String getStartDate() { + return startDate; + } + + public void setStartDate(String startDate) { + this.startDate = startDate; + } + + public String getEndDate() { + return endDate; + } + + public void setEndDate(String endDate) { + this.endDate = endDate; + } + + public int getCurrentPage() { + return currentPage; + } + + public void setCurrentPage(int currentPage) { + this.currentPage = currentPage; + } + + public int getPageSize() { + return pageSize; + } + + public void setPageSize(int pageSize) { + this.pageSize = pageSize; + } + + public Long getApiId() { + return apiId; + } + + public void setApiId(Long apiId) { + this.apiId = apiId; + } + + public String getUser() { + return user; + } + + public void setUser(String user) { + this.user = user; + } + + public Integer getStatus() { + return status; + } + + public void setStatus(Integer status) { + this.status = status; + } +} diff --git a/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/bo/VariableString.java b/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/bo/VariableString.java new file mode 100644 index 0000000000..21dce95a6e --- /dev/null +++ b/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/bo/VariableString.java @@ -0,0 +1,37 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package com.webank.wedatasphere.dss.apiservice.core.bo; + +/** + * VariableString + * + * @author lidongzhang + */ +public class VariableString { + private String path; + + public VariableString(String unparsedPath) { + this.path = unparsedPath; + } + + public String getPath() { + return path; + } + + public void setPath(String path) { + this.path = path; + } +} diff --git a/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/config/ApiServiceConfiguration.java b/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/config/ApiServiceConfiguration.java new file mode 100644 index 0000000000..feae84b499 --- /dev/null +++ b/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/config/ApiServiceConfiguration.java @@ -0,0 +1,44 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.apiservice.core.config; + +import com.webank.wedatasphere.linkis.common.conf.CommonVars; + +/** + * @author allenlliu + * @version 2.0.0 + * @date 2020/08/12 03:46 PM + */ +public class ApiServiceConfiguration { + public final static CommonVars LINKIS_AUTHOR_USER_TOKEN = CommonVars.apply("wds.linkis.client.api.service.author.user.token","WS-AUTH"); + public final static CommonVars LINKIS_ADMIN_USER = CommonVars.apply("wds.linkis.client.api.service.adminuser","ws"); + + public final static CommonVars LINKIS_CONNECTION_TIMEOUT = CommonVars.apply("wds.linkis.flow.connection.timeout",30000); + public final static CommonVars LINKIS_API_VERSION = CommonVars.apply("wds.linkis.server.version","v1"); + + public final static CommonVars API_SERVICE_TOKEN_KEY = CommonVars.apply("wds.dss.api.service.token.key","ApiServiceToken"); + + + public final static CommonVars DSS_API_TOKEN_SECRET_ID = CommonVars.apply("wds.dss.api.service.secret", "DSSSECRETTEST001002"); + + public final static CommonVars LINKIS_JOB_REQUEST_STATUS_TIME = CommonVars.apply("wds.linkis.job.status.timeout",3000); + + public final static CommonVars LOG_ARRAY_LEN = CommonVars.apply("wds.linkis.log.array.len",4); + + public final static CommonVars RESULT_PRINT_SIZE = CommonVars.apply("wds.linkis.result.print.size",10); + +} diff --git a/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/constant/ApiCommonConstant.java b/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/constant/ApiCommonConstant.java new file mode 100644 index 0000000000..8c49f7a423 --- /dev/null +++ b/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/constant/ApiCommonConstant.java @@ -0,0 +1,27 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.apiservice.core.constant; + +/** + * @author allenlliu + * @version 2.0.0 + * @date 2020/08/20 11:47 AM + */ +public class ApiCommonConstant { + public static final String DEFAULT_APPROVAL_NO="0001"; + public static final Integer API_DELETE_STATUS=2; +} diff --git a/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/constant/ApprovalStatus.java b/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/constant/ApprovalStatus.java new file mode 100644 index 0000000000..102bf71d07 --- /dev/null +++ b/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/constant/ApprovalStatus.java @@ -0,0 +1,51 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.apiservice.core.constant; + +/** + * @author: jinyangrao + */ +public enum ApprovalStatus { + INIT("init", 0), + APPLYING("apply", 1), + SUCCESS("success", 2), + FAIELD("failed", 3); + + String key; + int value; + + ApprovalStatus(String key, int value) { + this.key = key; + this.value = value; + } + + public String getKey() { + return key; + } + + public void setKey(String key) { + this.key = key; + } + + public int getValue() { + return value; + } + + public void setValue(int value) { + this.value = value; + } +} \ No newline at end of file diff --git a/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/constant/ParamType.java b/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/constant/ParamType.java new file mode 100644 index 0000000000..13a7709180 --- /dev/null +++ b/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/constant/ParamType.java @@ -0,0 +1,28 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package com.webank.wedatasphere.dss.apiservice.core.constant; + +/** + * ParamType + * + * @author lidongzhang + */ +public class ParamType { + public static final String number = "2"; + + public static final String array = "4"; + +} diff --git a/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/constant/ParamTypeEnum.java b/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/constant/ParamTypeEnum.java new file mode 100644 index 0000000000..7f20fdbd73 --- /dev/null +++ b/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/constant/ParamTypeEnum.java @@ -0,0 +1,58 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package com.webank.wedatasphere.dss.apiservice.core.constant; + +/** + * @author chongchuanbing + */ +public enum ParamTypeEnum { + + STRING(1, "String"), + + NUMBER(2, "Number"), + + DATE(3, "Date"), + + ARRAY(4, "Array"); + + private Integer index; + private String name; + + ParamTypeEnum(Integer index, String name) { + this.index = index; + this.name = name; + } + + public static ParamTypeEnum getEnum(Integer index) { + if (index == null) { + return STRING; + } + for (ParamTypeEnum statusEnum : values()) { + if (statusEnum.getIndex().equals(index)) { + return statusEnum; + } + } + return STRING; + } + + public Integer getIndex() { + return this.index; + } + + public String getName() { + return this.name; + } +} diff --git a/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/constant/RequireEnum.java b/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/constant/RequireEnum.java new file mode 100644 index 0000000000..8d0cc3e993 --- /dev/null +++ b/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/constant/RequireEnum.java @@ -0,0 +1,54 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package com.webank.wedatasphere.dss.apiservice.core.constant; + +/** + * @author chongchuanbing + */ +public enum RequireEnum { + + YES(1, "是"), + + NO(0, "否"); + + private Integer index; + private String name; + + RequireEnum(Integer index, String name) { + this.index = index; + this.name = name; + } + + public static RequireEnum getEnum(Integer index) { + if (index == null) { + return NO; + } + for (RequireEnum statusEnum : values()) { + if (statusEnum.getIndex().equals(index)) { + return statusEnum; + } + } + return NO; + } + + public Integer getIndex() { + return this.index; + } + + public String getName() { + return this.name; + } +} diff --git a/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/constant/SQLMetadataInfoCheckStatus.java b/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/constant/SQLMetadataInfoCheckStatus.java new file mode 100644 index 0000000000..11c7320419 --- /dev/null +++ b/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/constant/SQLMetadataInfoCheckStatus.java @@ -0,0 +1,51 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.apiservice.core.constant; + +import com.webank.wedatasphere.dss.apiservice.core.datamap.DataMapStatus; + +/** + * @author: jinyangrao + */ +public enum SQLMetadataInfoCheckStatus { + LEGAL(0, "该SQL查询合法"), ILLEGALSQL(1,"不支持无库表信息的SQL查询"); + + int index; + String desc; + + SQLMetadataInfoCheckStatus(int index, String desc) { + this.index = index; + this.desc = desc; + } + + public String getDescByIndex(int index) { + for (SQLMetadataInfoCheckStatus item : values()) { + if (item.getIndex() == index) { + return item.getDesc(); + } + } + return null; + } + + public int getIndex() { + return index; + } + + public String getDesc() { + return desc; + } +} diff --git a/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/constant/SaveTokenEnum.java b/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/constant/SaveTokenEnum.java new file mode 100644 index 0000000000..057dfa5a2c --- /dev/null +++ b/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/constant/SaveTokenEnum.java @@ -0,0 +1,47 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.apiservice.core.constant; + +/** + * @author: jinyangrao + */ +public enum SaveTokenEnum { + SUCCESS(1, "success"), FAILED(2, "failed"), DUPLICATE(3, "duplicate"); + + int index; + String value; + SaveTokenEnum(int index, String value) { + this.index = index; + this.value = value; + } + + public int getIndex() { + return index; + } + + public void setIndex(int index) { + this.index = index; + } + + public String getValue() { + return value; + } + + public void setValue(String value) { + this.value = value; + } +} diff --git a/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/constant/StatusEnum.java b/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/constant/StatusEnum.java new file mode 100644 index 0000000000..bb4b8c2656 --- /dev/null +++ b/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/constant/StatusEnum.java @@ -0,0 +1,54 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package com.webank.wedatasphere.dss.apiservice.core.constant; + +/** + * @author chongchuanbing + */ +public enum StatusEnum { + + DISABLE(0, "停止"), + + ENABLE(1, "运行中"); + + private Integer index; + private String name; + + StatusEnum(Integer index, String name) { + this.index = index; + this.name = name; + } + + public static StatusEnum getEnum(Integer index) { + if (index == null) { + return null; + } + for (StatusEnum statusEnum : values()) { + if (statusEnum.getIndex().equals(index)) { + return statusEnum; + } + } + return null; + } + + public Integer getIndex() { + return this.index; + } + + public String getName() { + return this.name; + } +} diff --git a/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/dao/ApiServiceAccessDao.java b/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/dao/ApiServiceAccessDao.java new file mode 100644 index 0000000000..11229d44ec --- /dev/null +++ b/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/dao/ApiServiceAccessDao.java @@ -0,0 +1,31 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.apiservice.core.dao; + +import com.webank.wedatasphere.dss.apiservice.core.vo.ApiAccessVo; + +/** + * @author allenlliu + * @date 2020/11/11 10:25 + */ +public interface ApiServiceAccessDao { + void addAccessRecord(ApiAccessVo apiAccessVo); + + ApiAccessVo queryByVersionId(Long versionId); + + ApiAccessVo queryByApiId(Long apiServiceId); +} diff --git a/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/dao/ApiServiceApprovalDao.java b/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/dao/ApiServiceApprovalDao.java new file mode 100644 index 0000000000..3293a159da --- /dev/null +++ b/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/dao/ApiServiceApprovalDao.java @@ -0,0 +1,74 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.apiservice.core.dao; + + +import com.webank.wedatasphere.dss.apiservice.core.vo.ApprovalVo; +import org.apache.ibatis.annotations.Param; + +/** + * @author: jinyangrao + * @date: 2020-08-26 18:03:34 + */ +public interface ApiServiceApprovalDao { + + /** + * 新增一条审批单 + */ + void insert(ApprovalVo approvalVo); + + /** + * 查询审批单 + * */ + ApprovalVo queryByApprovalNo(@Param("approvalNo") String approvalNo); + + /** + * set status to success + * */ + void setApprovalStatusSuccess(@Param("approvalNo") String approvalNo); + + /** + * set status to failed + * */ + void setApprovalStatusFailed(@Param("approvalNo") String approvalNo); + + /** + * set status to init + * */ + void setApprovalStatusInit(@Param("approvalNo") String approvalNo); + + /** + * set status to applying + * */ + void setApprovalStatusApplying(@Param("approvalNo") String approvalNo); + + /** + * update status + * */ + void updateApprovalStatus(@Param("approvalNo") String approvalNo, @Param("status") Integer status); + + + /** + * delete approval + * */ + void deleteApproval(@Param("approvalNo") String approvalNo); + + /** + * 通过版本ID查询审批单 + * */ + ApprovalVo queryByVersionId(@Param("apiVersionId") Long apiVersionId); +} diff --git a/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/dao/ApiServiceDao.java b/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/dao/ApiServiceDao.java new file mode 100644 index 0000000000..c5f6de7090 --- /dev/null +++ b/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/dao/ApiServiceDao.java @@ -0,0 +1,83 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.apiservice.core.dao; + + +import com.webank.wedatasphere.dss.apiservice.core.bo.ApiServiceQuery; +import com.webank.wedatasphere.dss.apiservice.core.vo.ApiServiceVo; +import org.apache.ibatis.annotations.Param; + +import java.util.List; + +/** + * @author jinyangrao + */ +public interface ApiServiceDao { + + /** + * Insert + */ + void insert(ApiServiceVo apiServiceVo); + + /** + * Update + */ + void update(ApiServiceVo apiServiceVo); + + + void updateToTarget(ApiServiceVo apiServiceVo); + + /** + * query + */ + List query(ApiServiceQuery query); + + /** + * query + */ + + List queryByScriptPath(@Param("scriptPath") String scriptPath); + + /** + * query + */ + ApiServiceVo queryByPath(String path); + + /** + * query api path count + */ + Integer queryCountByPath(@Param("scriptPath") String scriptPath, @Param("path") String path); + + /** + * query api name + */ + Integer queryCountByName(String name); + + Integer enableApi(@Param("id") Long id); + + Integer disableApi(@Param("id") Long id); + + Integer deleteApi(@Param("id") Long id); + + List queryAllTags(@Param("userName")String userName,@Param("workspaceId")Integer workspaceId); + + ApiServiceVo queryById(@Param("id") Long id); + + List queryByWorkspaceId(@Param("workspaceId") Integer workspaceId, @Param("userName") String userName); + + Integer updateApiServiceComment(@Param("id") Long id, @Param("comment") String comment); +} diff --git a/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/dao/ApiServiceParamDao.java b/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/dao/ApiServiceParamDao.java new file mode 100644 index 0000000000..68028347c3 --- /dev/null +++ b/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/dao/ApiServiceParamDao.java @@ -0,0 +1,61 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.apiservice.core.dao; + + +import com.webank.wedatasphere.dss.apiservice.core.vo.ParamVo; +import org.apache.ibatis.annotations.Param; + +import java.util.List; + +/** + * dao + * + * @author zhulixin + */ +public interface ApiServiceParamDao { + + /** + * insert + * + * @param param + * @return + */ + int insert(ParamVo param); + + /** + * Update + * + * @param param + */ + void update(ParamVo param); + + /** + * query + */ +// List queryByConfigIdAndVersion(@Param("configId") Long configId, @Param("version") String version); + + List queryByVersionId(@Param("apiVersionId") Long apiVersionId); + + /** + * query a line by id + * @param paramId + * @return paramVo + * */ + int queryById(@Param("paramId") Long paramId); + +} diff --git a/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/dao/ApiServiceTokenManagerDao.java b/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/dao/ApiServiceTokenManagerDao.java new file mode 100644 index 0000000000..eb1ea3d98f --- /dev/null +++ b/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/dao/ApiServiceTokenManagerDao.java @@ -0,0 +1,117 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.apiservice.core.dao; + +import com.webank.wedatasphere.dss.apiservice.core.vo.TokenManagerVo; +import com.webank.wedatasphere.dss.apiservice.core.bo.TokenQuery; +import org.apache.ibatis.annotations.Param; + +import java.util.List; + +/** + * @author jinyangrao + */ +public interface ApiServiceTokenManagerDao { + + /** + * insert a token + * */ + void insert(TokenManagerVo tokenManagerVo); + + /** + * insert batch token + * */ + void insertList(List tokenManagerVos); + + + /** + * 多条记录需要去重返回最新授权的 + * @param userName + * @return + */ + List queryByApplyUser(String userName); + + + List queryByApplyUserAndServiceId(@Param("userName") String userName, @Param("serviceId") Long serviceId); + + List queryByApplyUserAndVersionId(@Param("userName") String userName, @Param("apiVersionId") Long apiVersionId); + + + + /** + * query + * */ + List query(TokenQuery tokenQuery); + + /** + * query token by token id + * */ + TokenManagerVo queryTokenById(@Param("id") Long tokenId); + + /** + * query token by api service id + * */ + List queryTokenByApiServiceId(@Param("api_id") Long apiServiceId); + + /** + * query token avoid submit again + * */ + int queryApprovalNo(@Param("approvalNo") String approvalNo); + + /** + * query tokens according to status + * */ + List queryTokenByStatus(@Param("status") Integer status); + + /** + * disable token status + * */ + void disableTokenStatus(@Param("id") Long id); + + + /** + * disable token status + * */ + void disableTokenStatusByVersionId(@Param("apiVersionId") Long apiVersionId); + + + /** + * disable token status + * */ + void disableTokenStatusByApiId(@Param("apiId") Long apiId); + + /** + * enable token status + * */ + void enableTokenStatus(@Param("id") Long id); + + void enableTokenStatusByVersionId(@Param("apiVersionId") Long apiVersionId); + + void enableTokenStatusByApiId(@Param("apiId") Long apiId); + + + /** + * delete token + * */ + void deleteTokenById(@Param("id") Long tokenId); + + /** + * update token status + * */ +// void updateTokenStatus(@Param(id) Long tokenId, String status); + +} diff --git a/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/dao/ApiServiceVersionDao.java b/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/dao/ApiServiceVersionDao.java new file mode 100644 index 0000000000..eef5041fc7 --- /dev/null +++ b/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/dao/ApiServiceVersionDao.java @@ -0,0 +1,60 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.apiservice.core.dao; + + +import com.webank.wedatasphere.dss.apiservice.core.vo.ApiVersionVo; +import org.apache.ibatis.annotations.Param; + +import java.util.List; + +/** + * @author jinyangrao + */ +public interface ApiServiceVersionDao { + + /** + * inster apiversionVo + * */ + void insert(ApiVersionVo apiVersionVo); + + /** + * query version + * */ + ApiVersionVo queryApiVersionByVersionId(@Param("id") Long apiVersionId); + + List queryApiVersionByApiServiceId(@Param("api_id") Long apiServiceId); + + /** + * update api version + * */ + void updateApiVersionStatusById(@Param("id") Long apiVersionId, @Param("status") int status); + + void updateApiVersionStatus(ApiVersionVo apiVersionVo); + + /** + * update all api version status by service id + * */ + void updateAllApiVersionStatusByApiServiceId(@Param("api_id") Long apiServiceId, @Param("status") int status); + + void disableApiVersionStatusByApiId(@Param("api_id") Long apiServiceId); + + /** + * delete api version + * */ + void deleteApiVersionById(@Param("id") Long apiVersionId); +} diff --git a/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/dao/mapper/ApiServiceAccessMapper.xml b/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/dao/mapper/ApiServiceAccessMapper.xml new file mode 100644 index 0000000000..ecda53d982 --- /dev/null +++ b/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/dao/mapper/ApiServiceAccessMapper.xml @@ -0,0 +1,66 @@ + + + + + + + + + + + + + + + + + + `login_user`,`api_name`,`api_id`,`api_version_id`, `api_publisher`, `access_time`, `execute_user` + + + + `id`,`login_user`,`api_name`,`api_id`,`api_version_id`, `api_publisher`, `access_time`, `execute_user` + + + + + + + + + + + + + + + + diff --git a/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/dao/mapper/ApiServiceApiVersionMapper.xml b/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/dao/mapper/ApiServiceApiVersionMapper.xml new file mode 100644 index 0000000000..48460f34b5 --- /dev/null +++ b/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/dao/mapper/ApiServiceApiVersionMapper.xml @@ -0,0 +1,109 @@ + + + + + + + + + + + + + + + + + + + + + + `api_id`,`version`,`bml_resource_id`,`bml_version`,`source`,`creator`,`create_time`,`status`, + `metadata_info`, `auth_id` + + + + `id`,`api_id`,`version`,`bml_resource_id`,`bml_version`,`source`,`creator`,`create_time`,`status`, + `metadata_info`, `auth_id` + + + + + + + + + + + + + + UPDATE `dss_apiservice_api_version` + SET `status` = #{status} + WHERE id = #{id} + + + + UPDATE `dss_apiservice_api_version` + SET `status` = 0 + WHERE + api_id = #{api_id} + + + + UPDATE `dss_apiservice_api_version` + SET `status` = #{status} + WHERE id = #{id} + + + + UPDATE `dss_apiservice_api_version` + SET `status` = #{status} + WHERE api_id = #{api_id} + + + + + DELETE FROM + `dss_apiservice_api_version` + WHERE + id = #{id} + + + + \ No newline at end of file diff --git a/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/dao/mapper/ApiServiceApprovalMapper.xml b/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/dao/mapper/ApiServiceApprovalMapper.xml new file mode 100644 index 0000000000..95861ee906 --- /dev/null +++ b/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/dao/mapper/ApiServiceApprovalMapper.xml @@ -0,0 +1,107 @@ + + + + + + + + + + + + + + + + + + + + + + `api_id`, `api_version_id`, `approval_name`, `apply_user`, `execute_user`, `creator`, + `status`, `create_time`, `update_time`, `approval_no` + + + + `id`, `api_version_id`, `api_id`, `approval_name`, `apply_user`, `execute_user`, `creator`, + `status`, `create_time`, `update_time`, `approval_no` + + + + + + + + + + + + UPDATE `dss_apiservice_approval` + SET `status` = 3 + WHERE `approval_no` = #{approvalNo} + + + + UPDATE `dss_apiservice_approval` + SET `status` = 4 + WHERE `approval_no` = #{approvalNo} + + + + UPDATE `dss_apiservice_approval` + SET `status` = 1 + WHERE `approval_no` = #{approvalNo} + + + + UPDATE `dss_apiservice_approval` + SET `status` = 2 + WHERE `approval_no` = #{approvalNo} + + + + UPDATE `dss_apiservice_approval` + SET `status` = #{status} + WHERE `approval_no` = #{approvalNo} + + + + DELETE FROM + `dss_apiservice_approval` + WHERE `approval_no` = #{approvalNo} + + + + + + \ No newline at end of file diff --git a/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/dao/mapper/ApiServiceMapper.xml b/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/dao/mapper/ApiServiceMapper.xml new file mode 100644 index 0000000000..34d6ab3026 --- /dev/null +++ b/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/dao/mapper/ApiServiceMapper.xml @@ -0,0 +1,297 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + `name`,`alias_name`, `path`,`protocol`,`method`,`tag`, `scope`,`description`,`status`,`type`,`run_type`,`create_time`, + `modify_time`,`creator`,`modifier`,`script_path`,`workspaceID`,`api_comment` + + + + `id`, `alias_name`, `name`,`path`,`protocol`,`method`, `tag`, `scope`,`description`,`status`,`type`,`run_type`,`create_time`, + `modify_time`,`creator`,`modifier`,`script_path`,`workspaceID`,`api_comment` + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + `id` = #{id} + + + + + + + + + + + + + + + + + + + + + + + + + + + + `id` = #{id} + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/dao/mapper/ApiServiceParamMapper.xml b/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/dao/mapper/ApiServiceParamMapper.xml new file mode 100644 index 0000000000..e1b4490323 --- /dev/null +++ b/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/dao/mapper/ApiServiceParamMapper.xml @@ -0,0 +1,98 @@ + + + + + + + + + + + + + + + + + + + `api_version_id`,`name`,`type`,`required`,`description`, `display_name`, `default_value`, `details` + + + + `id`, `api_version_id`, `name`,`type`,`required`,`description`, `display_name`, `default_value`, `details` + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + `id` = #{id} + + + + + + + + + + + + + + + + diff --git a/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/dao/mapper/ApiServiceTokenManagerMapper.xml b/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/dao/mapper/ApiServiceTokenManagerMapper.xml new file mode 100644 index 0000000000..1ee6c7399b --- /dev/null +++ b/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/dao/mapper/ApiServiceTokenManagerMapper.xml @@ -0,0 +1,245 @@ + + + + + + + + + + + + + + + + + + + + + + + + `api_id`,`api_version_id`,`publisher`,`user`,`apply_time`,`duration`, + `reason`,`ip_whitelist`,`status`,`caller`,`access_limit`,`token`,`apply_source` + + + + `id`, `api_version_id`,`api_id`,`publisher`,`user`,`apply_time`,`duration`, + `reason`,`ip_whitelist`,`status`,`caller`,`access_limit`,`token`,`apply_source` + + + + `id`, `api_version_id`,`api_id`,`publisher`,`user`,`apply_time`,`duration`, + `reason`,`ip_whitelist`,`status`,`caller`,`access_limit`,`apply_source` + + + + + + + + + + + + + + + (#{item.apiId},#{item.apiVersionId}, #{item.publisher}, #{item.user}, #{item.applyTime}, + #{item.duration}, #{item.reason}, #{item.ipWhitelist}, #{item.status}, + #{item.caller}, #{item.accessLimit}, #{item.token}, #{item.applySource}) + + + + + + + + + + + + + + + + + + + + + + + + + + + UPDATE `dss_apiservice_token_manager` + SET `status` = 1 + WHERE `id` = #{id} + + + + UPDATE `dss_apiservice_token_manager` + SET `status` = 0 + WHERE `id` = #{id} + + + + UPDATE `dss_apiservice_token_manager` + SET `status` = 0 + WHERE `api_version_id` = #{apiVersionId} + + + + + UPDATE `dss_apiservice_token_manager` + SET `status` = 1 + WHERE `api_version_id` = #{apiVersionId} + + + + + + + UPDATE `dss_apiservice_token_manager` + SET `status` = 0 + WHERE `api_id` = #{apiId} + + + + + UPDATE `dss_apiservice_token_manager` + SET `status` = 1 + WHERE `api_id` = #{apiId} + + + + DELETE FROM + `dss_apiservice_token_manager` + WHERE + id = #{id} + + + \ No newline at end of file diff --git a/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/datamap/DataMapStatus.java b/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/datamap/DataMapStatus.java new file mode 100644 index 0000000000..f05fd9b53c --- /dev/null +++ b/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/datamap/DataMapStatus.java @@ -0,0 +1,78 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.apiservice.core.datamap; + +import com.webank.wedatasphere.dss.apiservice.core.constant.ParamTypeEnum; + +/** + * created by cooperyang on 2020/8/27 + * Description: + */ +public enum DataMapStatus { + /** + * datamap审批单的状态 + */ + INITED(1, "inited", "提单成功"), + APPROVING(2, "approving", "审批中"), + SUCCESS(3, "success", "审批通过"), + FAILED(4, "failed", "审批不通过"), + REJECT(5, "reject", "审批驳回"); + + private DataMapStatus(int index, String value, String desc) { + this.index = index; + this.value = value; + this.desc = desc; + } + + private int index; + private String value; + private String desc; + + + public static String getDescByIndex(int index) { + for (DataMapStatus dataMapStatus : values()) { + if (dataMapStatus.getIndex() == index) { + return dataMapStatus.desc; + } + } + return INITED.desc; + } + + public int getIndex() { + return index; + } + + public void setIndex(int index) { + this.index = index; + } + + public String getValue() { + return value; + } + + public void setValue(String value) { + this.value = value; + } + + public String getDesc() { + return desc; + } + + public void setDesc(String desc) { + this.desc = desc; + } +} diff --git a/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/datamap/UUIDGenerator.java b/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/datamap/UUIDGenerator.java new file mode 100644 index 0000000000..2de875f5b8 --- /dev/null +++ b/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/datamap/UUIDGenerator.java @@ -0,0 +1,33 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.apiservice.core.datamap; + +import java.util.UUID; + +/** + * @author: jinyangrao + */ +public class UUIDGenerator { + + /** + * 生成唯一标识uuid + * */ + public static String genUUID() { + String uuid = UUID.randomUUID().toString(); + return uuid; + } +} \ No newline at end of file diff --git a/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/exception/ApiExecuteException.java b/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/exception/ApiExecuteException.java new file mode 100644 index 0000000000..9996744cba --- /dev/null +++ b/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/exception/ApiExecuteException.java @@ -0,0 +1,35 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.apiservice.core.exception; + +import com.webank.wedatasphere.linkis.common.exception.ErrorException; + +/** + * @author allenlliu + * @version 2.0.0 + * @date 2020/08/31 02:31 PM + */ +public class ApiExecuteException extends ErrorException { + + public ApiExecuteException(int errCode, String desc) { + super(errCode, desc); + } + + public ApiExecuteException(int errCode, String desc, String ip, int port, String serviceKind) { + super(errCode, desc, ip, port, serviceKind); + } + } diff --git a/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/exception/ApiServiceQueryException.java b/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/exception/ApiServiceQueryException.java new file mode 100644 index 0000000000..59b5aebbcb --- /dev/null +++ b/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/exception/ApiServiceQueryException.java @@ -0,0 +1,35 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.apiservice.core.exception; + +import com.webank.wedatasphere.linkis.common.exception.ErrorException; + +/** + * @author allenlliu + * @version 2.0.0 + * @date 2020/08/26 07:51 PM + */ +public class ApiServiceQueryException extends ErrorException { + + public ApiServiceQueryException(int errCode, String desc) { + super(errCode, desc); + } + + public ApiServiceQueryException(int errCode, String desc, String ip, int port, String serviceKind) { + super(errCode, desc, ip, port, serviceKind); + } +} \ No newline at end of file diff --git a/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/exception/ApiServiceRuntimeException.java b/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/exception/ApiServiceRuntimeException.java new file mode 100644 index 0000000000..b9b428b890 --- /dev/null +++ b/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/exception/ApiServiceRuntimeException.java @@ -0,0 +1,32 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package com.webank.wedatasphere.dss.apiservice.core.exception; + +/** + * OneserviceRuntimeException + * + * @author lidongzhang + */ +public class ApiServiceRuntimeException extends RuntimeException { + + public ApiServiceRuntimeException(String message) { + super(message); + } + + public ApiServiceRuntimeException(String message, Throwable cause) { + super(message, cause); + } +} diff --git a/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/exception/ApiServiceTokenException.java b/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/exception/ApiServiceTokenException.java new file mode 100644 index 0000000000..bb7619bc0a --- /dev/null +++ b/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/exception/ApiServiceTokenException.java @@ -0,0 +1,35 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.apiservice.core.exception; + +import com.webank.wedatasphere.linkis.common.exception.ErrorException; + +/** + * @author allenlliu + * @version 2.0.0 + * @date 2020/08/14 04:22 PM + */ +public class ApiServiceTokenException extends ErrorException { + + public ApiServiceTokenException(int errCode, String desc) { + super(errCode, desc); + } + + public ApiServiceTokenException(int errCode, String desc, String ip, int port, String serviceKind) { + super(errCode, desc, ip, port, serviceKind); + } + } diff --git a/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/exception/AssertException.java b/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/exception/AssertException.java new file mode 100644 index 0000000000..394474b915 --- /dev/null +++ b/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/exception/AssertException.java @@ -0,0 +1,28 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package com.webank.wedatasphere.dss.apiservice.core.exception; + +/** + * AssertException + * + * @author lidongzhang + */ +public class AssertException extends ApiServiceRuntimeException { + + public AssertException(String message) { + super(message); + } +} diff --git a/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/exception/BeanValidationExceptionMapper.java b/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/exception/BeanValidationExceptionMapper.java new file mode 100644 index 0000000000..0f12126c6f --- /dev/null +++ b/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/exception/BeanValidationExceptionMapper.java @@ -0,0 +1,42 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.apiservice.core.exception; + +import com.webank.wedatasphere.linkis.server.Message; + +import javax.validation.ConstraintViolationException; +import javax.validation.ValidationException; +import javax.ws.rs.core.Response; +import javax.ws.rs.ext.ExceptionMapper; +import javax.ws.rs.ext.Provider; + +/** + * Map bean validation exception to response + * @author liaoyt + * 2020/02/11 + */ +@Provider +public class BeanValidationExceptionMapper implements ExceptionMapper { + @Override + public Response toResponse(ValidationException exception) { + StringBuilder stringBuilder = new StringBuilder(); + ((ConstraintViolationException)exception) + .getConstraintViolations().forEach(constraintViolation -> stringBuilder.append(constraintViolation.getMessage()).append(";")); + Message message = Message.error("Bean validation error[实例校验出错], detail:" + stringBuilder.toString()); + return Message.messageToResponse(message); + } +} diff --git a/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/execute/ApiServiceExecuteJob.java b/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/execute/ApiServiceExecuteJob.java new file mode 100644 index 0000000000..1b1e3896c1 --- /dev/null +++ b/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/execute/ApiServiceExecuteJob.java @@ -0,0 +1,64 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.apiservice.core.execute; + +import com.webank.wedatasphere.linkis.ujes.client.response.JobExecuteResult; + +import java.util.Map; + +/** + * @author allenlliu + * @version 2.0.0 + * @date 2020/08/12 09:01 PM + */ +public interface ApiServiceExecuteJob { + String getCode(); + + void setCode(String code); + + String getEngineType(); + + void setEngineType(String engineType); + + String getRunType(); + + void setRunType(String runType); + + String getUser(); + + void setUser(String user); + + String getJobName(); + + Map getParams(); + + void setParams(Map params); + + Map getRuntimeParams(); + void setRuntimeParams(Map runtimeParams); + + JobExecuteResult getJobExecuteResult(); + + void setJobExecuteResult(JobExecuteResult jobExecuteResult); + + Map getJobProps(); + + void setJobProps(Map jobProps); + + String getScriptPath(); + void setScriptePath(String path); +} diff --git a/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/execute/DefaultApiServiceJob.java b/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/execute/DefaultApiServiceJob.java new file mode 100644 index 0000000000..6d11d4197b --- /dev/null +++ b/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/execute/DefaultApiServiceJob.java @@ -0,0 +1,145 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.apiservice.core.execute; + +import com.webank.wedatasphere.linkis.ujes.client.request.JobExecuteAction; +import com.webank.wedatasphere.linkis.ujes.client.response.JobExecuteResult; + +import java.util.Map; + +/** + * @author allenlliu + * @version 2.0.0 + * @date 2020/08/12 09:06 PM + */ +public class DefaultApiServiceJob implements ApiServiceExecuteJob { + private Map jobProps; + private String user; + private Map source; + private Map variables; + + private Map configuration; + + private String code; + + private String engineType; + + private String runType; + + private Map params; + + private Map runtimeParams; + + private JobExecuteResult jobExecuteResult; + + private String scriptPath; + + @Override + public String getCode() { + return this.code; + } + + @Override + public void setCode(String code) { + this.code = code; + } + + @Override + public String getEngineType() { + return engineType; + } + + @Override + public void setEngineType(String engineType) { + this.engineType = engineType; + } + + @Override + public String getRunType() { + return runType; + } + + @Override + public void setRunType(String runType) { + this.runType = runType; + } + + @Override + public String getUser() { + return user; + } + + @Override + public void setUser(String user){ + this.user =user; + } + + @Override + public String getJobName() { + return null; + } + + @Override + public Map getParams() { + return params; + } + + @Override + public void setParams(Map params) { + this.params = params; + } + + @Override + public Map getRuntimeParams() { + return runtimeParams; + } + + @Override + public void setRuntimeParams(Map runtimeParams) { + this.runtimeParams = runtimeParams; + } + + @Override + public JobExecuteResult getJobExecuteResult() { + return this.jobExecuteResult; + } + + @Override + public void setJobExecuteResult(JobExecuteResult jobExecuteResult) { + this.jobExecuteResult = jobExecuteResult; + } + + @Override + public Map getJobProps() { + return jobProps; + } + + @Override + public void setJobProps(Map jobProps) { + this.jobProps = jobProps; + } + + @Override + public String getScriptPath() { + return scriptPath; + } + + @Override + public void setScriptePath(String path) { + this.scriptPath = path; + } +} diff --git a/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/execute/ExecuteCodeHelper.java b/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/execute/ExecuteCodeHelper.java new file mode 100644 index 0000000000..a8a3d2c743 --- /dev/null +++ b/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/execute/ExecuteCodeHelper.java @@ -0,0 +1,268 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.apiservice.core.execute; + +import com.webank.wedatasphere.dss.apiservice.core.action.ApiServiceGetAction; +import com.webank.wedatasphere.dss.apiservice.core.action.ResultSetDownloadAction; +import com.webank.wedatasphere.dss.apiservice.core.action.ResultWorkspaceIds; +import com.webank.wedatasphere.dss.apiservice.core.config.ApiServiceConfiguration; +import com.webank.wedatasphere.dss.apiservice.core.exception.ApiExecuteException; +import com.webank.wedatasphere.linkis.common.utils.Utils; +import com.webank.wedatasphere.linkis.httpclient.response.Result; +import com.webank.wedatasphere.linkis.ujes.client.UJESClient; +import com.webank.wedatasphere.linkis.ujes.client.request.ResultSetAction; +import com.webank.wedatasphere.linkis.ujes.client.request.ResultSetListAction; +import com.webank.wedatasphere.linkis.ujes.client.response.JobExecuteResult; +import com.webank.wedatasphere.linkis.ujes.client.response.JobInfoResult; +import com.webank.wedatasphere.linkis.ujes.client.response.JobLogResult; +import com.webank.wedatasphere.linkis.ujes.client.response.ResultSetListResult; +import org.apache.commons.lang.StringUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.InputStream; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.Map; + +/** + * created by cooperyang on 2020/8/26 + * Description: + */ +public class ExecuteCodeHelper { + + private static final Logger LOGGER = LoggerFactory.getLogger(ExecuteCodeHelper.class); + + private static final String RELEASE_SCALA_TEMPLATE = "import org.apache.spark.sql.DataFrame\n" + + "val sql = %s\n" + + "val df = sqlContext.sql(sql)\n" + + "show(df)\n"; + + + private static final String SCALA_MARK = "\"\"\""; + + private static final String EXECUTE_SCALA_TEMPLATE = "import org.apache.spark.sql.DataFrame\n" + + "val executeCode = %s\n" + + "val df = sqlContext.sql(executeCode)\n" + + "show(df)\n"; + + + + public static String packageCodeToRelease(String executeCode){ + String retStr = String.format(RELEASE_SCALA_TEMPLATE, SCALA_MARK + executeCode + SCALA_MARK); + LOGGER.info("release scala code is {}", retStr); + return retStr; + } + + public static String packageCodeToExecute(String executeCode, String metaDataInfo){ + String retStr = String.format(EXECUTE_SCALA_TEMPLATE, SCALA_MARK + executeCode + SCALA_MARK); + LOGGER.info("execute scala code is {}", retStr); + return retStr; + } + + + public static Map getMetaDataInfoByExecute(String user, + String executeCode, + Map Params, + String scriptPath) throws Exception { + Map props = new HashMap<>(); + Map resultMap = new HashMap<>(); + UJESClient client = LinkisJobSubmit.getClient(props); + ApiServiceExecuteJob job = new DefaultApiServiceJob(); + //sql代码封装成scala执行 + job.setCode(ExecuteCodeHelper.packageCodeToRelease(executeCode)); + job.setEngineType("spark"); + job.setRunType("scala"); + job.setUser(user); + job.setParams(null); + job.setRuntimeParams((Map)Params.get("variable"));// pattern注入 + job.setScriptePath(scriptPath); + JobExecuteResult jobExecuteResult = LinkisJobSubmit.execute(job,client, "IDE"); + job.setJobExecuteResult(jobExecuteResult); + try { + waitForComplete(job,client); + } catch (Exception e) { + LOGGER.warn("Failed to execute job", e); + String reason = getLog(job,client); + LOGGER.error("Reason for failure: " + reason); + throw new ApiExecuteException(800024,"获取库表信息失败，执行脚本出错！"); + } + int resultSize = getResultSize(job,client); + for(int i =0; i < resultSize; i++){ + String result = getResult(job, i, ApiServiceConfiguration.RESULT_PRINT_SIZE.getValue().intValue(),client); + LOGGER.info("The content of the " + (i + 1) + "th resultset is :" + + result); + resultMap.put(Integer.toString(i),result); + + } + + LOGGER.info("Finished to execute job"); + return resultMap; + } + + + public static void waitForComplete(ApiServiceExecuteJob job,UJESClient client) throws Exception { + JobInfoResult jobInfo = client.getJobInfo(job.getJobExecuteResult()); + while (!jobInfo.isCompleted()) { + LOGGER.info("Update Progress info:" + getProgress(job,client)); + LOGGER.info("<----linkis log ---->"); + Utils.sleepQuietly(ApiServiceConfiguration.LINKIS_JOB_REQUEST_STATUS_TIME.getValue(job.getJobProps())); + jobInfo = client.getJobInfo(job.getJobExecuteResult()); + } + if (!jobInfo.isSucceed()) { + throw new ApiExecuteException(90101, "Failed to execute Job: " + jobInfo.getTask().get("errDesc").toString()); + } + } + + + public static void cancel(ApiServiceExecuteJob job,UJESClient client) throws Exception { + client.kill(job.getJobExecuteResult()); + } + + + public static double getProgress(ApiServiceExecuteJob job,UJESClient client) { + return client.progress(job.getJobExecuteResult()).getProgress(); + } + + + public static Boolean isCompleted(ApiServiceExecuteJob job,UJESClient client) { + return client.getJobInfo(job.getJobExecuteResult()).isCompleted(); + } + + public static String getResult(ApiServiceExecuteJob job, int index, int maxSize,UJESClient client) { + String resultContent = null; + JobInfoResult jobInfo = client.getJobInfo(job.getJobExecuteResult()); + String[] resultSetList = jobInfo.getResultSetList(LinkisJobSubmit.getClient(job.getJobProps())); + if (resultSetList != null && resultSetList.length > 0) { + Object fileContent = client.resultSet(ResultSetAction.builder() + .setPath(resultSetList[index]) + .setUser(job.getJobExecuteResult().getUser()) + .setPageSize(maxSize).build()).getFileContent(); + if (fileContent instanceof ArrayList) { + ArrayList> resultSetRow = (ArrayList>) fileContent; + resultContent = StringUtils.join(resultSetRow.get(0), "\n"); + } else { + resultContent = fileContent.toString(); + } + } + return resultContent; + } + + + public static int getResultSize(ApiServiceExecuteJob job,UJESClient client) { + JobInfoResult jobInfo = client.getJobInfo(job.getJobExecuteResult()); + if (jobInfo.isSucceed()) { + String[] resultSetList = jobInfo.getResultSetList(LinkisJobSubmit.getClient(job.getJobProps())); + if (resultSetList != null && resultSetList.length > 0) { + return resultSetList.length; + } + } + return 0; + } + + + public static String getLog(ApiServiceExecuteJob job, UJESClient client) { + + JobLogResult jobLogResult = client + .log(job.getJobExecuteResult(), + 0, + 50); + + ArrayList logArray = jobLogResult.getLog(); + + if (logArray != null && logArray.size() + >= ApiServiceConfiguration.LOG_ARRAY_LEN.getValue() + && StringUtils.isNotEmpty(logArray.get(3))) { + return logArray.get(3); + } + return null; + } + + public static String getResultList(JobExecuteResult executeResult,UJESClient client, String path) { + ResultSetListResult resultList = (ResultSetListResult) client.executeUJESJob(ResultSetListAction.builder() + .setUser(executeResult.getUser()).setPath(path).build()); + return resultList.getResponseBody(); + } + + + + public static String getResultContent(String user, String path, int maxSize,UJESClient client) { + + String fileContent = client.resultSet(ResultSetAction.builder() + .setPath(path) + .setUser(user) + .setPageSize(maxSize).build()).getResponseBody(); + + return fileContent; + } + + public static InputStream downloadResultSet(String user, + String path, + String charset, + String outputFileType, + String csvSeperator, + String outputFileName, + String sheetName, + String nullValue, + UJESClient client) { + + ResultSetDownloadAction resultSetDownloadAction = new ResultSetDownloadAction(); + resultSetDownloadAction.setUser(user); + resultSetDownloadAction.setParameter("path",path); + resultSetDownloadAction.setParameter("charset",charset); + resultSetDownloadAction.setParameter("outputFileType",outputFileType); + resultSetDownloadAction.setParameter("csvSeperator",csvSeperator); + resultSetDownloadAction.setParameter("outputFileName",outputFileName); + resultSetDownloadAction.setParameter("sheetName",sheetName); + resultSetDownloadAction.setParameter("nullValue",nullValue); + client.executeUJESJob(resultSetDownloadAction); + return resultSetDownloadAction.getInputStream(); + } + + + + + + + + public static Map getTaskInfoById(JobExecuteResult jobExecuteResult, UJESClient client) { + + Map taskInfo = (Map)client.getJobInfo(jobExecuteResult).getTask(); + + return taskInfo; + } + + + + + public static String getUserWorkspaceIds(String userName,UJESClient client){ + ApiServiceGetAction apiServiceGetAction = new ApiServiceGetAction(); + apiServiceGetAction.setUser(userName); + apiServiceGetAction.setParameter("userName",userName); + ResultWorkspaceIds userWorkspaceIds = (ResultWorkspaceIds)client.executeUJESJob(apiServiceGetAction); + return userWorkspaceIds.getUserWorkspaceIds(); + + } + + + + + + + + +} diff --git a/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/execute/LinkisJobSubmit.java b/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/execute/LinkisJobSubmit.java new file mode 100644 index 0000000000..5c5d4f50c6 --- /dev/null +++ b/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/execute/LinkisJobSubmit.java @@ -0,0 +1,104 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.apiservice.core.execute; + +import com.webank.wedatasphere.dss.apiservice.core.config.ApiServiceConfiguration; +import com.webank.wedatasphere.linkis.common.conf.Configuration; +import com.webank.wedatasphere.linkis.httpclient.dws.authentication.TokenAuthenticationStrategy; +import com.webank.wedatasphere.linkis.httpclient.dws.config.DWSClientConfig; +import com.webank.wedatasphere.linkis.httpclient.dws.config.DWSClientConfigBuilder; +import com.webank.wedatasphere.linkis.ujes.client.UJESClient; +import com.webank.wedatasphere.linkis.ujes.client.UJESClientImpl; +import com.webank.wedatasphere.linkis.ujes.client.request.JobExecuteAction; +import com.webank.wedatasphere.linkis.ujes.client.response.JobExecuteResult; + +import java.util.HashMap; +import java.util.Map; +import java.util.concurrent.TimeUnit; + +/** + * @author allenlliu + * @version 2.0.0 + * @date 2020/08/12 03:33 PM + */ +public class LinkisJobSubmit { + + + + public static UJESClient getClient(Map props) { + + UJESClient client = getUJESClient( + Configuration.GATEWAY_URL().getValue(props), + ApiServiceConfiguration.LINKIS_ADMIN_USER.getValue(props), + ApiServiceConfiguration.LINKIS_AUTHOR_USER_TOKEN.getValue(props), + props); + + return client; + } + + + public static UJESClient getUJESClient(String url, String user, String token, Map jobProps){ + return new UJESClientImpl(getClientConfig(url,user,token, jobProps)); + } + + public static DWSClientConfig getClientConfig(String url, String user, String token, Map jobProps){ + DWSClientConfig clientConfig = ((DWSClientConfigBuilder) (DWSClientConfigBuilder.newBuilder() + .addServerUrl(url) + .connectionTimeout(ApiServiceConfiguration.LINKIS_CONNECTION_TIMEOUT.getValue(jobProps)) + .discoveryEnabled(false).discoveryFrequency(1, TimeUnit.MINUTES) + .loadbalancerEnabled(true) + .maxConnectionSize(5) + .retryEnabled(false).readTimeout(ApiServiceConfiguration.LINKIS_CONNECTION_TIMEOUT.getValue(jobProps)) + .setAuthenticationStrategy(new TokenAuthenticationStrategy()) + .setAuthTokenKey(user).setAuthTokenValue(token))) + .setDWSVersion(ApiServiceConfiguration.LINKIS_API_VERSION.getValue(jobProps)).build(); + return clientConfig; + } + + + + public static JobExecuteResult execute(ApiServiceExecuteJob job,UJESClient client) { + Map source = new HashMap<>(); + source.put("DSS-ApiService",job.getScriptPath()); + JobExecuteAction.Builder builder = JobExecuteAction.builder().setCreator("IDE") + .addExecuteCode(job.getCode()) + .setEngineTypeStr(job.getEngineType()) + .setRunTypeStr(job.getRunType()) + .setUser(job.getUser()) + .setParams(job.getParams()) + .setVariableMap(job.getRuntimeParams()) + .setSource(source); + JobExecuteAction jobAction = builder.build(); + JobExecuteResult res = client.execute(jobAction); + return res; + } + public static JobExecuteResult execute(ApiServiceExecuteJob job,UJESClient client, String creator) { + Map source = new HashMap<>(); + source.put("DSS-ApiService",job.getScriptPath()); + JobExecuteAction.Builder builder = JobExecuteAction.builder().setCreator(creator) + .addExecuteCode(job.getCode()) + .setEngineTypeStr(job.getEngineType()) + .setRunTypeStr(job.getRunType()) + .setUser(job.getUser()) + .setParams(job.getParams()) + .setVariableMap(job.getRuntimeParams()) + .setSource(source); + JobExecuteAction jobAction = builder.build(); + JobExecuteResult res = client.execute(jobAction); + return res; + } +} diff --git a/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/jdbc/DatasourceService.java b/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/jdbc/DatasourceService.java new file mode 100644 index 0000000000..6dc3784aa8 --- /dev/null +++ b/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/jdbc/DatasourceService.java @@ -0,0 +1,69 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package com.webank.wedatasphere.dss.apiservice.core.jdbc; + +import com.zaxxer.hikari.HikariConfig; +import com.zaxxer.hikari.HikariDataSource; +import org.springframework.context.annotation.Configuration; +import org.springframework.jdbc.core.namedparam.NamedParameterJdbcTemplate; + +import javax.sql.DataSource; +import java.util.HashMap; +import java.util.Map; + +/** + * DatasourceConfiguration + * + * @author lidongzhang + */ +@Configuration +public class DatasourceService { + private static final Map CLIENT_MAP = new HashMap<>(); + + + private DataSource getDatasource(String url, String username, String password) { + HikariConfig hikariConfig = new HikariConfig(); + hikariConfig.setJdbcUrl(url); + hikariConfig.setUsername(username); + hikariConfig.setPassword(password); + hikariConfig.setMaximumPoolSize(100); + hikariConfig.setMinimumIdle(10); + return new HikariDataSource(hikariConfig); + } + + + public NamedParameterJdbcTemplate getNamedParameterJdbcTemplate(String url, + String username, String password) { + String key = genKey(url, username); + NamedParameterJdbcTemplate jdbcTemplate = CLIENT_MAP.get(key); + + if (jdbcTemplate == null) { + synchronized (this) { + jdbcTemplate = CLIENT_MAP.get(key); + if (jdbcTemplate == null) { + jdbcTemplate = new NamedParameterJdbcTemplate(getDatasource(url, username, password)); + CLIENT_MAP.put(key, jdbcTemplate); + } + } + } + + return jdbcTemplate; + } + + private String genKey(String jdbcUrl, String username) { + return String.join("-", jdbcUrl, username); + } +} diff --git a/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/jdbc/JdbcUtil.scala b/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/jdbc/JdbcUtil.scala new file mode 100644 index 0000000000..00ce6a1b85 --- /dev/null +++ b/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/jdbc/JdbcUtil.scala @@ -0,0 +1,52 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.apiservice.core.jdbc + +import java.util + +import com.webank.wedatasphere.linkis.common.exception.WarnException +import com.webank.wedatasphere.linkis.common.utils.Logging +//import com.webank.wedatasphere.linkis.datasourcemanager.common.protocol.{DsInfoQueryRequest, DsInfoResponse} +import com.webank.wedatasphere.linkis.rpc.Sender + +object JdbcUtil extends Logging { + +// val sender : Sender = Sender.getSender("dsm-server") +// def getDatasourceInfo(params : util.Map[String, Any]) : (String, String, String) = { +// val datasourceId = params.get("configuration").asInstanceOf[util.Map[String, Any]] +// .getOrDefault("datasource", new util.HashMap[String, Any]()) +// .asInstanceOf[util.Map[String, Any]].get("datasourceId") +// logger.info(s"begin to get datasource info from dsm, datasourceId: ${datasourceId}") +// if (datasourceId != null) { +// val ds = sender.ask(DsInfoQueryRequest(String.valueOf(datasourceId), "BDP")) match { +// case r: DsInfoResponse => r +// case warn: WarnException => throw warn +// } +// logger.info(s"get datasource info result: ${ds}") +// if (ds.status) { +// val url = ds.params.get("jdbc.url").asInstanceOf[String] +// val userName = ds.params.get("jdbc.username").asInstanceOf[String] +// val password = ds.params.get("jdbc.password").asInstanceOf[String] +// logger.info(s"get from dsm: url: ${url}, username: ${userName}, password: ${password}") +// return (url, userName, password) +// } +// } +// +// ("", "", "") +// } + +} diff --git a/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/restful/ApiServiceCoreRestfulApi.java b/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/restful/ApiServiceCoreRestfulApi.java new file mode 100644 index 0000000000..5816b104a4 --- /dev/null +++ b/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/restful/ApiServiceCoreRestfulApi.java @@ -0,0 +1,542 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.apiservice.core.restful; + +import com.webank.wedatasphere.dss.apiservice.core.bo.ApiServiceQuery; +import com.webank.wedatasphere.dss.apiservice.core.util.ApiUtils; +import com.webank.wedatasphere.dss.apiservice.core.util.AssertUtil; +import com.webank.wedatasphere.dss.apiservice.core.vo.*; +import com.webank.wedatasphere.dss.apiservice.core.service.ApiServiceQueryService; +import com.webank.wedatasphere.dss.apiservice.core.service.ApiService; +import com.webank.wedatasphere.linkis.server.Message; +import com.webank.wedatasphere.linkis.server.security.SecurityFilter; +import org.apache.commons.collections.MapUtils; +import org.apache.commons.lang.StringUtils; +import org.codehaus.jackson.JsonNode; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; + +import javax.servlet.http.HttpServletRequest; +import javax.validation.ConstraintViolation; +import javax.validation.ConstraintViolationException; +import javax.validation.Validator; +import javax.validation.groups.Default; +import javax.ws.rs.*; +import javax.ws.rs.core.Context; +import javax.ws.rs.core.MediaType; +import javax.ws.rs.core.Response; +import java.util.*; +import java.util.regex.Pattern; +import java.util.stream.Collectors; + +/** + * api service + * + * @author zhulixin + */ +@Path("/dss/apiservice") +@Consumes(MediaType.APPLICATION_JSON) +@Produces(MediaType.APPLICATION_JSON) +@Component +public class ApiServiceCoreRestfulApi { + + private static final Logger LOG = LoggerFactory.getLogger(ApiServiceCoreRestfulApi.class); + + @Autowired + private ApiService apiService; + + @Autowired + private ApiServiceQueryService apiServiceQueryService; + + @Autowired + private Validator beanValidator; + + private static final Pattern WRITABLE_PATTERN = Pattern.compile("^\\s*(insert|update|delete|drop|alter|create).*", Pattern.CASE_INSENSITIVE | Pattern.DOTALL); + + @POST + @Path("/api") + public Response insert(ApiServiceVo apiService, @Context HttpServletRequest req) { + return ApiUtils.doAndResponse(() -> { + + if (apiService.getWorkspaceId() == null){ + apiService.setWorkspaceId(180L); + } + + if (StringUtils.isBlank(apiService.getAliasName())) { + return Message.error("'api service alias name' is missing[缺少中文名]"); + } + + if (StringUtils.isBlank(apiService.getScriptPath())) { + return Message.error("'api service script path' is missing[缺少脚本路径]"); + } + if (StringUtils.isBlank(apiService.getContent())) { + return Message.error("'api service script content' is missing[缺少脚本内容]"); + } + + if (null == apiService.getWorkspaceId()) { + return Message.error("'api service workspaceId ' is missing[缺少工作空间ID]"); + } + if (apiService.getContent().contains(";")) { + if(!apiService.getContent().toLowerCase().startsWith("use ")) { + return Message.error("'api service script content exists semicolon[脚本内容包含分号]"); + } + } + +// check data change script + if (WRITABLE_PATTERN.matcher(apiService.getContent()).matches()) { + return Message.error("'api service script content' only supports query[脚本内容只支持查询语句]"); + } + + Map metadata = apiService.getMetadata(); + if (apiService.getScriptPath().endsWith(".jdbc")) { + if (MapUtils.isEmpty(metadata)) { + return Message.error("'api service metadata' is missing[请选择数据源]"); + } + + Map configuration = (Map) metadata.get("configuration"); + if (MapUtils.isEmpty(configuration)) { + return Message.error("'api service metadata.configuration' is missing[请选择数据源]"); + } + + Map datasource = (Map) configuration.get("datasource"); + if (MapUtils.isEmpty(datasource)) { + return Message.error("'api service metadata.configuration.datasource' is missing[请选择数据源]"); + } + } + + String userName = SecurityFilter.getLoginUsername(req); + Set> result = beanValidator.validate(apiService, Default.class); + if (result.size() > 0) { + throw new ConstraintViolationException(result); + } + + ApprovalVo approvalVo = apiService.getApprovalVo(); + +// if (StringUtils.isBlank(approvalVo.getApprovalName())) { +// return Message.error("'approvalName' is missing[缺少审批单名字]"); +// } + + if (StringUtils.isBlank(approvalVo.getApplyUser())) { + return Message.error("'applyUser' is missing[缺少申请用户名字]"); + } + + apiService.setCreator(userName); + apiService.setModifier(userName); + this.apiService.save(apiService); + return Message.ok().data("insert_id", apiService.getId()).data("approval_no",approvalVo.getApprovalNo()); + }, "/apiservice/api", "Fail to insert service api[新增服务api失败]"); + } + + @POST + @Path("/create") + public Response create(ApiServiceVo apiService, @Context HttpServletRequest req) { + return ApiUtils.doAndResponse(() -> { + + if (apiService.getWorkspaceId() == null){ + apiService.setWorkspaceId(180L); + } + + if (StringUtils.isBlank(apiService.getAliasName())) { + return Message.error("'api service alias name' is missing[缺少中文名]"); + } + + if (StringUtils.isBlank(apiService.getScriptPath())) { + return Message.error("'api service script path' is missing[缺少脚本路径]"); + } + if (StringUtils.isBlank(apiService.getContent())) { + return Message.error("'api service script content' is missing[缺少脚本内容]"); + } + + if (null == apiService.getWorkspaceId()) { + return Message.error("'api service workspaceId ' is missing[缺少工作空间ID]"); + } + if (apiService.getContent().contains(";")) { + if(!apiService.getContent().toLowerCase().startsWith("use ")) { + return Message.error("'api service script content exists semicolon[脚本内容包含分号]"); + } + } + +// check data change script + if (WRITABLE_PATTERN.matcher(apiService.getContent()).matches()) { + return Message.error("'api service script content' only supports query[脚本内容只支持查询语句]"); + } + + Map metadata = apiService.getMetadata(); + if (apiService.getScriptPath().endsWith(".jdbc")) { + if (MapUtils.isEmpty(metadata)) { + return Message.error("'api service metadata' is missing[请选择数据源]"); + } + + Map configuration = (Map) metadata.get("configuration"); + if (MapUtils.isEmpty(configuration)) { + return Message.error("'api service metadata.configuration' is missing[请选择数据源]"); + } + + Map datasource = (Map) configuration.get("datasource"); + if (MapUtils.isEmpty(datasource)) { + return Message.error("'api service metadata.configuration.datasource' is missing[请选择数据源]"); + } + } + + String userName = SecurityFilter.getLoginUsername(req); + Set> result = beanValidator.validate(apiService, Default.class); + if (result.size() > 0) { + throw new ConstraintViolationException(result); + } + + ApprovalVo approvalVo = apiService.getApprovalVo(); + + if (StringUtils.isBlank(approvalVo.getApprovalName())) { + return Message.error("'approvalName' is missing[缺少审批单名字]"); + } + + if (StringUtils.isBlank(approvalVo.getApplyUser())) { + return Message.error("'applyUser' is missing[缺少申请用户名字]"); + } + + apiService.setCreator(userName); + apiService.setModifier(userName); + this.apiService.saveByApp(apiService); + return Message.ok().data("insert_id", apiService.getId()).data("approval_no",approvalVo.getApprovalNo()); + }, "/apiservice/api", "Fail to insert service api[新增服务api失败]"); + } + + @PUT + @Path("/api/{api_service_version_id}") + public Response update(ApiServiceVo apiService, + @PathParam("api_service_version_id") Long apiServiceVersionId, + @Context HttpServletRequest req) { + return ApiUtils.doAndResponse(() -> { + + + if (StringUtils.isBlank(apiService.getScriptPath())) { + return Message.error("'api service script path' is missing[缺少脚本路径]"); + } + if(apiServiceVersionId !=0) { + if (StringUtils.isBlank(apiService.getPath())) { + return Message.error("'api service api path' is missing[缺少api路径]"); + } + } + if (StringUtils.isBlank(apiService.getContent())) { + return Message.error("'api service script content' is missing[缺少脚本内容]"); + } + + if (null == apiService.getWorkspaceId()) { + return Message.error("'api service workspaceId ' is missing[缺少工作空间ID]"); + } + + if (null == apiService.getTargetServiceId()) { + return Message.error("'api service update to target service id ' is missing[缺少更新目标服务ID]"); + } + + if (apiService.getContent().contains(";")) { + return Message.error("'api service script content exists semicolon[脚本内容包含分号]"); + } + + ApprovalVo approvalVo = apiService.getApprovalVo(); + +// if (StringUtils.isBlank(approvalVo.getApprovalName())) { +// return Message.error("'approvalName' is missing[缺少审批单名字]"); +// } + + if (StringUtils.isBlank(approvalVo.getApplyUser())) { + return Message.error("'applyUser' is missing[缺少申请用户名字]"); + } +// if (StringUtils.isBlank(apiService.getResourceId())) { +// return Message.error("'api service resourceId' is missing[缺少bml resourceId]"); +// } + +// check data change script + if (WRITABLE_PATTERN.matcher(apiService.getContent()).matches()) { + return Message.error("'api service script content' only supports query[脚本内容只支持查询语句]"); + } + + Map metadata = apiService.getMetadata(); + if (apiService.getScriptPath().endsWith(".jdbc")) { + if (MapUtils.isEmpty(metadata)) { + return Message.error("'api service metadata' is missing[请选择数据源]"); + } + + Map configuration = (Map) metadata.get("configuration"); + if (MapUtils.isEmpty(configuration)) { + return Message.error("'api service metadata.configuration' is missing[请选择数据源]"); + } + + Map datasource = (Map) configuration.get("datasource"); + if (MapUtils.isEmpty(datasource)) { + return Message.error("'api service metadata.configuration.datasource' is missing[请选择数据源]"); + } + } + + String userName = SecurityFilter.getLoginUsername(req); +// Bean validation + Set> result = beanValidator.validate(apiService, Default.class); + if (result.size() > 0) { + throw new ConstraintViolationException(result); + } + apiService.setLatestVersionId(apiServiceVersionId); + apiService.setModifier(userName); + apiService.setModifyTime(Calendar.getInstance().getTime()); + this.apiService.update(apiService); + return Message.ok().data("update_id", apiServiceVersionId); + }, "/apiservice/api/" + apiServiceVersionId, "Fail to update service api[更新服务api失败]"); + } + + + + + @GET + @Path("/search") + public Response query(@QueryParam("name") String name, + @QueryParam("tag") String tag, + @QueryParam("status") Integer status, + @QueryParam("creator") String creator, + @QueryParam("workspaceId") Integer workspaceId, + @Context HttpServletRequest req) { + String userName = SecurityFilter.getLoginUsername(req); + + return ApiUtils.doAndResponse(() -> { + if (null == workspaceId) { + return Message.error("'api service search workspaceId' is missing[缺少工作空间Id]"); + } + ApiServiceQuery query = new ApiServiceQuery(userName,name, tag, status, creator); + query.setWorkspaceId(workspaceId); + if(!this.apiService.checkUserWorkspace(userName,workspaceId) ){ + return Message.error("'api service search workspaceId' is wrong[该用户不属于该工作空间Id]"); + } + List queryList = apiService.query(query); + return Message.ok().data("query_list", queryList); + }, "/apiservice/search", "Fail to query page of service api[查询服务api失败]"); + } + + + @GET + @Path("/getUserServices") + public Response getUserServices(@QueryParam("workspaceId") Integer workspaceId, + @Context HttpServletRequest req){ + String userName = SecurityFilter.getLoginUsername(req); + return ApiUtils.doAndResponse(() -> { + if(!this.apiService.checkUserWorkspace(userName,workspaceId) ){ + return Message.error("'api service getUserServices workspaceId' is wrong[该用户不属于该工作空间Id]"); + } + List apiServiceList = apiService.queryByWorkspaceId(workspaceId,userName); + return Message.ok().data("query_list", apiServiceList); + }, "/apiservice/getUserServices", "Fail to query page of user service api[查询用户服务api列表失败]"); + } + + + + @GET + @Path("/tags") + public Response query( @Context HttpServletRequest req,@QueryParam("workspaceId") Integer workspaceId) { + String userName = SecurityFilter.getLoginUsername(req); + return ApiUtils.doAndResponse(() -> { + + List tags= apiService.queryAllTags(userName,workspaceId); + return Message.ok().data("tags", tags); + }, "/apiservice/tags", "Fail to query page of service tag[查询服务tag失败]"); + } + + + + + @GET + @Path("/query") + public Response queryByScriptPath(@QueryParam("scriptPath") String scriptPath, + @Context HttpServletRequest req) { + return ApiUtils.doAndResponse(() -> { + String userName = SecurityFilter.getLoginUsername(req); + if (StringUtils.isBlank(scriptPath)) { + return Message.error("'api service scriptPath' is missing[缺少脚本路径]"); + } + ApiServiceVo apiServiceVo = apiService.queryByScriptPath(scriptPath); + if(null != apiServiceVo) { + if (!this.apiService.checkUserWorkspace(userName, apiServiceVo.getWorkspaceId().intValue())) { + return Message.error("'api service query workspaceId' is wrong[该用户不属于该工作空间Id]"); + } + + if (apiServiceVo.getCreator().equals(userName)) { + return Message.ok().data("result", apiServiceVo); + } else { + return Message.error("'api service belong to others' [该脚本路径数据服务不属于当前用户]"); + } + }else { + return Message.ok().data("result", apiServiceVo); + } + }, "/apiservice/query", "Fail to query page of service api[查询服务api失败]"); + } + + @GET + @Path("/queryById") + public Response queryById(@QueryParam("id") Long id, + @Context HttpServletRequest req) { + String userName = SecurityFilter.getLoginUsername(req); + return ApiUtils.doAndResponse(() -> { + if (id==null) { + return Message.error("'api service id' is missing[缺少服务ID]"); + } + ApiServiceVo apiServiceVo = apiService.queryById(id,userName); + AssertUtil.notNull(apiServiceVo,"未找到数据服务，有可能已经被删除"); + if(!this.apiService.checkUserWorkspace(userName,apiServiceVo.getWorkspaceId().intValue()) ){ + return Message.error("'api service queryById for workspaceId' is wrong[该用户不属于该工作空间Id]"); + } + return Message.ok().data("result", apiServiceVo); + }, "/apiservice/queryById", "Fail to query page of service api[查询服务api失败]"); + } + + @GET + @Path("/checkPath") + public Response checkPath(@QueryParam("scriptPath") String scriptPath, @QueryParam("path") String path) { + //需要跨用户查询 + return ApiUtils.doAndResponse(() -> { + if (StringUtils.isBlank(scriptPath)) { + return Message.error("'api service scriptPath' is missing[缺少api脚本路径]"); + } + if (StringUtils.isBlank(path)) { + return Message.error("'api service path' is missing[缺少api路径]"); + } + Integer apiCount = apiService.queryCountByPath(scriptPath, path); + return Message.ok().data("result", 0 > Integer.valueOf(0).compareTo(apiCount)); + }, "/apiservice/checkPath", "Fail to check path of service api[校验服务api路径失败]"); + } + + @GET + @Path("/checkName") + public Response checkName(@QueryParam("name") String name) { + //需要跨用户查询 + return ApiUtils.doAndResponse(() -> { + if (StringUtils.isBlank(name)) { + return Message.error("'api service name' is missing[缺少api名称]"); + } + Integer count = apiService.queryCountByName(name); + return Message.ok().data("result", count > 0); + }, "/apiservice/checkName", "Fail to check name of service api[校验服务api名称失败]"); + } + + @GET + @Path("/apiDisable") + public Response apiDisable(@QueryParam("id") Long id, + @Context HttpServletRequest req) { + return ApiUtils.doAndResponse(() -> { + String userName = SecurityFilter.getLoginUsername(req); + if (null == id) { + return Message.error("'api service api id' is missing[缺少api id]"); + } + boolean resultFlag = apiService.disableApi(id,userName); + return Message.ok().data("result", resultFlag); + }, "/apiservice/apiDisable", "Fail to disable api[禁用api失败]"); + } + + @GET + @Path("/apiEnable") + public Response apiEnable(@QueryParam("id") Long id, + @Context HttpServletRequest req) { + return ApiUtils.doAndResponse(() -> { + String userName = SecurityFilter.getLoginUsername(req); + if (null == id) { + return Message.error("'api service api id' is missing[缺少api id]"); + } + boolean resultFlag = apiService.enableApi(id,userName); + return Message.ok().data("result", resultFlag); + }, "/apiservice/apiEnable", "Fail to enable api[启用api失败]"); + } + + @GET + @Path("/apiDelete") + public Response apiDelete(@QueryParam("id") Long id, + @Context HttpServletRequest req) { + //目前暂时不实际删除数据，只做不可见和不可用。 + return ApiUtils.doAndResponse(() -> { + String userName = SecurityFilter.getLoginUsername(req); + if (null == id) { + return Message.error("'api service api id' is missing[缺少api id]"); + } + boolean resultFlag = apiService.deleteApi(id,userName); + return Message.ok().data("result", resultFlag); + }, "/apiservice/apiDelete", "Fail to delete api[删除api失败]"); + } + + @POST + @Path("/apiCommentUpdate") + public Response apiCommentUpdate(@Context HttpServletRequest req, JsonNode json) { + //目前暂时不实际删除数据，只做不可见和不可用。 + Long id = json.get("id").getLongValue(); + String comment=json.get("comment").getTextValue(); + return ApiUtils.doAndResponse(() -> { + String userName = SecurityFilter.getLoginUsername(req); + if (null == id) { + return Message.error("'api service api id' is missing[缺少api id]"); + } + boolean resultFlag = apiService.updateComment(id,comment,userName); + return Message.ok().data("result", resultFlag); + }, "/apiservice/apiDelete", "Fail to delete api[删除api失败]"); + } + + + @GET + @Path("/apiParamQuery") + public Response apiParamQuery(@QueryParam("scriptPath") String scriptPath, + @QueryParam("versionId") Long versionId, + @Context HttpServletRequest req) { + return ApiUtils.doAndResponse(() -> { + String userName = SecurityFilter.getLoginUsername(req); + if (StringUtils.isEmpty(scriptPath)) { + return Message.error("'api service api scriptPath' is missing[缺少api scriptPath]"); + } + if (null == versionId) { + return Message.error("'api service api version' is missing[缺少api 版本号]"); + } + List queryParamVoList = apiServiceQueryService.queryParamList(scriptPath, versionId); + return Message.ok().data("result", queryParamVoList); + }, "/apiservice/apiParamQuery", "Fail to query api info[查询api信息失败]"); + } + + @GET + @Path("/apiVersionQuery") + public Response apiVersionQuery(@QueryParam("serviceId") Long serviceId, + @Context HttpServletRequest req) { + return ApiUtils.doAndResponse(() -> { + String userName = SecurityFilter.getLoginUsername(req); + if (null == serviceId) { + return Message.error("'api service api serviceId' is missing[缺少api serviceId]"); + } + List apiVersionVoList = apiServiceQueryService.queryApiVersionById(serviceId) + .stream().filter(apiVersionVo -> apiVersionVo.getCreator().equals(userName)) + .collect(Collectors.toList()); + return Message.ok().data("result", apiVersionVoList); + }, "/apiservice/apiVersionQuery", "Fail to query api version[查询api版本失败]"); + } + + @GET + @Path("/apiContentQuery") + public Response apiContentQuery(@QueryParam("versionId") Long versionId, + @Context HttpServletRequest req) { + String userName = SecurityFilter.getLoginUsername(req); + return ApiUtils.doAndResponse(() -> { + if (null== versionId) { + return Message.error("'api service api versionId' is missing[缺少api versionId]"); + } + ApiServiceVo apiServiceVo = apiServiceQueryService.queryByVersionId(userName,versionId); + if(!this.apiService.checkUserWorkspace(userName,apiServiceVo.getWorkspaceId().intValue()) ){ + return Message.error("'api service apiContentQuery for workspaceId' is wrong[该用户不属于该工作空间Id]"); + } + return Message.ok().data("result", apiServiceVo); + }, "/apiservice/apiContentQuery", "Fail to query api Content[查询api版本内容失败]"); + } +} diff --git a/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/restful/ApiServiceExecuteRestfulApi.java b/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/restful/ApiServiceExecuteRestfulApi.java new file mode 100644 index 0000000000..1b023dab07 --- /dev/null +++ b/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/restful/ApiServiceExecuteRestfulApi.java @@ -0,0 +1,319 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.apiservice.core.restful; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.JavaType; +import com.webank.wedatasphere.dss.apiservice.core.bo.*; +import com.webank.wedatasphere.dss.apiservice.core.config.ApiServiceConfiguration; +import com.webank.wedatasphere.dss.apiservice.core.exception.ApiServiceQueryException; +import com.webank.wedatasphere.dss.apiservice.core.execute.ExecuteCodeHelper; +import com.webank.wedatasphere.dss.apiservice.core.execute.LinkisJobSubmit; +import com.webank.wedatasphere.dss.apiservice.core.service.ApiServiceQueryService; +import com.webank.wedatasphere.dss.apiservice.core.token.JwtManager; +import com.webank.wedatasphere.dss.apiservice.core.util.ApiUtils; +import com.webank.wedatasphere.dss.apiservice.core.util.AssertUtil; +import com.webank.wedatasphere.dss.apiservice.core.vo.*; +import com.webank.wedatasphere.linkis.server.BDPJettyServerHelper; +import com.webank.wedatasphere.linkis.server.Message; +import com.webank.wedatasphere.linkis.server.security.SecurityFilter; +import com.webank.wedatasphere.linkis.ujes.client.UJESClient; +import com.webank.wedatasphere.linkis.ujes.client.response.JobExecuteResult; +import org.apache.commons.collections4.MapUtils; +import org.apache.commons.io.IOUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; +import org.springframework.util.StringUtils; +import org.springframework.web.bind.annotation.RequestBody; + +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; +import javax.ws.rs.*; +import javax.ws.rs.core.Context; +import javax.ws.rs.core.MediaType; +import javax.ws.rs.core.Response; +import java.io.IOException; +import java.io.InputStream; +import java.util.Enumeration; +import java.util.HashMap; +import java.util.Map; + +/** + * @author: jinyangrao + * @date: 2020-08-26 18:03:34 + */ + +@Path("/dss/apiservice") +@Consumes(MediaType.APPLICATION_JSON) +@Produces(MediaType.APPLICATION_JSON) +@Component +public class ApiServiceExecuteRestfulApi { + public static final String XLSX_RESPONSE_CONTENT_TYPE = "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"; + + private static final Logger logger = LoggerFactory.getLogger(ApiServiceExecuteRestfulApi.class); + @Autowired + private ApiServiceQueryService queryService; + + + private static final String SYS_COLUMN_PREFIX = "_"; + + private static final String requestBodyDemo = "{\"moduleName\":\"aladdin-demo\",\"params\":{\"param1\": \"value1\"}}"; + + + + @POST + @Path("/execute/{path:.*}") + public Response post(@PathParam("path") VariableString path, @RequestBody QueryRequest queryRequest, + @Context HttpServletRequest req) { + String userName = SecurityFilter.getLoginUsername(req); + return getResponse(userName,path.getPath(), queryRequest, HttpMethod.POST); + } + + @GET + @Path("/execute/{path:.*}") + public Response get(@PathParam("path") VariableString path, + @Context HttpServletRequest req) throws JsonProcessingException { + String userName = SecurityFilter.getLoginUsername(req); + + QueryRequest queryRequest = new QueryRequest(); + + + // 如果requestBody为空，尝试从url中获取参数 + if (MapUtils.isEmpty(queryRequest.getParams())) { + String paramSuffix = "params."; + Enumeration parameterNames = req.getParameterNames(); + Map params = new HashMap<>(); + while (parameterNames.hasMoreElements()) { + String name = parameterNames.nextElement(); + if (name.startsWith(paramSuffix)) { + params.put(name.substring(paramSuffix.length()), req.getParameter(name)); + } + } + queryRequest.setParams(params); + } + String paramsJsonStr = req.getParameter("params"); + JavaType javaType = BDPJettyServerHelper.jacksonJson().getTypeFactory().constructParametricType(Map.class,String.class,Object.class); + Map parmas = BDPJettyServerHelper.jacksonJson().readValue(paramsJsonStr, javaType); + queryRequest.setParams(parmas); + + if (StringUtils.isEmpty(queryRequest.getModuleName())) { + queryRequest.setModuleName(req.getParameter("moduleName")); + } + + return getResponse(userName,path.getPath(), queryRequest, HttpMethod.GET); + } + + @PUT + @Path("/execute/{path:.*}") + public Response put(@PathParam("path") VariableString path, @RequestBody QueryRequest queryRequest, + @Context HttpServletRequest req) { + String userName = SecurityFilter.getLoginUsername(req); + return getResponse(userName,path.getPath(), queryRequest, HttpMethod.PUT); + } + + @DELETE + @Path("/execute/{path:.*}") + public Response delete(@PathParam("path") VariableString path, @RequestBody QueryRequest queryRequest, + @Context HttpServletRequest req) { + String userName = SecurityFilter.getLoginUsername(req); + return getResponse(userName,path.getPath(), queryRequest, HttpMethod.DELETE); + } + + private void validParam(QueryRequest queryRequest) { + AssertUtil.notNull(queryRequest, "请求体不能为空，正确的格式:" + requestBodyDemo); + AssertUtil.notEmpty(queryRequest.getModuleName(), "moduleName不能为空，正确的格式:" + requestBodyDemo); + AssertUtil.notNull(queryRequest.getParams().get(ApiServiceConfiguration.API_SERVICE_TOKEN_KEY.getValue()),"请求token不能为空"); + } + + @GET + @Path("/getDirFileTrees") + public void getDirFileTrees(@Context HttpServletRequest req, @Context HttpServletResponse resp, + @QueryParam("path") String path, + @QueryParam("taskId") String taskId) throws IOException, ApiServiceQueryException { + String userName = SecurityFilter.getLoginUsername(req); + if (StringUtils.isEmpty(path)) { + throw new ApiServiceQueryException(80004, path); + } + String dirFileTree=""; + ApiServiceJob apiServiceJob = queryService.getJobByTaskId(taskId); + if(null != apiServiceJob && userName.equals(apiServiceJob.getSubmitUser())) { + JobExecuteResult jobExecuteResult = new JobExecuteResult(); + jobExecuteResult.setTaskID(taskId); + jobExecuteResult.setUser(apiServiceJob.getProxyUser()); + Map props = new HashMap<>(); + UJESClient client = LinkisJobSubmit.getClient(props); + + + dirFileTree = ExecuteCodeHelper.getResultList(jobExecuteResult, client, path); + }else{ + dirFileTree="当前用户不存在运行的TaskId: "+taskId; + } + resp.getWriter().println(dirFileTree); + resp.getWriter().flush(); + } + + + @GET + @Path("/openFile") + public void openFile(@Context HttpServletRequest req, + @QueryParam("path") String path, + @QueryParam("taskId") String taskId, + @DefaultValue("1") @QueryParam("page") Integer page, + @DefaultValue("5000") @QueryParam("pageSize") Integer pageSize, + @DefaultValue("utf-8") @QueryParam("charset") String charset, + @Context HttpServletResponse resp) throws IOException, ApiServiceQueryException { + String userName = SecurityFilter.getLoginUsername(req); + if (StringUtils.isEmpty(path)) { + throw new ApiServiceQueryException(80004, path); + } + if(StringUtils.isEmpty(taskId)){ + throw new ApiServiceQueryException(80005, "taskId is null"); + } + String fileContent=""; + ApiServiceJob apiServiceJob = queryService.getJobByTaskId(taskId); + if(null != apiServiceJob && userName.equals(apiServiceJob.getSubmitUser())) { + Map props = new HashMap<>(); + UJESClient client = LinkisJobSubmit.getClient(props); + fileContent = ExecuteCodeHelper.getResultContent(apiServiceJob.getProxyUser(), path, pageSize, client); + }else{ + fileContent="当前用户不存在运行的TaskId: "+taskId; + } + resp.getWriter().println(fileContent); + resp.getWriter().flush(); + } + + @GET + @Path("resultsetToExcel") + public void resultsetToExcel( + @Context HttpServletRequest req, + @Context HttpServletResponse resp, + @QueryParam("path") String path, + @QueryParam("taskId") String taskId, + @DefaultValue("utf-8") @QueryParam("charset") String charset, + @DefaultValue("csv") @QueryParam("outputFileType") String outputFileType, + @DefaultValue(",") @QueryParam("csvSeperator") String csvSeperator, + @DefaultValue("downloadResultset") @QueryParam("outputFileName") String outputFileName, + @DefaultValue("result") @QueryParam("sheetName") String sheetName, + @DefaultValue("NULL") @QueryParam("nullValue") String nullValue) throws ApiServiceQueryException, IOException { + + resp.addHeader("Content-Disposition", "attachment;filename=" + + new String(outputFileName.getBytes("UTF-8"), "ISO8859-1") + "." + outputFileType); + resp.setCharacterEncoding(charset); + + switch (outputFileType) { + case "csv": + resp.addHeader("Content-Type", "text/plain"); + break; + case "xlsx": + resp.addHeader("Content-Type", XLSX_RESPONSE_CONTENT_TYPE); + break; + default: + new ApiServiceQueryException(80015,"不支持的下载类型"); + + } + + String userName = SecurityFilter.getLoginUsername(req); + if (StringUtils.isEmpty(path)) { + throw new ApiServiceQueryException(80005, path); + } + InputStream inputStream; + ApiServiceJob apiServiceJob = queryService.getJobByTaskId(taskId); + if(null != apiServiceJob && userName.equals(apiServiceJob.getSubmitUser())) { + Map props = new HashMap<>(); + UJESClient client = LinkisJobSubmit.getClient(props); + inputStream = ExecuteCodeHelper.downloadResultSet(apiServiceJob.getProxyUser(), + path, + charset, + outputFileType, + csvSeperator, + outputFileName, + sheetName, + nullValue, + client); + } else{ + resp.getWriter().println("当前用户不存在运行的TaskId: "+taskId); + resp.getWriter().flush(); + return; + } + try { + IOUtils.copy(inputStream, resp.getOutputStream()); + resp.getOutputStream().flush(); + } finally { + IOUtils.closeQuietly(inputStream); + } + } + + @GET + @Path("/{id}/get") + public Response getTaskByID(@Context HttpServletRequest req, @PathParam("id") Long taskId) { + String username = SecurityFilter.getLoginUsername(req); + ApiServiceJob apiServiceJob = queryService.getJobByTaskId(taskId.toString()); + if (null != apiServiceJob && username.equals(apiServiceJob.getSubmitUser())) { + Map props = new HashMap<>(); + UJESClient client = LinkisJobSubmit.getClient(props); + JobExecuteResult jobExecuteResult = apiServiceJob.getJobExecuteResult(); + jobExecuteResult.setUser(apiServiceJob.getProxyUser()); + Map vo = ExecuteCodeHelper.getTaskInfoById(jobExecuteResult, client); + return Message.messageToResponse(Message.ok().data("task", vo)); + } else { + return Message.messageToResponse(Message.ok().data("task", null)); + } + } + + private Response getResponse(String user,String path, QueryRequest queryRequest, String httpMethod) { + return ApiUtils.doAndResponse(() -> { + + validParam(queryRequest); + String token = queryRequest.getParams().get(ApiServiceConfiguration.API_SERVICE_TOKEN_KEY.getValue()).toString(); + + MessageVo messageVo = null; + ApiServiceToken tokenDetail = null; + boolean isParseRight = true; + try { + tokenDetail = JwtManager.parseToken(token); + }catch (Exception e) { + isParseRight = false; + messageVo = new MessageVo().setData("token解析错误，该token无效！"); + } + if(false == isParseRight) { + return messageVo; + } + + if(tokenDetail.getApplyUser().equals(user)) { + + LinkisExecuteResult query = queryService.query("/" + path, + queryRequest.getParams() == null ? new HashMap<>() : queryRequest.getParams(), + queryRequest.getModuleName(), httpMethod,tokenDetail,user); + if(null == query) { + messageVo = new MessageVo().setMessage("用户任务执行出错，用户参数错误！").setStatus(1); + return messageVo; + } + + HashMap queryRes = new HashMap<>(); + queryRes.put("taskId",query.getTaskId()); + queryRes.put("execId",query.getExecId()); + messageVo = new MessageVo().setData(queryRes); + }else { + messageVo = new MessageVo().setData("Token is not correct"); + } + + return messageVo; + }); + } +} \ No newline at end of file diff --git a/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/restful/ApiServiceTokenRestfulApi.java b/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/restful/ApiServiceTokenRestfulApi.java new file mode 100644 index 0000000000..f25dd436c8 --- /dev/null +++ b/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/restful/ApiServiceTokenRestfulApi.java @@ -0,0 +1,136 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.apiservice.core.restful; + +import com.github.pagehelper.PageInfo; +import com.webank.wedatasphere.dss.apiservice.core.constant.SaveTokenEnum; +import com.webank.wedatasphere.dss.apiservice.core.datamap.DataMapStatus; +import com.webank.wedatasphere.dss.apiservice.core.service.ApprovalService; +import com.webank.wedatasphere.dss.apiservice.core.token.TokenAuth; +import com.webank.wedatasphere.dss.apiservice.core.service.TokenQueryService; +import com.webank.wedatasphere.dss.apiservice.core.util.ApiUtils; +import com.webank.wedatasphere.dss.apiservice.core.util.DateUtil; +import com.webank.wedatasphere.dss.apiservice.core.vo.ApprovalVo; +import com.webank.wedatasphere.dss.apiservice.core.vo.TokenManagerVo; +import com.webank.wedatasphere.dss.apiservice.core.bo.TokenQuery; +import com.webank.wedatasphere.linkis.server.Message; +import com.webank.wedatasphere.linkis.server.security.SecurityFilter; +import org.apache.commons.lang.StringUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; + +import javax.servlet.http.HttpServletRequest; +import javax.ws.rs.*; +import javax.ws.rs.core.Context; +import javax.ws.rs.core.MediaType; +import javax.ws.rs.core.Response; +import java.util.Calendar; +import java.util.Date; +import java.util.List; + +/** + * @author allenlliu + * @version 2.0.0 + * @date 2020/08/17 05:37 PM + */ + +@Path("/dss/apiservice") +@Consumes(MediaType.APPLICATION_JSON) +@Produces(MediaType.APPLICATION_JSON) +@Component +public class ApiServiceTokenRestfulApi { + private static final Logger LOG = LoggerFactory.getLogger(ApiServiceTokenRestfulApi.class); + + @Autowired + TokenQueryService tokenQueryService; + + @Autowired + ApprovalService approvalService; + + @Autowired + TokenAuth tokenAuth; + + + @GET + @Path("/tokenQuery") + public Response apiServiceTokenQuery(@QueryParam("apiId") Long apiId, + @QueryParam("user") String user, + @QueryParam("status") Integer status, + @QueryParam("startDate") String startDateStr, + @QueryParam("endDate") String endDateStr, + @QueryParam("currentPage") Integer currentPage, + @QueryParam("pageSize") Integer pageSize, + @Context HttpServletRequest req) { + String userName = SecurityFilter.getLoginUsername(req); + return ApiUtils.doAndResponse(() -> { + + TokenQuery query = null; + + if(StringUtils.isBlank(startDateStr) || StringUtils.isBlank(endDateStr)) { + query = new TokenQuery(apiId, user, status); + } else { + Long startDateSecond = Long.parseLong(startDateStr); + Long endDateSeconed = Long.parseLong(endDateStr); + String startDate = DateUtil.format(new Date(startDateSecond), DateUtil.FORMAT_LONG); + Date endDateReal = new Date(endDateSeconed); + Calendar cal = Calendar.getInstance(); + cal.setTime(endDateReal); + cal.add(Calendar.DATE, 1); + Date realDate = cal.getTime(); + String endDate = DateUtil.format(realDate, DateUtil.FORMAT_LONG); + + query = new TokenQuery(apiId, user, status, startDate, endDate); + + } + query.setCreator(userName); + query.setCurrentPage(null != currentPage ? currentPage : 1); + query.setPageSize(null != pageSize ? pageSize : 10); + PageInfo tokenList = tokenQueryService.query(query); + return Message.ok().data("queryList", tokenList.getList()).data("total", tokenList.getTotal()); + }, "/apiservice/tokenQuery", "Fail to query page of token[查询token信息失败]"); + } + + + @GET + @Path("/approvalRefresh") + public Response refresh(@QueryParam("approvalNo") String approvalNo, + @Context HttpServletRequest req) { + String userName = SecurityFilter.getLoginUsername(req); + return ApiUtils.doAndResponse(() ->{ + + ApprovalVo approvalVo = approvalService.refreshStatus(approvalNo); + if(approvalVo.getCreator().equals(userName) == false){ + return Message.error("'api service user check failed' [用户检查失败！]"); + } + if(null == approvalVo) { + return Message.error("'api service approvalNo' is missing[未查询到对应的审批单号]"); + } + if(approvalVo.getStatus().equals(DataMapStatus.SUCCESS.getIndex())) { + List tokenManagerVoList = tokenAuth.genTokenRecord(approvalVo); + + SaveTokenEnum res = tokenAuth.saveTokensToDb(tokenManagerVoList, approvalNo); + if(SaveTokenEnum.FAILED == res) { + return Message.error("'token auth process occur error' [token认证过程出现错误]"); + } + } + String approvalStatusDesc = DataMapStatus.getDescByIndex(approvalVo.getStatus()); + return Message.ok().data("approvalStatus", approvalStatusDesc); + }, "/apiservice/approvalRefresh/", "提交审批单查询出错"); + } +} diff --git a/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/service/ApiService.java b/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/service/ApiService.java new file mode 100644 index 0000000000..28b6ebb8ae --- /dev/null +++ b/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/service/ApiService.java @@ -0,0 +1,100 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.apiservice.core.service; + +import com.webank.wedatasphere.dss.apiservice.core.bo.ApiServiceQuery; +import com.webank.wedatasphere.dss.apiservice.core.exception.ApiServiceQueryException; +import com.webank.wedatasphere.dss.apiservice.core.vo.*; + +import java.util.List; + +/** + * @author zhulixin + */ +public interface ApiService { + + /** + * Save + * + * @param oneService oneService info + */ + void save(ApiServiceVo oneService) throws Exception; + + + + void saveByApp(ApiServiceVo apiService) throws Exception; + + /** + * Update + * + * @param oneService oneService info + */ + void update(ApiServiceVo oneService) throws Exception; + + /** + * query + * + * @param apiServiceQuery + * @return + */ + List query(ApiServiceQuery apiServiceQuery) throws ApiServiceQueryException; + + List queryByWorkspaceId(Integer workspaceId, String userName); + + List queryAllTags(String userName,Integer workspaceId); + + + /** + * query + * + * @param scriptPath + * @return + */ + ApiServiceVo queryByScriptPath(String scriptPath); + + Integer queryCountByPath(String scriptPath, String path); + + Integer queryCountByName(String name); + + /** + * enable api + * @param id api record id + * @return + */ + Boolean enableApi(Long id,String userName); + + /** + * disable api + * @param id api record id + * @return + */ + Boolean disableApi(Long id,String userName); + + Boolean deleteApi(Long id,String userName); + + Boolean updateComment(Long id,String comment,String userName); + + ApiServiceVo queryById(Long id,String userName); + + ApiVersionVo getMaxVersion(long serviceId); + + boolean checkUserWorkspace(String userName,Integer workspaceId); + +// List genDataMapApplyContentDatas(ApiServiceVo apiServiceVo, ApiVersionVo apiVersionVo, String metaDtaInfo); + + +} diff --git a/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/service/ApiServiceQueryService.java b/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/service/ApiServiceQueryService.java new file mode 100644 index 0000000000..310cba28f7 --- /dev/null +++ b/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/service/ApiServiceQueryService.java @@ -0,0 +1,58 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package com.webank.wedatasphere.dss.apiservice.core.service; + +import com.webank.wedatasphere.dss.apiservice.core.bo.ApiServiceJob; +import com.webank.wedatasphere.dss.apiservice.core.bo.ApiServiceToken; +import com.webank.wedatasphere.dss.apiservice.core.bo.LinkisExecuteResult; +import com.webank.wedatasphere.dss.apiservice.core.exception.ApiServiceQueryException; +import com.webank.wedatasphere.dss.apiservice.core.vo.*; + +import java.util.List; +import java.util.Map; + +/** + * 接口调用service + * + * @author lidongzhang + */ +public interface ApiServiceQueryService { + /** + * 执行查询 + * + * @param path path + * @param params params + * @param moduleName moduleName + * @param httpMethod httpMethod + * @return 查询接口 + */ + LinkisExecuteResult query(String path, Map params, String moduleName, String httpMethod, ApiServiceToken tokenDetail, String loginUser); + + List queryParamList(String scriptPath, Long versionId); + + /** + * 查询api 版本信息 + * + * @param serviceId + * @return + */ + List queryApiVersionById(Long serviceId); + + ApiServiceVo queryByVersionId(String userName,Long versionId) throws ApiServiceQueryException; + + ApiServiceJob getJobByTaskId(String taskId); + +} diff --git a/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/service/ApprovalService.java b/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/service/ApprovalService.java new file mode 100644 index 0000000000..248228ea46 --- /dev/null +++ b/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/service/ApprovalService.java @@ -0,0 +1,35 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.apiservice.core.service; + +import com.webank.wedatasphere.dss.apiservice.core.vo.ApprovalVo; + +/** + * @author: jinyangrao + */ +public interface ApprovalService { + + /** + * 根据审批单号，查询对应数据 + * */ + ApprovalVo query(String approvalNo); + + /** + * 查询DataMap，依据DataMap审批结果更新审批单状态 + * */ + ApprovalVo refreshStatus(String approvalNo) throws Exception; +} diff --git a/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/service/TokenQueryService.java b/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/service/TokenQueryService.java new file mode 100644 index 0000000000..58e0a30143 --- /dev/null +++ b/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/service/TokenQueryService.java @@ -0,0 +1,31 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.apiservice.core.service; + +import com.github.pagehelper.PageInfo; +import com.webank.wedatasphere.dss.apiservice.core.vo.TokenManagerVo; +import com.webank.wedatasphere.dss.apiservice.core.bo.TokenQuery; + + +public interface TokenQueryService { + + /** + * 查询 + * */ + PageInfo query(TokenQuery tokenQuery); + +} diff --git a/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/service/impl/ApiServiceImpl.java b/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/service/impl/ApiServiceImpl.java new file mode 100644 index 0000000000..7d17636025 --- /dev/null +++ b/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/service/impl/ApiServiceImpl.java @@ -0,0 +1,664 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package com.webank.wedatasphere.dss.apiservice.core.service.impl; + +import com.google.common.base.Splitter; +import com.webank.wedatasphere.dss.apiservice.core.bo.ApiServiceQuery; +import com.webank.wedatasphere.dss.apiservice.core.bo.ApiServiceToken; +import com.webank.wedatasphere.dss.apiservice.core.constant.ApiCommonConstant; +import com.webank.wedatasphere.dss.apiservice.core.constant.SQLMetadataInfoCheckStatus; +import com.webank.wedatasphere.dss.apiservice.core.constant.SaveTokenEnum; +import com.webank.wedatasphere.dss.apiservice.core.dao.*; +import com.webank.wedatasphere.dss.apiservice.core.datamap.DataMapStatus; +import com.webank.wedatasphere.dss.apiservice.core.datamap.UUIDGenerator; +import com.webank.wedatasphere.dss.apiservice.core.exception.ApiServiceQueryException; +import com.webank.wedatasphere.dss.apiservice.core.exception.ApiServiceTokenException; +import com.webank.wedatasphere.dss.apiservice.core.execute.ExecuteCodeHelper; +import com.webank.wedatasphere.dss.apiservice.core.execute.LinkisJobSubmit; +import com.webank.wedatasphere.dss.apiservice.core.token.JwtManager; +import com.webank.wedatasphere.dss.apiservice.core.token.TokenAuth; +import com.webank.wedatasphere.dss.apiservice.core.vo.*; +import com.webank.wedatasphere.dss.apiservice.core.service.ApiService; +import com.webank.wedatasphere.linkis.bml.client.BmlClient; +import com.webank.wedatasphere.linkis.bml.client.BmlClientFactory; +import com.webank.wedatasphere.linkis.bml.protocol.BmlUpdateResponse; +import com.webank.wedatasphere.linkis.bml.protocol.BmlUploadResponse; +import com.webank.wedatasphere.linkis.common.exception.ErrorException; +import com.webank.wedatasphere.linkis.common.io.FsPath; +import com.webank.wedatasphere.linkis.storage.script.ScriptFsWriter; +import com.webank.wedatasphere.linkis.storage.script.ScriptMetaData; +import com.webank.wedatasphere.linkis.storage.script.ScriptRecord; +import com.webank.wedatasphere.linkis.storage.script.Variable; +import com.webank.wedatasphere.linkis.storage.script.VariableParser; +import com.webank.wedatasphere.linkis.storage.script.writer.StorageScriptFsWriter; +import com.webank.wedatasphere.linkis.ujes.client.UJESClient; +import org.apache.commons.lang.StringUtils; +import org.apache.http.Consts; +import org.apache.ibatis.annotations.Param; +import org.mortbay.log.Log; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; +import org.springframework.transaction.annotation.Transactional; + +import javax.annotation.PostConstruct; +import java.io.IOException; +import java.io.InputStream; +import java.util.*; +import java.util.stream.Collectors; + +/** + * service impl + * + * @author zhulixin + */ +@Service +public class ApiServiceImpl implements ApiService { + + private static final Logger LOG = LoggerFactory.getLogger(ApiServiceImpl.class); + + + @Autowired + private ApiServiceDao apiServiceDao; + + @Autowired + private ApiServiceParamDao apiServiceParamDao; + + @Autowired + private ApiServiceVersionDao apiServiceVersionDao; + + @Autowired + private ApiServiceTokenManagerDao apiServiceTokenManagerDao; + + @Autowired + TokenAuth tokenAuth; + + @Autowired + ApiServiceApprovalDao apiServiceApprovalDao; + + + /** + * Bml client + */ + private BmlClient client; + + private UJESClient ujesClient; + + @PostConstruct + public void buildClient() { + LOG.info("build client start ======"); + client = BmlClientFactory.createBmlClient(); + Map props = new HashMap<>(); + ujesClient = LinkisJobSubmit.getClient(props); + LOG.info("build client end ======="); + } + + @Override + @Transactional(rollbackFor = Exception.class) + public void save(ApiServiceVo apiService) throws Exception { + String user = apiService.getCreator(); + String resourceId = null; + try { + // check script path if already created + String scriptPath = apiService.getScriptPath(); + + // upload to bml + Map uploadResult = uploadBml(user, scriptPath, + apiService.getMetadata(), apiService.getContent()); + + // insert linkis_oneservice_config + String version = uploadResult.get("version"); + resourceId = uploadResult.get("resourceId"); + apiServiceDao.insert(apiService); + + //insert into version + ApiVersionVo apiVersionVo = new ApiVersionVo(); + apiVersionVo.setApiId(apiService.getId()); + apiVersionVo.setBmlResourceId(resourceId); + apiVersionVo.setBmlVersion(version); + apiVersionVo.setVersion(version); + apiVersionVo.setCreator(user); + apiVersionVo.setCreateTime(Calendar.getInstance().getTime()); + apiVersionVo.setSource(apiService.getScriptPath()); + //1为正常 0为禁用 + apiVersionVo.setStatus(1); + + + //生成审批记录，必须使用发布用户执行sql + sendApprovalToDM(user,apiService,apiVersionVo); + //顺序不能改变，版本信息依赖审批单信息 + //todo update by query + apiVersionVo.setMetadataInfo("default"); + apiServiceVersionDao.insert(apiVersionVo); + + addApprovalToDB(apiService,apiVersionVo.getId(),apiVersionVo.getAuthId()); + + // insert linkis_oneservice_params + List params = apiService.getParams(); + if (params != null && !params.isEmpty()) { + for (ParamVo param : params) { + param.setApiVersionId(apiVersionVo.getId()); + apiServiceParamDao.insert(param); + } + } + + //insert a token record for self + genTokenForPublisher(apiService,apiVersionVo.getId()); + } catch (Exception e) { + LOG.error("one service insert error", e); + if (StringUtils.isNotBlank(resourceId)) { +// removeBml(user, resourceId); + } + if (e.getCause() instanceof ErrorException) { + throw (ErrorException) e.getCause(); + } + throw e; + } + } + + + @Override + @Transactional(rollbackFor = Exception.class) + public void saveByApp(ApiServiceVo apiService) throws Exception { + String user = apiService.getCreator(); + String resourceId = null; + try { + // check script path if already created + String scriptPath = apiService.getScriptPath(); + + // upload to bml + Map uploadResult = uploadBml(user, scriptPath, + apiService.getMetadata(), apiService.getContent()); + + // insert linkis_oneservice_config + String version = uploadResult.get("version"); + resourceId = uploadResult.get("resourceId"); + apiServiceDao.insert(apiService); + + //insert into version + ApiVersionVo apiVersionVo = new ApiVersionVo(); + apiVersionVo.setApiId(apiService.getId()); + apiVersionVo.setBmlResourceId(resourceId); + apiVersionVo.setBmlVersion(version); + apiVersionVo.setVersion(version); + apiVersionVo.setCreator(user); + apiVersionVo.setCreateTime(Calendar.getInstance().getTime()); + apiVersionVo.setSource(apiService.getScriptPath()); + //1为正常 0为禁用 + apiVersionVo.setStatus(1); + + + //生成审批记录，必须使用发布用户执行sql +// checkApprovalFromDM(user,apiService,apiVersionVo); + //顺序不能改变，版本信息依赖审批单信息 + apiServiceVersionDao.insert(apiVersionVo); + + addApprovalToDB(apiService,apiVersionVo.getId(),apiVersionVo.getAuthId()); + + // insert linkis_oneservice_params + List params = apiService.getParams(); + if (params != null && !params.isEmpty()) { + for (ParamVo param : params) { + param.setApiVersionId(apiVersionVo.getId()); + apiServiceParamDao.insert(param); + } + } + + //insert a token record for self + genTokenForPublisher(apiService,apiVersionVo.getId()); + } catch (Exception e) { + LOG.error("one service insert error", e); + if (StringUtils.isNotBlank(resourceId)) { +// removeBml(user, resourceId); + } + if (e.getCause() instanceof ErrorException) { + throw (ErrorException) e.getCause(); + } + throw e; + } + } + + + + + + @Override + @Transactional(rollbackFor = Exception.class) + public void update(ApiServiceVo apiService) throws Exception { + try { + if(null!=apiService.getTargetServiceId()) { + ApiVersionVo maxTargetApiVersionVo = getMaxVersion(apiService.getTargetServiceId()); + if(!checkUserWorkspace(apiService.getModifier(),apiService.getWorkspaceId().intValue())){ + throw new ApiServiceQueryException(800035,"Only can update the api service by owner workspace! "); + } + if(maxTargetApiVersionVo.getCreator().equals(apiService.getModifier())) { + Map updateResult = updateBml(apiService.getModifier(), maxTargetApiVersionVo.getBmlResourceId(), + apiService.getScriptPath(), apiService.getMetadata(), apiService.getContent()); + apiService.setCreator(maxTargetApiVersionVo.getCreator()); + apiService.setId(maxTargetApiVersionVo.getApiId()); + apiServiceDao.updateToTarget(apiService); + Log.info("Update to other Api Service, ID: " + apiService.getTargetServiceId() + ",resourceId:" + maxTargetApiVersionVo.getBmlResourceId()); + + + String version = updateResult.get("version"); + String resourceId = updateResult.get("resourceId"); + + + //update api version + //insert into version + ApiVersionVo apiServiceVersionVo = new ApiVersionVo(); + apiServiceVersionVo.setApiId(apiService.getId()); + apiServiceVersionVo.setBmlResourceId(resourceId); + apiServiceVersionVo.setBmlVersion(version); + apiServiceVersionVo.setVersion(version); + apiServiceVersionVo.setCreator(apiService.getModifier()); + apiServiceVersionVo.setCreateTime(Calendar.getInstance().getTime()); + apiServiceVersionVo.setSource(apiService.getScriptPath()); + //0默认已禁用 1为正常 + apiServiceVersionVo.setStatus(1); + //生成审批记录 + sendApprovalToDM(apiService.getModifier(), apiService, apiServiceVersionVo); + //todo update by query + apiServiceVersionVo.setMetadataInfo("null"); + //顺序不能改变，版本信息依赖审批单信息 + apiServiceVersionDao.insert(apiServiceVersionVo); + + //改变历史版本状态 + apiServiceVersionDao.updateApiVersionStatusById(maxTargetApiVersionVo.getId(), 0); + + //改变历史Token状态 + apiServiceTokenManagerDao.disableTokenStatusByVersionId(maxTargetApiVersionVo.getId()); + // insert params + List params = apiService.getParams(); + if (params != null && !params.isEmpty()) { + for (ParamVo param : params) { + param.setApiVersionId(apiServiceVersionVo.getId()); + apiServiceParamDao.insert(param); + } + } + + addApprovalToDB(apiService, apiServiceVersionVo.getId(), apiServiceVersionVo.getAuthId()); + + //insert a token record for self + genTokenForPublisher(apiService, apiServiceVersionVo.getId()); + }else { + throw new ApiServiceQueryException(800036,"Only can update the api service by owner! "); + } + }else { + throw new ApiServiceQueryException(800037,"Target service id can not be null for update"); + } + + } catch (Exception e) { + LOG.error("api service update error", e); + if (e.getCause() instanceof ErrorException) { + throw (ErrorException) e.getCause(); + } + throw e; + } + } + + @Override + public List query(ApiServiceQuery apiServiceQuery) throws ApiServiceQueryException { + //todo 查询需要优化，量小时不影响效率 + List queryList = apiServiceDao.query(apiServiceQuery); + + //add auth check + List userTokenManagerVos = apiServiceTokenManagerDao.queryByApplyUser(apiServiceQuery.getUserName()); + + List authQueryList = queryList.stream().filter(apiServiceVo -> { + TokenManagerVo findUserTokenManagerVo = userTokenManagerVos.stream().filter(userTokenManagerVo -> + userTokenManagerVo.getApiId().equals(apiServiceVo.getId()) + && userTokenManagerVo.getUser().equals(apiServiceQuery.getUserName()) + ).findAny().orElse(null); + if (null != findUserTokenManagerVo) { + //检查token状态为有效的 + if(findUserTokenManagerVo.getStatus().equals(1) || findUserTokenManagerVo.getPublisher().equals(apiServiceVo.getCreator())) { + return true; + }else { + return false; + } + } else { + return false; + } + }).collect(Collectors.toList()); + + // query param + if (authQueryList != null && !authQueryList.isEmpty()) { + for (ApiServiceVo apiServiceVo : authQueryList) { + ApiVersionVo maxApiVersionVo = getMaxVersion(apiServiceVo.getId()); + if(null == maxApiVersionVo){ + throw new ApiServiceQueryException(800032,"数据服务API版本记录为空"+apiServiceVo.getName()); + } + apiServiceVo.setParams(apiServiceParamDao.queryByVersionId(maxApiVersionVo.getId())); + apiServiceVo.setLatestVersionId(maxApiVersionVo.getId()); + } + } + + return authQueryList; + } + + @Override + public List queryByWorkspaceId(Integer workspaceId, String userName){ + List result = apiServiceDao.queryByWorkspaceId(workspaceId,userName); + result.stream().forEach(apiServiceVo -> { + ApiVersionVo apiVersionVo = getMaxVersion(apiServiceVo.getId()); + ApprovalVo approvalVo =apiServiceApprovalDao.queryByVersionId(apiVersionVo.getId()); + apiServiceVo.setApprovalVo(approvalVo); + }); + return result; + } + + + @Override + public ApiServiceVo queryById(Long id,String userName) { + ApiServiceVo apiServiceVo = apiServiceDao.queryById(id); + ApiVersionVo maxApiVersionVo = getMaxVersion(apiServiceVo.getId()); + List userTokenManagerVos = apiServiceTokenManagerDao.queryByApplyUserAndVersionId(userName,maxApiVersionVo.getId()); + if(userTokenManagerVos.size()>0) { + // query param + if (apiServiceVo != null) { + apiServiceVo.setParams(apiServiceParamDao.queryByVersionId(maxApiVersionVo.getId())); + apiServiceVo.setUserToken(userTokenManagerVos.get(0).getToken()); + apiServiceVo.setLatestVersionId(maxApiVersionVo.getId()); + apiServiceVo.setApprovalVo(apiServiceApprovalDao.queryByVersionId(maxApiVersionVo.getId())); + } + }else { + return null; + } + return apiServiceVo; + } + + + @Override + public List queryAllTags(String userName,Integer workspaceId) { + //todo 会有历史版本的tag + List tags = apiServiceDao.queryAllTags(userName,workspaceId); + List tagList= + tags.stream().filter(tag->!StringUtils.isEmpty(tag)).map(tag -> Splitter.on(",").splitToList(tag)).flatMap(List::stream).distinct() + .collect(Collectors.toList()); + return tagList; + } + + + @Override + public ApiServiceVo queryByScriptPath(@Param("scriptPath") String scriptPath) { + List apiServiceList = apiServiceDao.queryByScriptPath(scriptPath); + ApiServiceVo latestApiService = apiServiceList.stream().max(Comparator.comparing(ApiServiceVo::getModifyTime)).orElse(null); + if(null == latestApiService){ + return null; + } + List apiVersionVos =apiServiceVersionDao.queryApiVersionByApiServiceId(latestApiService.getId()); + ApiVersionVo maxVersion =apiVersionVos.stream().max(Comparator.comparing(ApiVersionVo::getVersion)).orElse(null); + // query param + if (latestApiService != null && null != maxVersion ) { + latestApiService.setParams(apiServiceParamDao.queryByVersionId(maxVersion.getId())); + latestApiService.setLatestVersionId(maxVersion.getId()); + } + return latestApiService; + } + + + + + + + @Override + public Integer queryCountByPath(String scriptPath, String path) { + return apiServiceDao.queryCountByPath(scriptPath, path); + } + + @Override + public Integer queryCountByName(String name) { + return apiServiceDao.queryCountByName(name); + } + + @Override + public Boolean enableApi(Long id,String userName) { + ApiServiceVo apiServiceVo = apiServiceDao.queryById(id); + if(!checkUserWorkspace(userName,apiServiceVo.getWorkspaceId().intValue())){ + LOG.error("api service check workspace error"); + return false; + } + if(apiServiceVo.getCreator().equals(userName)) { + + Integer updateCount = apiServiceDao.enableApi(id); + List targetApiVersionList = apiServiceVersionDao.queryApiVersionByApiServiceId(id); + ApiVersionVo maxTargetApiVersionVo = targetApiVersionList.stream().max(Comparator.comparing(ApiVersionVo::getVersion)).orElse(null); + + apiServiceTokenManagerDao.enableTokenStatusByVersionId(maxTargetApiVersionVo.getId()); + apiServiceVersionDao.updateApiVersionStatusById(maxTargetApiVersionVo.getId(), 1); + return updateCount > 0; + }else { + return false; + } + } + + @Override + public Boolean disableApi(Long id,String userName) { + ApiServiceVo apiServiceVo = apiServiceDao.queryById(id); + if(!checkUserWorkspace(userName,apiServiceVo.getWorkspaceId().intValue())){ + LOG.error("api service check workspace error"); + return false; + } + if(apiServiceVo.getCreator().equals(userName)) { + Integer updateCount = apiServiceDao.disableApi(id); + apiServiceTokenManagerDao.disableTokenStatusByApiId(id); + apiServiceVersionDao.updateAllApiVersionStatusByApiServiceId(id, 0); + return updateCount > 0; + }else { + return false; + } + } + + + @Override + public Boolean deleteApi(Long id,String userName) { + ApiServiceVo apiServiceVo = apiServiceDao.queryById(id); + if(!checkUserWorkspace(userName,apiServiceVo.getWorkspaceId().intValue())){ + LOG.error("api service check workspace error"); + return false; + } + if(apiServiceVo.getCreator().equals(userName)) { + Integer updateCount = apiServiceDao.deleteApi(id); + apiServiceTokenManagerDao.disableTokenStatusByApiId(id); + apiServiceVersionDao.updateAllApiVersionStatusByApiServiceId(id, 0); + return updateCount > 0; + }else { + return false; + } + } + + @Override + public Boolean updateComment(Long id, String comment, String userName) { + ApiServiceVo apiServiceVo = apiServiceDao.queryById(id); + if(!checkUserWorkspace(userName,apiServiceVo.getWorkspaceId().intValue())){ + LOG.error("api service check workspace error"); + return false; + } + if(apiServiceVo.getCreator().equals(userName)) { + Integer updateCount = apiServiceDao.updateApiServiceComment(id,comment); + return updateCount > 0; + }else { + return false; + } + } + + private Map uploadBml(String userName, String scriptPath, Map metadata, String scriptContent) { + try { + ScriptFsWriter writer = StorageScriptFsWriter.getScriptFsWriter(new FsPath(scriptPath), Consts.UTF_8.toString(), null); + List variableList=null; + if(metadata.entrySet().size() >0) { + Variable[] v = VariableParser.getVariables(metadata); + variableList = Arrays.stream(v).filter(var -> !StringUtils.isEmpty(var.value())).collect(Collectors.toList()); + + } + if(variableList!=null) { + writer.addMetaData(new ScriptMetaData(variableList.toArray(new Variable[0]))); + }else { + writer.addMetaData(null); + } + writer.addRecord(new ScriptRecord(scriptContent)); + InputStream inputStream = writer.getInputStream(); + // 新增文件 + BmlUploadResponse resource = client.uploadResource(userName, scriptPath, inputStream); + if (!resource.isSuccess()) { + throw new IOException("upload bml error"); + } + Map result = new HashMap<>(); + result.put("resourceId", resource.resourceId()); + result.put("version", resource.version()); + return result; + } catch (IOException e) { + LOG.error("upload bml error", e); + throw new RuntimeException(e); + } + } + + private Map updateBml(String userName, String resourceId, String scriptPath, Map metadata, String scriptContent) { + try { + ScriptFsWriter writer = StorageScriptFsWriter.getScriptFsWriter(new FsPath(scriptPath), Consts.UTF_8.toString(), null); + Variable[] v = VariableParser.getVariables(metadata); + List variableList = Arrays.stream(v).filter(var -> !StringUtils.isEmpty(var.value())).collect(Collectors.toList()); + writer.addMetaData(new ScriptMetaData(variableList.toArray(new Variable[0]))); + writer.addRecord(new ScriptRecord(scriptContent)); + InputStream inputStream = writer.getInputStream(); + + // 更新文件 + BmlUpdateResponse resource = client.updateResource(userName, resourceId, "", inputStream); + if (!resource.isSuccess()) { + throw new IOException("update bml error"); + } + Map result = new HashMap<>(); + result.put("resourceId", resource.resourceId()); + result.put("version", resource.version()); + return result; + } catch (IOException e) { + LOG.error("update bml error", e); + throw new RuntimeException(e); + } + } + + private void removeBml(String userName, String resourceId) { + try { + LOG.info("delete bml resource: userName: " + userName + ", resourceId: " + resourceId); + client.deleteResource(userName, resourceId); + } catch (Exception e) { + LOG.error("remove bml error", e); + } + } + + + public SQLMetadataInfoCheckStatus sendApprovalToDM(String user, ApiServiceVo apiService, ApiVersionVo apiVersionVo) throws Exception { + // 需要把发布者的SQL脚本内容，解析出对应的metadata库表信息 + +// ApprovalVo approvalVo = apiService.getApprovalVo(); + String approvalNo = UUIDGenerator.genUUID(); + apiVersionVo.setAuthId(approvalNo); + return SQLMetadataInfoCheckStatus.LEGAL; + } + + + public void addApprovalToDB(ApiServiceVo apiService,Long apiVersionId,String approvalNumber){ + + ApprovalVo approvalVo = apiService.getApprovalVo(); + // 需要插入审批单记录 + approvalVo.setApiId(apiService.getId()); + approvalVo.setApiVersionId(apiVersionId); + approvalVo.setApprovalNo(approvalNumber); + + approvalVo.setCreator(apiService.getCreator()); + + approvalVo.setStatus(DataMapStatus.INITED.getIndex()); + approvalVo.setCreateTime(new Date()); + approvalVo.setUpdateTime(new Date()); + if(StringUtils.isEmpty(approvalVo.getApprovalName())){ + approvalVo.setApprovalName("default"); + } + if(StringUtils.isEmpty(approvalVo.getExecuteUser())) { + approvalVo.setExecuteUser(approvalVo.getCreator()); + } + apiServiceApprovalDao.insert(approvalVo); + + } + + public void genTokenForPublisher(ApiServiceVo apiService,Long apiVersionId) throws ApiServiceTokenException { + + List tokenManagerVoList = genTokenForAccessApi(apiService,apiVersionId); + SaveTokenEnum res = tokenAuth.saveTokensToDb(tokenManagerVoList, ApiCommonConstant.DEFAULT_APPROVAL_NO); + } + + /** + * 为申请用户和自己生成访问token + * @param apiService + * @param apiVersionId + * @return + */ + private List genTokenForAccessApi(ApiServiceVo apiService,Long apiVersionId){ + List tokenManagerVoList = new ArrayList<>(); + List userNamesTmp = Arrays.asList(apiService.getApprovalVo().getApplyUser().split(",")); + List userNames = new ArrayList<>(userNamesTmp); + userNames.add(apiService.getCreator()); + userNames.stream().forEach(userName ->{ + TokenManagerVo tokenManagerVo = new TokenManagerVo(); + tokenManagerVo.setApiId(apiService.getId()); + tokenManagerVo.setApiVersionId(apiVersionId); + tokenManagerVo.setApplyTime(Calendar.getInstance().getTime()); + tokenManagerVo.setDuration(365L); + tokenManagerVo.setReason("create api service"); + tokenManagerVo.setStatus(1); + tokenManagerVo.setIpWhitelist(""); + tokenManagerVo.setCaller("scriptis"); + tokenManagerVo.setUser(userName); + tokenManagerVo.setPublisher(apiService.getCreator()); + + ApiServiceToken apiServiceToken = new ApiServiceToken(); + apiServiceToken.setApplyUser(userName); + apiServiceToken.setPublisher(apiService.getCreator()); + apiServiceToken.setApplyTime(tokenManagerVo.getApplyTime()); + apiServiceToken.setApiServiceId(tokenManagerVo.getApiId()); + + tokenManagerVo.setToken(JwtManager.createToken(userName,apiServiceToken,tokenManagerVo.getDuration())); + //审批单号使用默认的，自己给自己授权 + tokenManagerVo.setApplySource(ApiCommonConstant.DEFAULT_APPROVAL_NO); + tokenManagerVoList.add(tokenManagerVo); + }); + + return tokenManagerVoList; + } + + + @Override + public ApiVersionVo getMaxVersion(long serviceId){ + List apiVersionVoList = apiServiceVersionDao.queryApiVersionByApiServiceId(serviceId); + ApiVersionVo maxApiVersionVo = apiVersionVoList.stream().max(Comparator.comparing(ApiVersionVo::getVersion)).orElse(null); + return maxApiVersionVo; + } + + @Override + public boolean checkUserWorkspace(String userName,Integer workspaceId){ + //todo cache user workspaceIds + return true; +// String workspaceIds = ExecuteCodeHelper.getUserWorkspaceIds(userName,ujesClient); +// if(Arrays.stream(workspaceIds.split(",")).map(Integer::valueOf).collect(Collectors.toList()).contains(workspaceId)){ +// return true; +// }else { +// return false; +// } + + } + +} diff --git a/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/service/impl/ApiServiceQueryServiceImpl.java b/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/service/impl/ApiServiceQueryServiceImpl.java new file mode 100644 index 0000000000..2da0401a6f --- /dev/null +++ b/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/service/impl/ApiServiceQueryServiceImpl.java @@ -0,0 +1,559 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package com.webank.wedatasphere.dss.apiservice.core.service.impl; + +import com.google.common.cache.Cache; +import com.google.common.cache.CacheBuilder; +import com.google.common.cache.RemovalCause; +import com.webank.wedatasphere.dss.apiservice.core.bo.ApiServiceJob; +import com.webank.wedatasphere.dss.apiservice.core.bo.ApiServiceToken; +import com.webank.wedatasphere.dss.apiservice.core.bo.LinkisExecuteResult; +import com.webank.wedatasphere.dss.apiservice.core.config.ApiServiceConfiguration; +import com.webank.wedatasphere.dss.apiservice.core.constant.ParamType; +import com.webank.wedatasphere.dss.apiservice.core.constant.ParamTypeEnum; +import com.webank.wedatasphere.dss.apiservice.core.constant.RequireEnum; +import com.webank.wedatasphere.dss.apiservice.core.dao.*; +import com.webank.wedatasphere.dss.apiservice.core.exception.ApiExecuteException; +import com.webank.wedatasphere.dss.apiservice.core.exception.ApiServiceQueryException; +import com.webank.wedatasphere.dss.apiservice.core.execute.ApiServiceExecuteJob; +import com.webank.wedatasphere.dss.apiservice.core.execute.DefaultApiServiceJob; +import com.webank.wedatasphere.dss.apiservice.core.execute.ExecuteCodeHelper; +import com.webank.wedatasphere.dss.apiservice.core.execute.LinkisJobSubmit; +import com.webank.wedatasphere.dss.apiservice.core.jdbc.DatasourceService; +import com.webank.wedatasphere.dss.apiservice.core.service.ApiService; +import com.webank.wedatasphere.dss.apiservice.core.util.DateUtil; +import com.webank.wedatasphere.dss.apiservice.core.util.SQLCheckUtil; +import com.webank.wedatasphere.dss.apiservice.core.vo.*; +//import com.webank.wedatasphere.dss.oneservice.core.jdbc.JdbcUtil; +import com.webank.wedatasphere.dss.apiservice.core.exception.ApiServiceRuntimeException; +import com.webank.wedatasphere.dss.apiservice.core.service.ApiServiceQueryService; +import com.webank.wedatasphere.dss.apiservice.core.util.AssertUtil; +import com.webank.wedatasphere.dss.apiservice.core.util.ModelMapperUtil; +//import com.webank.wedatasphere.dss.oneservice.core.vo.*; +import com.webank.wedatasphere.dss.apiservice.core.vo.ApiServiceVo; +import com.webank.wedatasphere.linkis.bml.client.BmlClient; +import com.webank.wedatasphere.linkis.bml.client.BmlClientFactory; +import com.webank.wedatasphere.linkis.bml.protocol.BmlDownloadResponse; +import com.webank.wedatasphere.linkis.common.io.FsPath; +import com.webank.wedatasphere.linkis.storage.source.FileSource; +import com.webank.wedatasphere.linkis.storage.source.FileSource$; +import com.webank.wedatasphere.linkis.ujes.client.UJESClient; +import com.webank.wedatasphere.linkis.ujes.client.response.JobExecuteResult; +import org.apache.commons.collections4.MapUtils; +import org.apache.commons.lang.StringUtils; +import org.apache.commons.math3.util.Pair; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.jdbc.core.RowMapper; +import org.springframework.jdbc.support.JdbcUtils; +import org.springframework.stereotype.Service; +import org.springframework.util.LinkedCaseInsensitiveMap; +import scala.Tuple3; + +import javax.annotation.PostConstruct; +import java.io.IOException; +import java.io.InputStream; +import java.sql.ResultSet; +import java.sql.ResultSetMetaData; +import java.sql.SQLException; +import java.util.*; +import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; + +import static java.util.stream.Collectors.toMap; + +/** + * 接口调用service + * + * @author lidongzhang + */ +@Service +public class ApiServiceQueryServiceImpl implements ApiServiceQueryService { + private static final Logger LOG = LoggerFactory.getLogger(ApiServiceQueryServiceImpl.class); + + + Map runJobs = new HashMap<>(); + + /** + * key:resourceId+version + * value:bml + */ + private static Cache>> bmlCache = CacheBuilder.newBuilder() + .expireAfterWrite(6, TimeUnit.HOURS) + .maximumSize(10000) + .removalListener((notification) -> { + if (notification.getCause() == RemovalCause.SIZE) { + LOG.warn("bml缓存容量不足，移除key:" + notification.getKey()); + } + }) + .build(); + + /** + * key:resourceId+version + * value:configParam + */ + private static Cache> configParamCache = CacheBuilder.newBuilder() + .expireAfterWrite(6, TimeUnit.HOURS) + .maximumSize(10000) + .removalListener((notification) -> { + if (notification.getCause() == RemovalCause.SIZE) { + LOG.warn("configParamCache缓存容量不足，移除key:" + notification.getKey()); + } + }) + .build(); + + /** + * key:datasourceMap + * value:jdbc连接信息 + */ + private static Cache, Tuple3> datasourceCache = CacheBuilder.newBuilder() + .expireAfterWrite(1, TimeUnit.MINUTES) + .maximumSize(2000) + .removalListener((notification) -> { + if (notification.getCause() == RemovalCause.SIZE) { + LOG.warn("datasource缓存容量不足，移除key:" + notification.getKey()); + } + }) + .build(); + + @Autowired + private ApiServiceDao apiServiceDao; + + @Autowired + private ApiServiceParamDao apiServiceParamDao; + + @Autowired + private DatasourceService datasourceService; + + @Autowired + private ApiServiceVersionDao apiServiceVersionDao; + + @Autowired + private ApiServiceApprovalDao apiServiceApprovalDao; + + + @Autowired + private ApiServiceTokenManagerDao apiServiceTokenManagerDao; + + + @Autowired + private ApiService apiService; + + + @Autowired + private ApiServiceAccessDao apiServiceAccessDao; + + /** + * Bml client + */ + private BmlClient client; + + @PostConstruct + public void init() { + LOG.info("build client start ======"); + client = BmlClientFactory.createBmlClient(); + LOG.info("build client end ======="); + } + + @Override + public LinkisExecuteResult query(String path, + Map reqParams, + String moduleName, + String httpMethod, + ApiServiceToken tokenDetail, + String loginUser) { + // 根据path查询resourceId和version + + // 得到metadata + + // 执行查询 + //path 必须唯一 + ApiServiceVo apiServiceVo = apiServiceDao.queryByPath(path); + if(null == apiServiceVo){ + throw new ApiServiceRuntimeException("根据脚本路径未匹配到数据服务！"); + } + if(!apiService.checkUserWorkspace(loginUser,apiServiceVo.getWorkspaceId().intValue())){ + throw new ApiServiceRuntimeException("用户工作空间检查不通过！"); + } + if(!apiServiceVo.getId().equals(tokenDetail.getApiServiceId())){ + throw new ApiServiceRuntimeException("用户token中服务ID不匹配！"); + } + + ApiVersionVo maxApiVersionVo =apiService.getMaxVersion(apiServiceVo.getId()); + + AssertUtil.notNull(apiServiceVo, "接口不存在，path=" + path); + AssertUtil.isTrue(StringUtils.equals(httpMethod, apiServiceVo.getMethod().toUpperCase()), + "该接口不支持" + httpMethod + "请求，请用" + apiServiceVo.getMethod() + "请求"); + AssertUtil.isTrue(1 == apiServiceVo.getStatus(), "接口已禁用"); + AssertUtil.notNull(maxApiVersionVo, "未找到最新的版本，path=" + path); + + try { + Pair> collect = queryBml(apiServiceVo.getCreator(), maxApiVersionVo.getBmlResourceId(), + maxApiVersionVo.getBmlVersion(), apiServiceVo.getScriptPath()); + String executeCode = collect.getSecond().get(0)[0]; + + Map variable = (Map) ((Map) collect.getFirst()).get("variable"); + + //没有传入的参数，使用默认值 + Map paramTypes = queryConfigParam(apiServiceVo.getId(), maxApiVersionVo.getVersion()); + if (variable != null) { + variable.forEach((k, v) -> { + if (!reqParams.containsKey(k)) { + if (ParamType.number.equals(paramTypes.get(k))) { + reqParams.put(k, Integer.valueOf(v.toString())); + } else { + reqParams.put(k, v); + } + } + }); + } + + // 用户请求的参数值注入检查，排除token + for(String k: reqParams.keySet()){ + if(!k.equals(ApiServiceConfiguration.API_SERVICE_TOKEN_KEY.getValue()) + && SQLCheckUtil.doParamInjectionCheck(reqParams.get(k).toString())) { + // 如果注入直接返回null + LOG.warn("用户参数存在非法的关键字{}", reqParams.get(k).toString()); + return null; + } + } + + //数组类型，如果没有加单引号，自动添加 + reqParams.forEach((k,v) ->{ + if(ParamType.array.equals(paramTypes.get(k))){ + String sourceStr = v.toString(); + String targetStr =sourceStr; + sourceStr= sourceStr.replaceAll("(\n\r|\r\n|\r|\n)", ","); + sourceStr= sourceStr.replaceAll(",,", ","); + + if(!sourceStr.contains("\'")){ + targetStr= Arrays.stream(sourceStr.split(",")).map(s -> "\'" + s + "\'").collect(Collectors.joining(",")); + reqParams.put(k, targetStr); + }else { + reqParams.put(k, sourceStr); + } + } + }); + + +// AssertUtil.isTrue(MapUtils.isNotEmpty((Map) collect.getKey()), "数据源不能为空"); + + //获取代理执行用户 + ApprovalVo approvalVo = apiServiceApprovalDao.queryByVersionId(maxApiVersionVo.getId()); + ApiServiceExecuteJob job = new DefaultApiServiceJob(); + //sql代码封装成scala执行 + job.setCode(ExecuteCodeHelper.packageCodeToExecute(executeCode, maxApiVersionVo.getMetadataInfo())); + job.setEngineType(apiServiceVo.getType()); + job.setRunType("scala"); + //不允许创建用户自己随意代理执行，创建用户只能用自己用户执行 + job.setUser(loginUser); + if( !apiServiceVo.getCreator().equals(loginUser) && StringUtils.isNotEmpty(approvalVo.getExecuteUser())){ + if("hadoop".equals(approvalVo.getExecuteUser().toLowerCase())){ + throw new ApiExecuteException(80004, "非法使用Hadoop用户作为执行用户"); + } + job.setUser(approvalVo.getExecuteUser()); + } + job.setParams(null); + job.setRuntimeParams(reqParams); + job.setScriptePath(apiServiceVo.getScriptPath()); + UJESClient ujesClient = LinkisJobSubmit.getClient(paramTypes); + + //记录用户Api被执行信息 + ApiAccessVo apiAccessVo = new ApiAccessVo(); + apiAccessVo.setUser(loginUser); + apiAccessVo.setApiPublisher(apiServiceVo.getCreator()); + apiAccessVo.setApiServiceName(apiServiceVo.getName()); + apiAccessVo.setApiServiceId(apiServiceVo.getId()); + apiAccessVo.setApiServiceVersionId(maxApiVersionVo.getId()); + apiAccessVo.setProxyUser(job.getUser()); + apiAccessVo.setAccessTime(DateUtil.getNow()); + apiServiceAccessDao.addAccessRecord(apiAccessVo); + + + JobExecuteResult jobExecuteResult = LinkisJobSubmit.execute(job,ujesClient); + + //记录执行任务用户和代理用户关系，没有代理用户的统一设置为登录用户 + ApiServiceJob apiServiceJob = new ApiServiceJob(); + apiServiceJob.setSubmitUser(loginUser); + apiServiceJob.setProxyUser(job.getUser()); + apiServiceJob.setJobExecuteResult(jobExecuteResult); + runJobs.put(jobExecuteResult.getTaskID(),apiServiceJob); + + + LinkisExecuteResult linkisExecuteResult = new LinkisExecuteResult(jobExecuteResult.getTaskID(), jobExecuteResult.getExecID()); + return linkisExecuteResult; + } catch (IOException | ApiExecuteException e) { + throw new ApiServiceRuntimeException(e.getMessage(), e); + } + } + + + + + @Override + public ApiServiceVo queryByVersionId(String userName,Long versionId) throws ApiServiceQueryException { + ApiVersionVo apiVersionVo = apiServiceVersionDao.queryApiVersionByVersionId(versionId); + ApiServiceVo apiServiceVo = apiServiceDao.queryById(apiVersionVo.getApiId()); + //授权后才可以查看内容 + List userTokenManagerVos = apiServiceTokenManagerDao.queryByApplyUserAndVersionId(userName,versionId); + if(userTokenManagerVos.size()>0) { + try { + Pair> collect = queryBml(apiServiceVo.getCreator(), apiVersionVo.getBmlResourceId(), + apiVersionVo.getBmlVersion(), apiServiceVo.getScriptPath()); + String executeCode = collect.getSecond().get(0)[0]; + apiServiceVo.setContent(executeCode); + + } catch (IOException e) { + throw new ApiServiceQueryException(800002, "查询数据服务API内容异常"); + } + apiServiceVo.setScriptPath(apiVersionVo.getSource()); + return apiServiceVo; + }else { + + throw new ApiServiceQueryException(800003, "没有权限查看数据服务API内容，请先提单授权"); + } + } + + @Override + public List queryParamList(String scriptPath, Long versionId) { + ApiVersionVo targetApiVersionVo = apiServiceVersionDao.queryApiVersionByVersionId(versionId); + + ApiServiceVo apiServiceVo=apiServiceDao.queryById(targetApiVersionVo.getApiId()); + + AssertUtil.notNull(apiServiceVo, "接口不存在，path=" + scriptPath); + + AssertUtil.notNull(targetApiVersionVo, "目标参数版本不存在，path=" + scriptPath+",version:"+versionId); + + // todo~！ + List paramVoList = apiServiceParamDao.queryByVersionId(targetApiVersionVo.getId()); + + + List queryParamVoList = new ArrayList<>(); + + Map paramMap = paramVoList.stream() + .collect(Collectors.toMap(ParamVo::getName, k -> k, (k, v) -> k)); + Map variableMap = getVariable(apiServiceVo,versionId); + paramMap.keySet() + .forEach(keyItem -> { + ParamVo paramVo = paramMap.get(keyItem); + QueryParamVo queryParamVo = ModelMapperUtil.strictMap(paramVo, QueryParamVo.class); + queryParamVo.setTestValue(variableMap.containsKey(keyItem) ? variableMap.get(keyItem).toString() : ""); + queryParamVo.setRequireStr(RequireEnum.getEnum(paramVo.getRequired()).getName()); + queryParamVo.setType(paramVo.getType()); + + queryParamVoList.add(queryParamVo); + }); + + return queryParamVoList; + } + + @Override + public List queryApiVersionById(Long serviceId) { + List apiVersionVoList = apiServiceVersionDao.queryApiVersionByApiServiceId(serviceId); + return apiVersionVoList; + } + + private Map getVariable(ApiServiceVo apiServiceVo,Long versionId) { + Map variableMap = null; + ApiVersionVo apiVersionVo = apiServiceVersionDao.queryApiVersionByVersionId(versionId); + if(null != apiServiceVo) { + try { + Pair> collect = queryBml(apiServiceVo.getCreator(), apiVersionVo.getBmlResourceId(), + apiVersionVo.getBmlVersion(), apiServiceVo.getScriptPath()); + + variableMap = (Map) ((Map) collect.getFirst()).get("variable"); + } catch (IOException e) { + throw new ApiServiceRuntimeException(e.getMessage(), e); + } + } + return null == variableMap ? Collections.EMPTY_MAP : variableMap; + } + + private Pair> queryBml(String userName, String resourceId, String version, + String scriptPath) throws IOException { + String key = String.join("-", resourceId, version); + Pair> collect = bmlCache.getIfPresent(key); + + if (collect == null) { + synchronized (this) { + collect = bmlCache.getIfPresent(key); + if (collect == null) { + BmlDownloadResponse resource; + if (version == null) { + resource = client.downloadResource(userName, resourceId, null); + } else { + resource = client.downloadResource(userName, resourceId, version); + } + + AssertUtil.isTrue(resource.isSuccess(), "查询bml错误"); + + InputStream inputStream = resource.inputStream(); + + try (FileSource fileSource = FileSource$.MODULE$.create(new FsPath(scriptPath), inputStream)) { + //todo 数组取了第一个 + collect = fileSource.collect()[0]; + bmlCache.put(key, collect); + } + } + } + } + + + return collect; + } + + private Map queryConfigParam(long apiId, String version) { + String key = String.join("-", apiId + "", version); + Map collect = configParamCache.getIfPresent(key); + + if (collect == null) { + synchronized (this) { + collect = configParamCache.getIfPresent(key); + if (collect == null) { + List apiVersionVoList = apiServiceVersionDao.queryApiVersionByApiServiceId(apiId); + ApiVersionVo apiVersionVo = apiVersionVoList.stream().filter(apiVersionVoTmp -> apiVersionVoTmp.getVersion().equals(version)).findFirst().orElse(null); + + collect = apiServiceParamDao.queryByVersionId(apiVersionVo.getId()) + .stream() + .collect(toMap(ParamVo::getName, ParamVo::getType)); + configParamCache.put(key, collect); + } + } + } + + return collect; + } + + + + +// private Tuple3 getDatasourceInfo(final Map datasourceMap) { +// Tuple3 tuple3 = datasourceCache.getIfPresent(datasourceMap); +// +// if (tuple3 == null) { +// synchronized (this) { +// tuple3 = datasourceCache.getIfPresent(datasourceMap); +// if (tuple3 == null) { +// tuple3 = JdbcUtil.getDatasourceInfo(datasourceMap); +// datasourceCache.put(datasourceMap, tuple3); +// } +// } +// } +// +// return tuple3; +// } + +// private List> executeJob(String executeCode, +// Object datasourceMap, Map params) { +// +//// Tuple3 tuple3 = getDatasourceInfo((Map) datasourceMap); +//// final String jdbcUrl = tuple3._1().toString(); +//// final String username = tuple3._2().toString(); +//// final String password = tuple3._3().toString(); +// +//// NamedParameterJdbcTemplate namedParameterJdbcTemplate = datasourceService.getNamedParameterJdbcTemplate(jdbcUrl, username, password); +// +// String namedSql = genNamedSql(executeCode, params); +// +//// return namedParameterJdbcTemplate.query(namedSql, new MapSqlParameterSource(params), new ColumnAliasMapRowMapper()); +// +// } + + private static String genNamedSql(String executeCode, Map params) { + // 没有参数，无需生成namedSql + if (MapUtils.isEmpty(params)) { + return executeCode; + } + + for (String paramName : params.keySet()) { + for (String $name : new String[]{"'${" + paramName + "}'", "${" + paramName + "}", "\"${" + paramName + "}\""}) { + if (executeCode.contains($name)) { + executeCode = StringUtils.replace(executeCode, $name, ":" + paramName); + break; + } + } + } + + return executeCode; + } + + + public static class ColumnAliasMapRowMapper implements RowMapper> { + @Override + public Map mapRow(ResultSet rs, int rowNum) throws SQLException { + ResultSetMetaData rsmd = rs.getMetaData(); + int columnCount = rsmd.getColumnCount(); + Map mapOfColValues = createColumnMap(columnCount); + Map mapOfColSuffix = new LinkedCaseInsensitiveMap<>(columnCount); + for (int i = 1; i <= columnCount; i++) { + String key = getColumnKey(JdbcUtils.lookupColumnName(rsmd, i)); + if (mapOfColValues.containsKey(key)) { + if (!mapOfColSuffix.containsKey(key)) { + mapOfColSuffix.put(key, 1); + } else { + mapOfColSuffix.put(key, mapOfColSuffix.get(key) + 1); + } + + key = key + "_" + mapOfColSuffix.get(key); + } + + Object obj = getColumnValue(rs, i); + mapOfColValues.put(key, obj); + } + return mapOfColValues; + } + + protected Map createColumnMap(int columnCount) { + return new LinkedCaseInsensitiveMap<>(columnCount); + } + + protected String getColumnKey(String columnName) { + return columnName; + } + + protected Object getColumnValue(ResultSet rs, int index) throws SQLException { + return JdbcUtils.getResultSetValue(rs, index); + } + + } + + @Override + public ApiServiceJob getJobByTaskId(String taskId){ + ApiServiceJob apiServiceJob=runJobs.get(taskId); + return apiServiceJob; + } + + + private static String getRunTypeFromScriptsPath(String scriptsPath) { + + String res = "sql"; + String fileFlag = scriptsPath.substring(scriptsPath.lastIndexOf(".") + 1); + switch (fileFlag) { + case "sh": + res = "shell"; + break; + case "py": + res= "pyspark"; + break; + default: + res = fileFlag; + break; + } + return res; + + } +} diff --git a/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/service/impl/ApprovalServiceImpl.java b/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/service/impl/ApprovalServiceImpl.java new file mode 100644 index 0000000000..12b2213940 --- /dev/null +++ b/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/service/impl/ApprovalServiceImpl.java @@ -0,0 +1,75 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.apiservice.core.service.impl; + +import com.webank.wedatasphere.dss.apiservice.core.dao.ApiServiceApprovalDao; +import com.webank.wedatasphere.dss.apiservice.core.dao.ApiServiceVersionDao; +import com.webank.wedatasphere.dss.apiservice.core.datamap.DataMapStatus; +import com.webank.wedatasphere.dss.apiservice.core.service.ApprovalService; +import com.webank.wedatasphere.dss.apiservice.core.vo.ApprovalVo; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; + +/** + * @author: jinyangrao + */ +@Service +public class ApprovalServiceImpl implements ApprovalService { + + private static final Logger LOG = LoggerFactory.getLogger(ApprovalServiceImpl.class); + + @Autowired + private ApiServiceApprovalDao apiServiceApprovalDao; + + @Autowired + private ApiServiceVersionDao apiServiceVersionDao; + + @Override + public ApprovalVo refreshStatus(String approvalNo) throws Exception { + ApprovalVo approvalVo = apiServiceApprovalDao.queryByApprovalNo(approvalNo); + if (null != approvalVo) { + if (DataMapStatus.APPROVING.getIndex() == approvalVo.getStatus() || + DataMapStatus.INITED.getIndex() == approvalVo.getStatus() || + DataMapStatus.REJECT.getIndex() == approvalVo.getStatus()) { + + // use datamap interface + // 更新审批单的状态，驳回还会有后续状态，所以当为驳回状态时，还需要查询datamap +// DataMapStatus dataMapStatus = DataMapUtils.requestDataMapStatus(approvalNo, null); + DataMapStatus dataMapStatus= DataMapStatus.SUCCESS; + approvalVo.setStatus(dataMapStatus.getIndex()); + apiServiceApprovalDao.updateApprovalStatus(approvalNo, dataMapStatus.getIndex()); + LOG.info("更新审批单状态： {}" ,approvalNo, dataMapStatus.getValue()); + } + return approvalVo; + } else { + return null; + } + } + @Override + public ApprovalVo query(String approvalNo) { + ApprovalVo approvalVo = apiServiceApprovalDao.queryByApprovalNo(approvalNo); + + if(null != approvalVo) { + return approvalVo; + } else { + LOG.warn("无此审批单号！" + approvalNo); + return null; + } + } +} \ No newline at end of file diff --git a/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/service/impl/TokenQueryServiceImpl.java b/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/service/impl/TokenQueryServiceImpl.java new file mode 100644 index 0000000000..f5dc98e933 --- /dev/null +++ b/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/service/impl/TokenQueryServiceImpl.java @@ -0,0 +1,47 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.apiservice.core.service.impl; + +import com.github.pagehelper.PageHelper; +import com.github.pagehelper.PageInfo; +import com.webank.wedatasphere.dss.apiservice.core.dao.ApiServiceTokenManagerDao; +import com.webank.wedatasphere.dss.apiservice.core.service.TokenQueryService; +import com.webank.wedatasphere.dss.apiservice.core.vo.TokenManagerVo; +import com.webank.wedatasphere.dss.apiservice.core.bo.TokenQuery; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; + +import java.util.List; + +@Service +public class TokenQueryServiceImpl implements TokenQueryService { + private static final Logger LOG = LoggerFactory.getLogger(TokenQueryServiceImpl.class); + + @Autowired + ApiServiceTokenManagerDao apiServiceTokenManagerDao; + + @Override + public PageInfo query(TokenQuery tokenQuery) { + PageHelper.startPage(tokenQuery.getCurrentPage(), tokenQuery.getPageSize()); + List tokenList = apiServiceTokenManagerDao.query(tokenQuery); + LOG.info("token查询的结果列表大小为{}", tokenList.size()); + PageInfo pageInfo = new PageInfo<>(tokenList); + return pageInfo; + } +} diff --git a/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/token/DataMapTokenImpl.java b/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/token/DataMapTokenImpl.java new file mode 100644 index 0000000000..cdd7fdc778 --- /dev/null +++ b/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/token/DataMapTokenImpl.java @@ -0,0 +1,113 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.apiservice.core.token; + +import com.webank.wedatasphere.dss.apiservice.core.constant.ApiCommonConstant; +import com.webank.wedatasphere.dss.apiservice.core.constant.SaveTokenEnum; +import com.webank.wedatasphere.dss.apiservice.core.dao.ApiServiceTokenManagerDao; +import com.webank.wedatasphere.dss.apiservice.core.exception.ApiServiceTokenException; +import com.webank.wedatasphere.dss.apiservice.core.bo.ApiServiceToken; +import com.webank.wedatasphere.dss.apiservice.core.vo.ApprovalVo; +import com.webank.wedatasphere.dss.apiservice.core.vo.TokenManagerVo; +import org.apache.commons.lang.StringUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; + +import java.util.*; + +/** + * @author allenlliu + * @version 2.0.0 + * @date 2020/08/14 11:52 AM + */ + +@Component +public class DataMapTokenImpl implements TokenAuth { + + private static final Logger logger = LoggerFactory.getLogger(DataMapTokenImpl.class); + + + @Autowired + ApiServiceTokenManagerDao apiServiceTokenManagerDao; + + @Override + public SaveTokenEnum saveTokensToDb(List tokenManagerVos, String approvalNo) throws ApiServiceTokenException { + boolean isEmptyToken = tokenManagerVos.stream().filter(tokenManagerVo -> StringUtils.isEmpty(tokenManagerVo.getToken())).count()>0; + if(checkDuplicateAuth(approvalNo) && approvalNo != ApiCommonConstant.DEFAULT_APPROVAL_NO){ +// throw new ApiServiceTokenException(800001,"ApprovalNo has been repeatedly authorized "); + return SaveTokenEnum.DUPLICATE; + }else if(isEmptyToken){ + throw new ApiServiceTokenException(800001,"Failed to save to db for Some token is empty"); + } + else { + + try { + apiServiceTokenManagerDao.insertList(tokenManagerVos); + } catch (Exception e) { + logger.error("Batch save token to db failed", e); + return SaveTokenEnum.FAILED; + } + } + + return SaveTokenEnum.SUCCESS; + } + + private boolean checkDuplicateAuth(String approvalNo){ + if(apiServiceTokenManagerDao.queryApprovalNo(approvalNo) > 0){ + return true; + }else { + return false; + } + } + + @Override + public List genTokenRecord(ApprovalVo approvalVo){ + + List tokenManagerVoList = new ArrayList<>(); + Arrays.stream(approvalVo.getApplyUser().split(",")).forEach(tempUser ->{ + + TokenManagerVo tmpToken = new TokenManagerVo(); + tmpToken.setApiId(approvalVo.getApiId()); + tmpToken.setApplyTime(new Date()); + tmpToken.setDuration(365L); + tmpToken.setReason("approval token auth"); + tmpToken.setStatus(1); + tmpToken.setIpWhitelist(""); + tmpToken.setCaller("scripts"); + tmpToken.setUser(tempUser); + tmpToken.setPublisher(approvalVo.getCreator()); + tmpToken.setApiVersionId(approvalVo.getApiVersionId()); + + ApiServiceToken apiServiceToken = new ApiServiceToken(); + apiServiceToken.setApplyUser(tempUser); + apiServiceToken.setPublisher(approvalVo.getCreator()); //todo creator + apiServiceToken.setApplyTime(tmpToken.getApplyTime()); + apiServiceToken.setApiServiceId(approvalVo.getApiId()); + + tmpToken.setToken(JwtManager.createToken(tempUser,apiServiceToken,tmpToken.getDuration())); + //审批单号 + tmpToken.setApplySource(approvalVo.getApprovalNo()); + tokenManagerVoList.add(tmpToken); + + + }); + + return tokenManagerVoList; + } +} diff --git a/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/token/JwtManager.java b/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/token/JwtManager.java new file mode 100644 index 0000000000..4bda7665d8 --- /dev/null +++ b/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/token/JwtManager.java @@ -0,0 +1,121 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.apiservice.core.token; + +import com.sun.jersey.core.util.Base64; +import com.webank.wedatasphere.dss.apiservice.core.config.ApiServiceConfiguration; +import com.webank.wedatasphere.dss.apiservice.core.bo.ApiServiceToken; +import com.webank.wedatasphere.linkis.common.conf.CommonVars$; +import io.jsonwebtoken.Claims; +import io.jsonwebtoken.ExpiredJwtException; +import io.jsonwebtoken.Jws; +import io.jsonwebtoken.Jwts; +import io.jsonwebtoken.MalformedJwtException; +import io.jsonwebtoken.SignatureAlgorithm; +import io.jsonwebtoken.SignatureException; +import io.jsonwebtoken.UnsupportedJwtException; +import io.jsonwebtoken.impl.crypto.MacProvider; +import org.mortbay.log.Log; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import javax.crypto.SecretKey; +import javax.crypto.spec.SecretKeySpec; +import java.time.Duration; +import java.time.Instant; +import java.time.temporal.TemporalAmount; +import java.util.Date; +import java.util.HashMap; +import java.util.Map; +import java.util.Properties; + + +public class JwtManager { + private static final Logger LOG = LoggerFactory.getLogger(JwtManager.class); + + private static final String CLAIM_ROLE = "role"; + + private static final SignatureAlgorithm SIGNATURE_ALGORITHM = SignatureAlgorithm.HS256; + private static final SecretKey SECRET_KEY = MacProvider.generateKey(SIGNATURE_ALGORITHM); + private static final TemporalAmount TOKEN_VALIDITY = getTokenHour(); + private static final String JWT_SECERT = ApiServiceConfiguration.DSS_API_TOKEN_SECRET_ID.getValue(); + + public final static TemporalAmount getTokenHour() { + Properties conf = CommonVars$.MODULE$.properties(); + if (null != conf) { + Object hour = conf.get("ujes.token.valid.hour"); + if (null == hour) { + return Duration.ofHours(8L); + } else { + return Duration.ofHours(Long.parseLong(hour.toString())); + } + } else { + return Duration.ofDays(365); + } + + } + + public static SecretKey generalKey() { + byte[] encodedKey = Base64.decode(JWT_SECERT); + SecretKey key = new SecretKeySpec(encodedKey, 0, encodedKey.length, "AES"); + Log.info("token key str:"+key.toString()); + return key; + } + + /** + * Builds a JWT with the given subject and role and returns it as a JWS signed compact String. + */ + public static String createToken(final String applyUser, ApiServiceToken apiServiceToken, Long duration) { + final String role = "developer"; + final Instant now = Instant.now(); + final Date expiryDate = Date.from(now.plus(Duration.ofDays(duration))); + Map tokenDetailMap = new HashMap(); + tokenDetailMap.put("tokenDetail", apiServiceToken); + return Jwts.builder() + .setSubject(applyUser) + .claim(CLAIM_ROLE, role) + .setClaims(tokenDetailMap) + .setExpiration(expiryDate) + .setIssuedAt(Date.from(now)) + .signWith(SIGNATURE_ALGORITHM, generalKey()) + .compact(); + } + + /** + * Parses the given JWS signed compact JWT, returning the claims. + * If this method returns without throwing an exception, the token can be trusted. + */ + public static ApiServiceToken parseToken(final String compactToken) + throws ExpiredJwtException, + UnsupportedJwtException, + MalformedJwtException, + SignatureException, + IllegalArgumentException { + + Jws jws = Jwts.parser().setSigningKey(generalKey()).parseClaimsJws(compactToken); + + Map tokenDetail = (Map) jws.getBody().get("tokenDetail"); + ApiServiceToken apiServiceToken = new ApiServiceToken(); + apiServiceToken.setApiServiceId(Long.valueOf(tokenDetail.get("apiServiceId").toString())); + apiServiceToken.setApplyTime(null); + apiServiceToken.setPublisher(tokenDetail.get("publisher").toString()); + apiServiceToken.setApplyUser(tokenDetail.get("applyUser").toString()); + + return apiServiceToken; + } + + +} diff --git a/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/token/JwtModuleTest.java b/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/token/JwtModuleTest.java new file mode 100644 index 0000000000..8b82e6a49a --- /dev/null +++ b/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/token/JwtModuleTest.java @@ -0,0 +1,109 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.apiservice.core.token; + +import com.webank.wedatasphere.dss.apiservice.core.bo.ApiServiceToken; +import io.jsonwebtoken.*; + +import javax.security.auth.login.CredentialExpiredException; +import javax.security.auth.login.FailedLoginException; +import javax.security.auth.login.LoginException; +import java.util.logging.Logger; + + +public class JwtModuleTest { + + private static final Logger LOG = Logger.getLogger( JwtModuleTest.class.getName() ); + + public static void main(String args[]){ + + ApiServiceToken apiServiceToken = new ApiServiceToken(); + apiServiceToken.setApplyUser("allenlliu"); + apiServiceToken.setApiServiceId(110L); + + String token = JwtManager.createToken("allenlliu",apiServiceToken,30L); + try { + parse(token); + } catch (LoginException e) { + e.printStackTrace(); + } + } + + public static void parse(String jwtToken) throws LoginException { + // String jwt = getJwt(); + + if ( jwtToken != null ) { + try { + LOG.info( "JWT provided" ); + // Date i = CommonVars$.MODULE$.apply("", new TimeType("3h")).getValue().toDate(); + // JwtManager jwtManager = lookupJwtManager(); + + // verify the received token + ApiServiceToken tokenDetail = JwtManager.parseToken( jwtToken ); + + // now we can trust its information... +// String user = jws.getBody().getSubject(); + //identity = new SimplePrincipal( user ); + +// String role = (String) jws.getBody().get( "role" ); + +// Map tokenDetail = (Map)jws.getBody().get("tokenDetail"); + //group = new SimplePrincipal( role ); + +// Map details = jws.getBody().get("tokenDetail"); + + LOG.info( "JWT is valid, logging in user " + tokenDetail.getApplyUser() ); + + + } catch ( SignatureException | MalformedJwtException + | UnsupportedJwtException | IllegalArgumentException e ) { + throw new FailedLoginException( "Invalid security token provided" ); + } catch ( ExpiredJwtException e ) { + throw new CredentialExpiredException( "The security token is expired" ); + } + } + + } + +// @Override +// public boolean commit() throws LoginException { +// Set principals = subject.getPrincipals(); +// principals.add( identity ); +// +// SimpleGroup roles = new SimpleGroup( "Roles" ); +// roles.addMember( group ); +// principals.add( roles ); +// +// return true; +// } + +// public JwtManager lookupJwtManager() { +// try { +// BeanManager beanManager = InitialContext.doLookup( "java:comp/BeanManager" ); +// Set> beans = beanManager.getBeans( JwtManager.class ); +// if ( beans.isEmpty() ) { +// throw new RuntimeException( "Failed looking up CDI Bean " + JwtManager.class.getName() +// + ": Found " + beans.size() + " " ); +// } +// Bean bean = beans.iterator().next(); +// CreationalContext context = beanManager.createCreationalContext( bean ); +// return (JwtManager) beanManager.getReference( bean, JwtManager.class, context ); +// } catch ( NamingException e ) { +// throw new RuntimeException( e ); +// } +// } +} diff --git a/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/token/TokenAuth.java b/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/token/TokenAuth.java new file mode 100644 index 0000000000..5976cfec00 --- /dev/null +++ b/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/token/TokenAuth.java @@ -0,0 +1,42 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.apiservice.core.token; + +import com.webank.wedatasphere.dss.apiservice.core.constant.SaveTokenEnum; +import com.webank.wedatasphere.dss.apiservice.core.exception.ApiServiceTokenException; +import com.webank.wedatasphere.dss.apiservice.core.vo.ApprovalVo; +import com.webank.wedatasphere.dss.apiservice.core.vo.TokenManagerVo; + +import java.util.List; + +/** + * @author allenlliu + * @version 2.0.0 + * @date 2020/08/14 11:41 AM + */ +public interface TokenAuth { + + + /** + * batch save token to db + * @param tokenManagerVos + * @return + */ + SaveTokenEnum saveTokensToDb(List tokenManagerVos, String approvalNo) throws ApiServiceTokenException; + + List genTokenRecord(ApprovalVo approvalVo); +} diff --git a/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/token/TokenStatusCheckerTask.java b/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/token/TokenStatusCheckerTask.java new file mode 100644 index 0000000000..95203fc078 --- /dev/null +++ b/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/token/TokenStatusCheckerTask.java @@ -0,0 +1,61 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.apiservice.core.token; + +import com.webank.wedatasphere.dss.apiservice.core.dao.ApiServiceTokenManagerDao; +import com.webank.wedatasphere.dss.apiservice.core.util.DateUtil; +import com.webank.wedatasphere.dss.apiservice.core.vo.TokenManagerVo; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.scheduling.annotation.EnableScheduling; +import org.springframework.scheduling.annotation.Scheduled; +import org.springframework.stereotype.Component; + +import java.util.Calendar; +import java.util.Date; +import java.util.List; + +@Component +@EnableScheduling +public class TokenStatusCheckerTask { + private static final Logger LOG = LoggerFactory.getLogger(TokenStatusCheckerTask.class); + + @Autowired + ApiServiceTokenManagerDao atmd; + + @Scheduled(cron = "0 0/5 * * * ?") + public void doTokenStatusCheckTask() { + // 查询启用状态的token + List tokenManagerVos = atmd.queryTokenByStatus(1); + if (null != tokenManagerVos) { + for (TokenManagerVo tmv : tokenManagerVos) { + Date applyTime = tmv.getApplyTime(); + Date nowTime = Calendar.getInstance().getTime(); + Calendar cal = Calendar.getInstance(); + cal.setTime(applyTime); + cal.add(Calendar.DATE, tmv.getDuration().intValue()); + Date endTime = cal.getTime(); + if (endTime.compareTo(nowTime) < 0) { + LOG.warn("token id:" + tmv.getId() + " 已经过期！"); + atmd.disableTokenStatus(tmv.getId()); + } + } + } + } + +} diff --git a/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/util/ApiUtils.java b/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/util/ApiUtils.java new file mode 100644 index 0000000000..134e7e77a2 --- /dev/null +++ b/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/util/ApiUtils.java @@ -0,0 +1,124 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.apiservice.core.util; + +import com.webank.wedatasphere.dss.apiservice.core.exception.*; +import com.webank.wedatasphere.dss.apiservice.core.vo.MessageVo; +import com.webank.wedatasphere.linkis.common.exception.WarnException; +import com.webank.wedatasphere.linkis.server.Message; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import javax.validation.ConstraintViolationException; +import javax.ws.rs.core.Response; + +/** + * + */ +public class ApiUtils { + + private static final Logger LOG = LoggerFactory.getLogger(ApiUtils.class); + + /** + * @param tryOperation operate function + * @param failMessage message + */ + public static Response doAndResponse(TryOperation tryOperation, String method, String failMessage) { + try { + Message message = tryOperation.operateAndGetMessage(); + return Message.messageToResponse(setMethod(message, method)); + } catch (ConstraintViolationException e) { + LOG.error("api error, method: " + method, e); + return new BeanValidationExceptionMapper().toResponse(e); + } catch (WarnException e) { + LOG.error("api error, method: " + method, e); + return Message.messageToResponse(setMethod(Message.warn(e.getMessage()), method)); + } catch (AssertException e) { + LOG.error("api error, method: " + method, e); + return Message.messageToResponse(setMethod(Message.error(e.getMessage()), method)); + + } catch (ApiExecuteException e) { + LOG.error("api error, method: " + method, e); + return Message.messageToResponse(setMethod(Message.error(e.getMessage()), method)); + } + catch (ApiServiceQueryException e) { + LOG.error("api error, method: " + method, e); + return Message.messageToResponse(setMethod(Message.error(e.getMessage()), method)); + } + catch (Exception e) { + LOG.error("api error, method: " + method, e); + return Message.messageToResponse(setMethod(Message.error(failMessage, e), method)); + } + } + + /** + * @param tryOperation operate function + */ + public static Response doAndResponse(Operation tryOperation) { + Object msg = null; + try { + msg = tryOperation.operateAndGetMessage(); + return Response.ok(msg).build(); + } catch (ConstraintViolationException e) { + LOG.error("api error ", e); + return new BeanValidationExceptionMapper().toResponse(e); + } catch (WarnException e) { + LOG.error("api error ", e); + return Response.ok(setMsg("系统异常")).build(); + } catch (AssertException e) { + LOG.error("api error ", e); + return Response.ok(setMsg(e.getMessage())).build(); + }catch (ApiServiceRuntimeException e){ + LOG.error("api error ", e); + return Response.ok(setMsg(e.getMessage())).build(); + } + catch (Exception e) { + LOG.error("api error ", e); + return Response.ok(setMsg(String.valueOf(e.getCause()))).build(); + } + } + + public static Object setMsg(String message) { + MessageVo messageVo = new MessageVo(); + messageVo.setMessage(message); + + return messageVo; + } + + private static Message setMethod(Message message, String method) { + message.setMethod(method); + return message; + } + + @FunctionalInterface + public interface TryOperation { + + /** + * Operate method + */ + Message operateAndGetMessage() throws Exception; + } + + @FunctionalInterface + public interface Operation { + + /** + * Operate method + */ + Object operateAndGetMessage() throws Exception; + } +} diff --git a/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/util/AssertUtil.java b/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/util/AssertUtil.java new file mode 100644 index 0000000000..bb618124c2 --- /dev/null +++ b/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/util/AssertUtil.java @@ -0,0 +1,49 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package com.webank.wedatasphere.dss.apiservice.core.util; + + +import com.webank.wedatasphere.dss.apiservice.core.exception.AssertException; + +/** + * @author lidongzhang + */ +public class AssertUtil { + + public static void isTrue(boolean b, String message) { + if (!b) { + throw new AssertException(message); + } + } + + public static void isFalse(boolean b, String message) { + if (b) { + throw new AssertException(message); + } + } + + public static void notEmpty(String str, String message) { + if (str == null || str.isEmpty()) { + throw new AssertException(message); + } + } + + public static void notNull(Object str, String message) { + if (str == null) { + throw new AssertException(message); + } + } +} diff --git a/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/util/DateUtil.java b/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/util/DateUtil.java new file mode 100644 index 0000000000..b356fe1217 --- /dev/null +++ b/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/util/DateUtil.java @@ -0,0 +1,357 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package com.webank.wedatasphere.dss.apiservice.core.util; + +import java.sql.Timestamp; +import java.text.ParseException; +import java.text.SimpleDateFormat; +import java.util.Calendar; +import java.util.Date; + +/** + * 时间的工具类 + * + * @author chongchuanbing + */ +public class DateUtil { + + /** + * 英文简写（默认）如：2010-12-01 + */ + public static String FORMAT_MONTH = "yyyy-MM"; + /** + * 英文简写（默认）如：2010-12-01 + */ + public static String FORMAT_SHORT = "yyyy-MM-dd"; + + public static String FORMAT_SHORT_REPORT = "yyyy/MM/dd"; + /** + * 英文全称 如：2010-12-01 23:15:06 + */ + public static String FORMAT_LONG = "yyyy-MM-dd HH:mm:ss"; + public static String FORMAT_WITHOUT_SECOND = "yyyy-MM-dd HH:mm"; + + public static String FORMAT_MINUTE_REPORT = "yyyy/MM/dd HH:mm"; + public static String FORMAT_MINUTE = "yyyy-MM-dd HH:mm"; + /** + * 精确到毫秒的完整时间 如：yyyy-MM-dd HH:mm:ss.S + */ + public static String FORMAT_FULL = "yyyy-MM-dd HH:mm:ss.S"; + /** + * 中文简写 如：2010年12月01日 + */ + public static String FORMAT_SHORT_CN = "yyyy年MM月dd日"; + /** + * 中文全称 如：2010年12月01日 23时15分06秒 + */ + public static String FORMAT_LONG_CN = "yyyy年MM月dd日 HH时mm分ss秒"; + /** + * 精确到毫秒的完整中文时间 + */ + public static String FORMAT_FULL_CN = "yyyy年MM月dd日 HH时mm分ss秒SSS毫秒"; + + /** + * 获得默认的 date pattern + */ + public static String getDatePattern() { + return FORMAT_LONG; + } + + /** + * 根据预设格式返回当前日期 + * + * @return + */ + public static String getNow() { + return format(new Date()); + } + + /** + * 根据用户格式返回当前日期 + * + * @param format + * @return + */ + public static String getNow(String format) { + return format(new Date(), format); + } + + /** + * 使用预设格式格式化日期 + * + * @param date + * @return + */ + public static String format(Date date) { + return format(date, getDatePattern()); + } + + /** + * 使用用户格式格式化日期 + * + * @param date 日期 + * @param pattern 日期格式 + * @return + */ + public static String format(Date date, String pattern) { + String returnValue = ""; + if (date != null) { + SimpleDateFormat df = new SimpleDateFormat(pattern); + returnValue = df.format(date); + } + return (returnValue); + } + + /** + * 使用预设格式提取字符串日期 + * + * @param strDate 日期字符串 + * @return + */ + public static Date parse(String strDate) { + return parse(strDate, getDatePattern()); + } + + /** + * 使用用户格式提取字符串日期 + * + * @param strDate 日期字符串 + * @param pattern 日期格式 + * @return + */ + public static Date parse(String strDate, String pattern) { + SimpleDateFormat df = new SimpleDateFormat(pattern); + try { + return df.parse(strDate); + } catch (ParseException e) { + e.printStackTrace(); + return null; + } + } + + /** + * 在日期上增加数个整月 + * + * @param date 日期 + * @param n 要增加的月数 + * @return + */ + public static Date addMonth(Date date, int n) { + Calendar cal = Calendar.getInstance(); + cal.setTime(date); + cal.add(Calendar.MONTH, n); + return cal.getTime(); + } + + /** + * 在日期上增加天数 + * + * @param date 日期 + * @param n 要增加的天数 + * @return + */ + public static Date addDay(Date date, int n) { + Calendar cal = Calendar.getInstance(); + cal.setTime(date); + cal.add(Calendar.DATE, n); + return cal.getTime(); + } + + /** + * 获取时间戳 + */ + public static String getTimeString() { + SimpleDateFormat df = new SimpleDateFormat(FORMAT_FULL); + Calendar calendar = Calendar.getInstance(); + return df.format(calendar.getTime()); + } + + /** + * 获取日期年份 + * + * @param date 日期 + * @return + */ + public static String getYear(Date date) { + return format(date).substring(0, 4); + } + + /** + * 按默认格式的字符串距离今天的天数 + * + * @param date 日期字符串 + * @return + */ + public static int countDays(String date) { + long t = Calendar.getInstance().getTime().getTime(); + Calendar c = Calendar.getInstance(); + c.setTime(parse(date)); + long t1 = c.getTime().getTime(); + return (int) (t / 1000 - t1 / 1000) / 3600 / 24; + } + + /** + * 按用户格式字符串距离今天的天数 + * + * @param date 日期字符串 + * @param format 日期格式 + * @return + */ + public static int countDaysAbs(String date, String format) { + long t = Calendar.getInstance().getTime().getTime(); + Calendar c = Calendar.getInstance(); + c.setTime(parse(date, format)); + long t1 = c.getTime().getTime(); + return (int) Math.abs(t / 1000 - t1 / 1000) / 3600 / 24; + } + + /** + * 按用户格式字符串距离今天的天数 + * + * @param date 日期字符串 + * @param format 日期格式 + * @return + */ + public static int countDays(String date, String format) { + long t = Calendar.getInstance().getTime().getTime(); + Calendar c = Calendar.getInstance(); + c.setTime(parse(date, format)); + long t1 = c.getTime().getTime(); + return (int) (t / 1000 - t1 / 1000) / 3600 / 24; + } + + public static String timeStrComplete(String timeStr) { + if (null == timeStr || "".equals(timeStr)) { + return null; + } + String str = timeStr; + str = str.replaceAll("[0-9]", "0"); + + String tempStr = "0000-00-00 00:00:00"; + String tempSplitStr = "1970-01-01 00:00:00"; + int index = tempStr.indexOf(str); + if (0 == index) { + /** + * 说明符合日期格式 截取tempStr后面的部分与timeStr拼接构成时间 + */ + String lastStr = tempSplitStr.substring(timeStr.length()); + return timeStr + lastStr; + } + return null; + } + + public static Timestamp str2Time(String timeStr) { + if (null == timeStr || "".equals(timeStr)) { + return null; + } + String str = timeStr; + str = str.replaceAll("[0-9]", "0"); + + String tempStr = "0000-00-00 00:00:00"; + String tempSplitStr = "1970-01-01 00:00:00"; + int index = tempStr.indexOf(str); + if (0 == index) { + /** + * 说明符合日期格式 截取tempStr后面的部分与timeStr拼接构成时间 + */ + String lastStr = tempSplitStr.substring(timeStr.length()); + return Timestamp.valueOf(timeStr + lastStr); + } + + return null; + } + + public static Timestamp str2TimeMax(String timeStr) { + if (null == timeStr || "".equals(timeStr)) { + return null; + } + String str = timeStr; + str = str.replaceAll("[0-9]", "0"); + + String tempStr = "0000-00-00 00:00:00"; + String tempSplitStr = "1970-01-01 23:59:59"; + int index = tempStr.indexOf(str); + if (0 == index) { + /** + * 说明符合日期格式 截取tempStr后面的部分与timeStr拼接构成时间 + */ + String lastStr = tempSplitStr.substring(timeStr.length()); + return Timestamp.valueOf(timeStr + lastStr); + } + + return null; + } + + public static Date str2DateTime(String timeStr) { + if (null == timeStr || "".equals(timeStr)) { + return null; + } + String str = timeStr; + str = str.replaceAll("[0-9]", "0"); + + String tempStr = "0000-00-00 00:00:00"; + String tempSplitStr = "1970-01-01 00:00:00"; + int index = tempStr.indexOf(str); + if (0 == index) { + /** + * 说明符合日期格式 截取tempStr后面的部分与timeStr拼接构成时间 + */ + String lastStr = tempSplitStr.substring(timeStr.length()); + return parse(timeStr + lastStr); + } + + return null; + } + + public static Date str2DateTimeMax(String timeStr) { + if (null == timeStr || "".equals(timeStr)) { + return null; + } + String str = timeStr; + str = str.replaceAll("[0-9]", "0"); + + String tempStr = "0000-00-00 00:00:00"; + String tempSplitStr = "1970-01-01 23:59:59"; + int index = tempStr.indexOf(str); + if (0 == index) { + /** + * 说明符合日期格式 截取tempStr后面的部分与timeStr拼接构成时间 + */ + String lastStr = tempSplitStr.substring(timeStr.length()); + return parse(timeStr + lastStr); + } + + return null; + } + + /** + * 字符串数字转化为double + * + * @param string + * @return + */ + public static double str2double(String string) { + if (null == string || "".equals(string)) { + return 0; + } + try { + return Double.valueOf(string); + } catch (NumberFormatException e) { + return 0; + } + } + +} diff --git a/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/util/HttpClientUtil.java b/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/util/HttpClientUtil.java new file mode 100644 index 0000000000..e01f5a7bcd --- /dev/null +++ b/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/util/HttpClientUtil.java @@ -0,0 +1,477 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.apiservice.core.util; + +import org.apache.http.Consts; +import org.apache.http.HttpEntity; +import org.apache.http.HttpResponse; +import org.apache.http.NameValuePair; +import org.apache.http.client.ClientProtocolException; +import org.apache.http.client.config.RequestConfig; +import org.apache.http.client.entity.UrlEncodedFormEntity; +import org.apache.http.client.methods.CloseableHttpResponse; +import org.apache.http.client.methods.HttpGet; +import org.apache.http.client.methods.HttpPost; +import org.apache.http.entity.ContentType; +import org.apache.http.entity.StringEntity; +import org.apache.http.impl.client.CloseableHttpClient; +import org.apache.http.impl.client.HttpClients; +import org.apache.http.impl.conn.PoolingHttpClientConnectionManager; +import org.apache.http.message.BasicNameValuePair; +import org.apache.http.util.EntityUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import javax.net.ssl.TrustManager; +import javax.net.ssl.X509TrustManager; +import java.io.IOException; +import java.io.UnsupportedEncodingException; +import java.net.SocketTimeoutException; +import java.net.URLEncoder; +import java.security.cert.CertificateException; +import java.text.SimpleDateFormat; +import java.util.*; +import java.util.Map.Entry; + +//import com.alibaba.fastjson.JSONArray; +//import com.alibaba.fastjson.JSONObject; +//import com.alibaba.fastjson.serializer.SerializerFeature; + +@SuppressWarnings("all") +public final class HttpClientUtil { + private final static Logger logger = LoggerFactory.getLogger(HttpClientUtil.class); + public final static int connectTimeout = 5000; + private static PoolingHttpClientConnectionManager connManager = null; + private static CloseableHttpClient httpclient = null; + + /** + * 重写验证方法，取消检测ssl + */ + private static TrustManager trustAllManager = new X509TrustManager() { + @Override + public void checkClientTrusted(java.security.cert.X509Certificate[] arg0, String arg1) + throws CertificateException { + } + @Override + public void checkServerTrusted(java.security.cert.X509Certificate[] arg0, String arg1) + throws CertificateException { + } + @Override + public java.security.cert.X509Certificate[] getAcceptedIssuers() { + return null; + } + + }; + + static { + httpclient = HttpClients.createDefault(); + } + + + /** + * + * @param url + * @param timeout + * @param headerMap + * @param paramsList + * @param encoding + * @return + */ + public static String postForm(String url, int timeout, Map headerMap, List paramsList, String encoding){ + HttpPost post = new HttpPost(url); + try { + if(headerMap != null){ + for(Entry entry : headerMap.entrySet()){ + post.setHeader(entry.getKey(), entry.getValue().toString()); + } + } + //post.setHeader("Content-type", "application/json"); + RequestConfig requestConfig = RequestConfig.custom() + .setSocketTimeout(timeout) + .setConnectTimeout(timeout) + .setConnectionRequestTimeout(timeout) + .setExpectContinueEnabled(false).build(); + post.setConfig(requestConfig); + + post.setEntity(new UrlEncodedFormEntity(paramsList, encoding)); + CloseableHttpResponse response = httpclient.execute(post); + try { + HttpEntity entity = response.getEntity(); + try { + if(entity != null){ + String str = EntityUtils.toString(entity, encoding); + return str; + } + } finally { + if(entity != null){ + entity.getContent().close(); + } + } + } finally { + if(response != null){ + response.close(); + } + } + } catch (Exception e) { + throw new RuntimeException("invoke http post error!",e); + } finally { + post.releaseConnection(); + } + return ""; + } + + /** + * 调用saltapi时 + * + * @author: XIEJIAN948@pingan.com.cn + */ + public static String postJsonBody(String url, int timeout, Map headerMap, + String paraData, String encoding) { + + logger.info("successfully start post Json Body url{} ", url); + HttpPost post = new HttpPost(url); + try { + if (headerMap != null) { + for (Entry entry : headerMap.entrySet()) { + post.setHeader(entry.getKey(), entry.getValue().toString()); + } + } + RequestConfig requestConfig = RequestConfig.custom().setSocketTimeout(timeout).setConnectTimeout(timeout) + .setConnectionRequestTimeout(timeout).setExpectContinueEnabled(false).build(); + StringEntity jsonEntity = new StringEntity(paraData, ContentType.APPLICATION_JSON); + post.setConfig(requestConfig); + post.setEntity(jsonEntity); + CloseableHttpResponse response = httpclient.execute(post); + try { + HttpEntity entity = response.getEntity(); + try { + if (entity != null) { + String str = EntityUtils.toString(entity, encoding); + return str; + } + } finally { + if (entity != null) { + entity.getContent().close(); + } + } + } finally { + if (response != null) { + response.close(); + } + } + } catch (UnsupportedEncodingException e) { + logger.error("UnsupportedEncodingException", e); + throw new RuntimeException("failed post json return blank!"); + } catch (Exception e) { + logger.error("Exception", e); + throw new RuntimeException("failed post json return blank!"); + } finally { + post.releaseConnection(); + } + logger.info("successfully end post Json Body url{} ", url); + return ""; + } + + @SuppressWarnings("deprecation") + public static String invokeGet(String url, Map params, String encode, int connectTimeout, + int soTimeout) { + String responseString = null; + RequestConfig requestConfig = RequestConfig.custom().setSocketTimeout(connectTimeout) + .setConnectTimeout(connectTimeout).setConnectionRequestTimeout(connectTimeout).build(); + + StringBuilder sb = new StringBuilder(); + sb.append(url); + int i = 0; + if (params != null) { + for (Entry entry : params.entrySet()) { + if (i == 0 && !url.contains("?")) { + sb.append("?"); + } else { + sb.append("&"); + } + sb.append(entry.getKey()); + sb.append("="); + String value = entry.getValue(); + try { + sb.append(URLEncoder.encode(value, "UTF-8")); + } catch (UnsupportedEncodingException e) { + logger.warn("encode http get params error, value is " + value, e); + sb.append(URLEncoder.encode(value)); + } + i++; + } + } + HttpGet get = new HttpGet(sb.toString()); + get.setConfig(requestConfig); + try { + CloseableHttpResponse response = httpclient.execute(get); + try { + HttpEntity entity = response.getEntity(); + try { + if (entity != null) { + responseString = EntityUtils.toString(entity, encode); + } + } finally { + if (entity != null) { + entity.getContent().close(); + } + } + } catch (Exception e) { + logger.error(String.format("[HttpUtils Get]get response error, url:%s", sb.toString()), e); + return responseString; + } finally { + if (response != null) { + response.close(); + } + } + // System.out.println(String.format("[HttpUtils Get]Debug url:%s , + // response string %s:", sb.toString(), responseString)); + } catch (SocketTimeoutException e) { + logger.error(String.format("[HttpUtils Get]invoke get timout error, url:%s", sb.toString()), e); + return responseString; + } catch (Exception e) { + logger.error(String.format("[HttpUtils Get]invoke get error, url:%s", sb.toString()), e); + } finally { + get.releaseConnection(); + } + return responseString; + } + + /** + * HTTPS请求，默认超时为5S + * + * @param reqURL + * @param params + * @return + */ + public static String connectPostHttps(String reqURL, Map params) { + + String responseContent = null; + HttpPost httpPost = new HttpPost(reqURL); + try { + RequestConfig requestConfig = RequestConfig.custom().setSocketTimeout(connectTimeout) + .setConnectTimeout(connectTimeout).setConnectionRequestTimeout(connectTimeout).build(); + List formParams = new ArrayList(); + httpPost.setEntity(new UrlEncodedFormEntity(formParams, Consts.UTF_8)); + httpPost.setConfig(requestConfig); + // 绑定到请求 Entry + for (Entry entry : params.entrySet()) { + formParams.add(new BasicNameValuePair(entry.getKey(), entry.getValue())); + } + CloseableHttpResponse response = httpclient.execute(httpPost); + try { + // 执行POST请求 + HttpEntity entity = response.getEntity(); // 获取响应实体 + try { + if (null != entity) { + responseContent = EntityUtils.toString(entity, Consts.UTF_8); + } + } finally { + if (entity != null) { + entity.getContent().close(); + } + } + } finally { + if (response != null) { + response.close(); + } + } + logger.info("requestURI : " + httpPost.getURI() + ", responseContent: " + responseContent); + } catch (ClientProtocolException e) { + logger.error("ClientProtocolException", e); + } catch (IOException e) { + logger.error("IOException", e); + } finally { + httpPost.releaseConnection(); + } + return responseContent; + + } + + class Test { + String v; + String k; + + public String getV() { + return v; + } + + public void setV(String v) { + this.v = v; + } + + public String getK() { + return k; + } + + public void setK(String k) { + this.k = k; + } + + } + + // 随机4位数 + public static String getRandomValue() { + String str = "0123456789"; + StringBuilder sb = new StringBuilder(4); + for (int i = 0; i < 4; i++) { + char ch = str.charAt(new Random().nextInt(str.length())); + sb.append(ch); + } + return sb.toString(); + } + + // 当前时间到秒 + public static String getTimestamp() { + + Date date = new Date(); + String timestamp = String.valueOf(date.getTime() / 1000); + return timestamp; + } + + // 当前时间到秒 + public static String getNowDate() { + Date date = new Date(); + SimpleDateFormat sdf = new SimpleDateFormat("yyyyMMddHHmmss"); + return sdf.format(date); + } + + /** + * 调用saltapi时 + * + * @author: XIEJIAN948@pingan.com.cn + */ + public static String postJsonBody2(String url, int timeout, Map headerMap, + List paramsList, String encoding) { + logger.info("successfully start post Json Body url{} ", url); + HttpPost post = new HttpPost(url); + try { + if (headerMap != null) { + for (Entry entry : headerMap.entrySet()) { + post.setHeader(entry.getKey(), entry.getValue().toString()); + } + } + RequestConfig requestConfig = RequestConfig.custom().setSocketTimeout(timeout).setConnectTimeout(timeout) + .setConnectionRequestTimeout(timeout).setExpectContinueEnabled(false).build(); + post.setConfig(requestConfig); + if (paramsList.size() > 0) { + UrlEncodedFormEntity entity = new UrlEncodedFormEntity(paramsList, encoding); + post.setEntity(entity); + } + CloseableHttpResponse response = httpclient.execute(post); + try { + HttpEntity entity = response.getEntity(); + try { + if (entity != null) { + String str = EntityUtils.toString(entity, encoding); + return str; + } + } finally { + if (entity != null) { + entity.getContent().close(); + } + } + } finally { + if (response != null) { + response.close(); + } + } + } catch (UnsupportedEncodingException e) { + logger.error("UnsupportedEncodingException", e); + throw new RuntimeException("failed post json return blank!"); + } catch (Exception e) { + logger.error("Exception", e); + throw new RuntimeException("failed post json return blank!"); + } finally { + post.releaseConnection(); + } + logger.info("successfully end post Json Body url{} ", url); + return ""; + } + + /** + * 调用saltapi时 + * + * @author: XIEJIAN948@pingan.com.cn + */ + public static String postJsonBody3(String url, int timeout, Map headerMap, + Map paramsList, String encoding) { + HttpPost post = new HttpPost(url); + try { + if (headerMap != null) { + for (Entry entry : headerMap.entrySet()) { + post.setHeader(entry.getKey(), entry.getValue().toString()); + } + } + RequestConfig requestConfig = RequestConfig.custom().setSocketTimeout(timeout).setConnectTimeout(timeout) + .setConnectionRequestTimeout(timeout).setExpectContinueEnabled(false).build(); + post.setConfig(requestConfig); + if (paramsList.size() > 0) { + //JSONArray jsonArray = JSONArray.fromObject(paramsList); + //post.setEntity(new StringEntity(jsonArray.get(0).toString(), encoding)); + post.setEntity(new StringEntity(null, encoding)); + //logger.info("successfully start post Json Body url{},params ", url,jsonArray.get(0).toString()); + logger.info("successfully start post Json Body url{},params ", url,null); + } + CloseableHttpResponse response = httpclient.execute(post); + try { + HttpEntity entity = response.getEntity(); + try { + if (entity != null) { + String str = EntityUtils.toString(entity, encoding); + return str; + } + } finally { + if (entity != null) { + entity.getContent().close(); + } + } + } finally { + if (response != null) { + response.close(); + } + } + } catch (UnsupportedEncodingException e) { + logger.error("UnsupportedEncodingException", e); + throw new RuntimeException("failed post json return blank!"); + } catch (Exception e) { + logger.error("Exception", e); + throw new RuntimeException("failed post json return blank!"); + } finally { + post.releaseConnection(); + } + logger.info("successfully end post Json Body url{} ", url); + return ""; + } + + public static String executeGet(String url) + { + String rtnStr = ""; + HttpGet httpGet = new HttpGet(url); + try { + HttpResponse httpResponse = httpclient.execute(httpGet); + //获得返回的结果 + rtnStr = EntityUtils.toString(httpResponse.getEntity()); + } catch (IOException e) { + e.printStackTrace(); + } finally { + httpGet.releaseConnection(); + } + return rtnStr; + } + +} diff --git a/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/util/ModelMapperUtil.java b/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/util/ModelMapperUtil.java new file mode 100644 index 0000000000..fb06dd7934 --- /dev/null +++ b/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/util/ModelMapperUtil.java @@ -0,0 +1,54 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package com.webank.wedatasphere.dss.apiservice.core.util; + +import org.modelmapper.ModelMapper; +import org.modelmapper.convention.MatchingStrategies; + +import java.util.ArrayList; +import java.util.Iterator; +import java.util.List; + +/** + * @author chongchuanbing + */ +public class ModelMapperUtil { + private static ModelMapper modelMapper = new ModelMapper(); + + public ModelMapperUtil() { + } + + public static D strictMap(Object source, Class destinationType) { + return modelMapper.map(source, destinationType); + } + + public static List strictMapList(Object source, Class componentType) { + List list = new ArrayList(); + List objectList = (List)source; + Iterator var4 = objectList.iterator(); + + while(var4.hasNext()) { + Object obj = var4.next(); + list.add(modelMapper.map(obj, componentType)); + } + + return list; + } + + static { + modelMapper.getConfiguration().setMatchingStrategy(MatchingStrategies.STRICT); + } +} diff --git a/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/util/SQLCheckUtil.java b/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/util/SQLCheckUtil.java new file mode 100644 index 0000000000..aab3cf0d4d --- /dev/null +++ b/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/util/SQLCheckUtil.java @@ -0,0 +1,63 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.apiservice.core.util; + +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +/** + * @author: jinyangrao + */ +public class SQLCheckUtil { + + // sql 注入检查 + private static String injectionReg = "(?:--)|" + + "(\\b(select|update|union|and|or|delete|insert|trancate|char|substr|ascii|declare|exec|count|master|into|drop|execute)\\b)"; + private static Pattern paramInjectionPattern = Pattern.compile(injectionReg, Pattern.CASE_INSENSITIVE | Pattern.DOTALL); + + /** + * 1. -- 注释 + * 2. select * from table -- 注释 + * 但是不包含 -- '注释' + * */ + private static String sqlCommentReg = "\\-\\-([^\\'\\r\\n]{0,}(\\'[^\\'\\r\\n]{0,}\\'){0,1}[^\\'\\r\\n]{0,}){0,}"; + private static Pattern sqlCommentPattern = Pattern.compile(sqlCommentReg, Pattern.CASE_INSENSITIVE | Pattern.DOTALL); + + public static boolean doParamInjectionCheck(String str) { + Matcher matcher = paramInjectionPattern.matcher(str); + if(matcher.find()) { + return true; + } + return false; + } + + public static String sqlCommentReplace(String sql) { + String newSql = sql.replaceAll(sqlCommentReg, ""); + return newSql; + } + + public static void main(String[] args) { + String sql = "--注释\n" + + "-- 注释\n" + + "select * from tb --注释\n"; + String newSql = sqlCommentReplace(sql); + String[] selects = newSql.split("select"); + System.out.println(newSql); + System.out.println(selects.length); + } + +} \ No newline at end of file diff --git a/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/vo/ApiAccessVo.java b/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/vo/ApiAccessVo.java new file mode 100644 index 0000000000..700f74368e --- /dev/null +++ b/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/vo/ApiAccessVo.java @@ -0,0 +1,103 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.apiservice.core.vo; + +/** + * @author allenlliu + * @date 2020/11/11 10:27 + */ +public class ApiAccessVo { + Long id; + + String user; + + String apiServiceName; + + Long apiServiceId; + + Long apiServiceVersionId; + + String apiPublisher; + + String accessTime; + + String proxyUser; + + public Long getId() { + return id; + } + + public void setId(Long id) { + this.id = id; + } + + + public String getProxyUser() { + return proxyUser; + } + + public void setProxyUser(String proxyUser) { + this.proxyUser = proxyUser; + } + public String getUser() { + return user; + } + + public void setUser(String user) { + this.user = user; + } + + public String getApiServiceName() { + return apiServiceName; + } + + public void setApiServiceName(String apiServiceName) { + this.apiServiceName = apiServiceName; + } + + public Long getApiServiceId() { + return apiServiceId; + } + + public void setApiServiceId(Long apiServiceId) { + this.apiServiceId = apiServiceId; + } + + public Long getApiServiceVersionId() { + return apiServiceVersionId; + } + + public void setApiServiceVersionId(Long apiServiceVersionId) { + this.apiServiceVersionId = apiServiceVersionId; + } + + public String getApiPublisher() { + return apiPublisher; + } + + public void setApiPublisher(String apiPublisher) { + this.apiPublisher = apiPublisher; + } + + public String getAccessTime() { + return accessTime; + } + + public void setAccessTime(String accessTime) { + this.accessTime = accessTime; + } +} diff --git a/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/vo/ApiItsmVo.java b/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/vo/ApiItsmVo.java new file mode 100644 index 0000000000..0fd07d1416 --- /dev/null +++ b/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/vo/ApiItsmVo.java @@ -0,0 +1,138 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.apiservice.core.vo; + +public class ApiItsmVo { + + private String formId; + private String requestCreateDate; + private String status; + private String creator; + private String name; + private String description; + private String grantedUser; + private String grantedSystem; + private String grantedStartTime; + private Integer grantedDays; + private String reason; + private String ipAcceptList; + private String priority; + + public String getFormId() { + return formId; + } + + public void setFormId(String formId) { + this.formId = formId; + } + + public String getRequestCreateDate() { + return requestCreateDate; + } + + public void setRequestCreateDate(String requestCreateDate) { + this.requestCreateDate = requestCreateDate; + } + + public String getStatus() { + return status; + } + + public void setStatus(String status) { + this.status = status; + } + + public String getCreator() { + return creator; + } + + public void setCreator(String creator) { + this.creator = creator; + } + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public String getDescription() { + return description; + } + + public void setDescription(String description) { + this.description = description; + } + + public String getGrantedUser() { + return grantedUser; + } + + public void setGrantedUser(String grantedUser) { + this.grantedUser = grantedUser; + } + + public String getGrantedSystem() { + return grantedSystem; + } + + public void setGrantedSystem(String grantedSystem) { + this.grantedSystem = grantedSystem; + } + + public String getGrantedStartTime() { + return grantedStartTime; + } + + public void setGrantedStartTime(String grantedStartTime) { + this.grantedStartTime = grantedStartTime; + } + + public Integer getGrantedDays() { + return grantedDays; + } + + public void setGrantedDays(Integer grantedDays) { + this.grantedDays = grantedDays; + } + + public String getReason() { + return reason; + } + + public void setReason(String reason) { + this.reason = reason; + } + + public String getIpAcceptList() { + return ipAcceptList; + } + + public void setIpAcceptList(String ipAcceptList) { + this.ipAcceptList = ipAcceptList; + } + + public String getPriority() { + return priority; + } + + public void setPriority(String priority) { + this.priority = priority; + } +} diff --git a/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/vo/ApiServiceVo.java b/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/vo/ApiServiceVo.java new file mode 100644 index 0000000000..71d2735cf5 --- /dev/null +++ b/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/vo/ApiServiceVo.java @@ -0,0 +1,281 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package com.webank.wedatasphere.dss.apiservice.core.vo; + + +import java.util.Date; +import java.util.List; +import java.util.Map; + +/** + * @author jinyangrao + */ +public class ApiServiceVo { + private Long id; + private String name; + private String aliasName; + private String path; + private int protocol; + private String method; + private String tag; + private String scope; + private String description; + private Integer status = 1; + private String type; + private String runType; + private Date createTime; + private Date modifyTime; + private String creator; + private String modifier; + private String scriptPath; + private ApprovalVo approvalVo; + + private Long latestVersionId; + private String userToken; + + private List params; + + private Map metadata; + private String content; + + private List versionVos; + + private Long workspaceId = 1L; + private Long targetServiceId; + private String comment; + + public String getComment() { + return comment; + } + + public void setComment(String comment) { + this.comment = comment; + } + + + + public Long getWorkspaceId() { + return workspaceId; + } + + public void setWorkspaceId(Long workspaceId) { + this.workspaceId = workspaceId; + } + + + public Long getTargetServiceId() { + return targetServiceId; + } + + public void setTargetServiceId(Long targetServiceId) { + this.targetServiceId = targetServiceId; + } + + + + public ApprovalVo getApprovalVo() { + return approvalVo; + } + + public void setApprovalVo(ApprovalVo approvalVo) { + this.approvalVo = approvalVo; + } + + public Long getLatestVersionId() { + return latestVersionId; + } + + public void setLatestVersionId(Long latestVersionId) { + this.latestVersionId = latestVersionId; + } + + public Long getId() { + return id; + } + + public void setId(Long id) { + this.id = id; + } + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public String getPath() { + return path; + } + + public void setPath(String path) { + this.path = path; + } + + public int getProtocol() { + return protocol; + } + + public void setProtocol(int protocol) { + this.protocol = protocol; + } + + public String getMethod() { + return method; + } + + public void setMethod(String method) { + this.method = method; + } + + public String getTag() { + return tag; + } + + public void setTag(String tag) { + this.tag = tag; + } + + public String getScope() { + return scope; + } + + public void setScope(String scope) { + this.scope = scope; + } + + public String getDescription() { + return description; + } + + public void setDescription(String description) { + this.description = description; + } + + public Integer getStatus() { + return status; + } + + public void setStatus(Integer status) { + this.status = status; + } + + public String getType() { + return type; + } + + public void setType(String type) { + this.type = type; + } + + public Date getCreateTime() { + return createTime; + } + + public void setCreateTime(Date createTime) { + this.createTime = createTime; + } + + public Date getModifyTime() { + return modifyTime; + } + + public void setModifyTime(Date modifyTime) { + this.modifyTime = modifyTime; + } + + public String getCreator() { + return creator; + } + + public void setCreator(String creator) { + this.creator = creator; + } + + public String getModifier() { + return modifier; + } + + public void setModifier(String modifier) { + this.modifier = modifier; + } + + public String getContent() { + return content; + } + + public void setContent(String content) { + this.content = content; + } + + public String getScriptPath() { + return scriptPath; + } + + public void setScriptPath(String scriptPath) { + this.scriptPath = scriptPath; + } + + public Map getMetadata() { + return metadata; + } + + public void setMetadata(Map metadata) { + this.metadata = metadata; + } + + public List getVersionVos() { + return versionVos; + } + + public void setVersionVos(List versionVos) { + this.versionVos = versionVos; + } + + public List getParams() { + return params; + } + + public void setParams(List params) { + this.params = params; + } + + + public String getUserToken() { + return userToken; + } + + public void setUserToken(String userToken) { + this.userToken = userToken; + } + public String getRunType() { + return runType; + } + + public void setRunType(String runType) { + this.runType = runType; + } + + public String getAliasName() { + return aliasName; + } + + public void setAliasName(String aliasName) { + this.aliasName = aliasName; + } + +} \ No newline at end of file diff --git a/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/vo/ApiVersionVo.java b/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/vo/ApiVersionVo.java new file mode 100644 index 0000000000..18ab3c3b63 --- /dev/null +++ b/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/vo/ApiVersionVo.java @@ -0,0 +1,142 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package com.webank.wedatasphere.dss.apiservice.core.vo; + +import java.util.Date; +import java.util.List; + +/** + * @author jinyangrao + */ +public class ApiVersionVo { + + private Long id; + private Long apiId; + private String version; + private String bmlResourceId; + private String bmlVersion; + private String source; + private String creator; + private Date createTime; + private String metadataInfo; + private String authId; + + + public int getStatus() { + return status; + } + + public void setStatus(int status) { + this.status = status; + } + + private int status; + + /** + * one version data corresponds to multiple param data + * */ + private List paramVos; + + public List getParamVos() { + return paramVos; + } + + public String getMetadataInfo() { + return metadataInfo; + } + + public String getAuthId() { + return authId; + } + + public void setAuthId(String authId) { + this.authId = authId; + } + + public void setMetadataInfo(String metadataInfo) { + this.metadataInfo = metadataInfo; + } + + public void setParamVos(List paramVos) { + this.paramVos = paramVos; + } + + public Long getId() { + return id; + } + + public void setId(Long id) { + this.id = id; + } + + public Long getApiId() { + return apiId; + } + + public void setApiId(Long apiId) { + this.apiId = apiId; + } + + public String getVersion() { + return version; + } + + public void setVersion(String version) { + this.version = version; + } + + public String getBmlResourceId() { + return bmlResourceId; + } + + public void setBmlResourceId(String bmlResourceId) { + this.bmlResourceId = bmlResourceId; + } + + public String getBmlVersion() { + return bmlVersion; + } + + public void setBmlVersion(String bmlVersion) { + this.bmlVersion = bmlVersion; + } + + public String getSource() { + return source; + } + + public void setSource(String source) { + this.source = source; + } + + public String getCreator() { + return creator; + } + + public void setCreator(String creator) { + this.creator = creator; + } + + public Date getCreateTime() { + return createTime; + } + + public void setCreateTime(Date createTime) { + this.createTime = createTime; + } + + +} diff --git a/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/vo/ApprovalVo.java b/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/vo/ApprovalVo.java new file mode 100644 index 0000000000..ca0f3b329c --- /dev/null +++ b/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/vo/ApprovalVo.java @@ -0,0 +1,125 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.apiservice.core.vo; + +import java.util.Date; + +/** + * @author: jinyangrao + * @date: 2020-08-26 18:13:41 + */ +public class ApprovalVo { + Long id; + Long apiId; + Long apiVersionId; + String approvalName; + String applyUser; + String executeUser; + String creator; + Integer status; + Date createTime; + Date updateTime; + String approvalNo; + + public String getCreator() { + return creator; + } + + public Long getApiVersionId() { + return apiVersionId; + } + + public void setApiVersionId(Long apiVersionId) { + this.apiVersionId = apiVersionId; + } + + public void setCreator(String creator) { + this.creator = creator; + } + + public Long getId() { + return id; + } + + public void setId(Long id) { + this.id = id; + } + + public Long getApiId() { + return apiId; + } + + public void setApiId(Long apiId) { + this.apiId = apiId; + } + + public String getApprovalName() { + return approvalName; + } + + public void setApprovalName(String approvalName) { + this.approvalName = approvalName; + } + + public String getApplyUser() { + return applyUser; + } + + public void setApplyUser(String applyUser) { + this.applyUser = applyUser; + } + + public String getExecuteUser() { + return executeUser; + } + + public void setExecuteUser(String executeUser) { + this.executeUser = executeUser; + } + + public Integer getStatus() { + return status; + } + + public void setStatus(Integer status) { + this.status = status; + } + + public Date getCreateTime() { + return createTime; + } + + public void setCreateTime(Date createTime) { + this.createTime = createTime; + } + + public Date getUpdateTime() { + return updateTime; + } + + public void setUpdateTime(Date updateTime) { + this.updateTime = updateTime; + } + + public String getApprovalNo() { + return approvalNo; + } + + public void setApprovalNo(String approvalNo) { + this.approvalNo = approvalNo; + } +} \ No newline at end of file diff --git a/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/vo/MessageExtVo.java b/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/vo/MessageExtVo.java new file mode 100644 index 0000000000..3435e07b18 --- /dev/null +++ b/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/vo/MessageExtVo.java @@ -0,0 +1,40 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package com.webank.wedatasphere.dss.apiservice.core.vo; + +import java.util.LinkedHashMap; + +/** + * MessageExtVo + * + * @author lidongzhang + */ +public class MessageExtVo extends LinkedHashMap { + public static MessageExtVo build(MessageVo messageVo) { + MessageExtVo messageExtVo = new MessageExtVo(); + + messageExtVo.put("status", messageVo.getStatus()); + messageExtVo.put("message", messageVo.getMessage()); + messageExtVo.put("data", messageVo.getData()); + + return messageExtVo; + } + + public MessageExtVo setMessage(String message) { + put("message", message); + return this; + } +} diff --git a/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/vo/MessageVo.java b/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/vo/MessageVo.java new file mode 100644 index 0000000000..146d2b9a44 --- /dev/null +++ b/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/vo/MessageVo.java @@ -0,0 +1,65 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package com.webank.wedatasphere.dss.apiservice.core.vo; + +/** + * MessageVo + * + * @author lidongzhang + */ +public class MessageVo { + private Integer status = 0; + + private String message = "OK"; + + private Object data; + + public MessageVo() { + } + + public MessageVo(Integer status, String message, Object data) { + this.status = status; + this.message = message; + this.data = data; + } + + public Integer getStatus() { + return status; + } + + public MessageVo setStatus(Integer status) { + this.status = status; + return this; + } + + public String getMessage() { + return message; + } + + public MessageVo setMessage(String message) { + this.message = message; + return this; + } + + public Object getData() { + return data; + } + + public MessageVo setData(Object data) { + this.data = data; + return this; + } +} diff --git a/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/vo/ParamVo.java b/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/vo/ParamVo.java new file mode 100644 index 0000000000..13425163f6 --- /dev/null +++ b/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/vo/ParamVo.java @@ -0,0 +1,104 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package com.webank.wedatasphere.dss.apiservice.core.vo; + +/** + * @author jinyangrao + */ +public class ParamVo { + + private Long id; + private Long apiVersionId; + private String name; + private String type; + private Integer required = 1; + private String description; + private String displayName; + private String defaultValue; + private String details; + + public Long getId() { + return id; + } + + public void setId(Long id) { + this.id = id; + } + + public Long getApiVersionId() { + return apiVersionId; + } + + public void setApiVersionId(Long apiVersionId) { + this.apiVersionId = apiVersionId; + } + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public String getType() { + return type; + } + + public void setType(String type) { + this.type = type; + } + + public Integer getRequired() { + return required; + } + + public void setRequired(Integer required) { + this.required = required; + } + + public String getDescription() { + return description; + } + + public void setDescription(String description) { + this.description = description; + } + + public String getDisplayName() { + return displayName; + } + + public void setDisplayName(String displayName) { + this.displayName = displayName; + } + + public String getDefaultValue() { + return defaultValue; + } + + public void setDefaultValue(String defaultValue) { + this.defaultValue = defaultValue; + } + + public String getDetails() { + return details; + } + + public void setDetails(String details) { + this.details = details; + } +} diff --git a/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/vo/QueryParamVo.java b/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/vo/QueryParamVo.java new file mode 100644 index 0000000000..ed0f6f91a3 --- /dev/null +++ b/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/vo/QueryParamVo.java @@ -0,0 +1,52 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package com.webank.wedatasphere.dss.apiservice.core.vo; + +/** + * @author chongchuanbing + */ +public class QueryParamVo extends ParamVo { + + private String typeStr; + + private String requireStr; + + private String testValue; + + public String getTypeStr() { + return typeStr; + } + + public void setTypeStr(String typeStr) { + this.typeStr = typeStr; + } + + public String getRequireStr() { + return requireStr; + } + + public void setRequireStr(String requireStr) { + this.requireStr = requireStr; + } + + public String getTestValue() { + return testValue; + } + + public void setTestValue(String testValue) { + this.testValue = testValue; + } +} diff --git a/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/vo/TokenManagerVo.java b/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/vo/TokenManagerVo.java new file mode 100644 index 0000000000..35de30ad03 --- /dev/null +++ b/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/vo/TokenManagerVo.java @@ -0,0 +1,150 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.apiservice.core.vo; + +import java.util.Date; + +/** + * @author jinyangrao + */ +public class TokenManagerVo { + private Long id; + private Long apiId; + private Long apiVersionId; + public String publisher; + private String user; + private Date applyTime; + private Long duration; + private String reason; + private String ipWhitelist; + private Integer status; + private String caller; + private String accessLimit; + private String token; + private String applySource; + + public Long getId() { + return id; + } + + public void setId(Long id) { + this.id = id; + } + + public Long getApiId() { + return apiId; + } + + public void setApiId(Long apiId) { + this.apiId = apiId; + } + + public String getUser() { + return user; + } + + public void setUser(String user) { + this.user = user; + } + + public String getPublisher() { + return publisher; + } + + public void setPublisher(String publisher) { + this.publisher = publisher; + } + + public Date getApplyTime() { + return applyTime; + } + + public void setApplyTime(Date applyTime) { + this.applyTime = applyTime; + } + + public Long getDuration() { + return duration; + } + + public void setDuration(Long duration) { + this.duration = duration; + } + + public String getReason() { + return reason; + } + + public void setReason(String reason) { + this.reason = reason; + } + + public String getIpWhitelist() { + return ipWhitelist; + } + + public void setIpWhitelist(String ipWhitelist) { + this.ipWhitelist = ipWhitelist; + } + + public Integer getStatus() { + return status; + } + + public void setStatus(Integer status) { + this.status = status; + } + + public String getCaller() { + return caller; + } + + public void setCaller(String caller) { + this.caller = caller; + } + + public String getAccessLimit() { + return accessLimit; + } + + public void setAccessLimit(String accessLimit) { + this.accessLimit = accessLimit; + } + + public String getToken() { + return token; + } + + public void setToken(String token) { + this.token = token; + } + + public String getApplySource() { + return applySource; + } + + public void setApplySource(String applySource) { + this.applySource = applySource; + } + public Long getApiVersionId() { + return apiVersionId; + } + + public void setApiVersionId(Long apiVersionId) { + this.apiVersionId = apiVersionId; + } +} diff --git a/dss-apps/dss-apiservice-server/src/main/scala/com/webank/wedatasphere/dss/apiservice/DSSApiServiceServerApplication.scala b/dss-apps/dss-apiservice-server/src/main/scala/com/webank/wedatasphere/dss/apiservice/DSSApiServiceServerApplication.scala new file mode 100644 index 0000000000..289d999050 --- /dev/null +++ b/dss-apps/dss-apiservice-server/src/main/scala/com/webank/wedatasphere/dss/apiservice/DSSApiServiceServerApplication.scala @@ -0,0 +1,39 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.apiservice + +import com.webank.wedatasphere.dss.common.utils.DSSMainHelper +import com.webank.wedatasphere.linkis.DataWorkCloudApplication +import com.webank.wedatasphere.linkis.common.utils.{Logging, Utils} + + +object DSSApiServiceServerApplication extends Logging { + + val userName: String = System.getProperty("user.name") + val hostName: String = Utils.getComputerName + + def main(args: Array[String]): Unit = { + val serviceName = System.getProperty("serviceName")//ProjectConf.SERVICE_NAME.getValue + DSSMainHelper.formatPropertyFiles(serviceName) + val allArgs = args ++ DSSMainHelper.getExtraSpringOptions + System.setProperty("hostName", hostName) + System.setProperty("userName", userName) + info(s"Ready to start $serviceName with args: ${allArgs.toList}.") + println(s"Test Ready to start $serviceName with args: ${allArgs.toList}.") + DataWorkCloudApplication.main(allArgs) + } +} \ No newline at end of file diff --git a/dss-apps/dss-apiservice-server/src/test/java/com/webank/wedatasphere/dss/apiservice/test/TestApiServiceDBOperation.java b/dss-apps/dss-apiservice-server/src/test/java/com/webank/wedatasphere/dss/apiservice/test/TestApiServiceDBOperation.java new file mode 100644 index 0000000000..d778106938 --- /dev/null +++ b/dss-apps/dss-apiservice-server/src/test/java/com/webank/wedatasphere/dss/apiservice/test/TestApiServiceDBOperation.java @@ -0,0 +1,198 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.apiservice.test; + +import com.ninja_squad.dbsetup.DbSetup; +import com.ninja_squad.dbsetup.DbSetupTracker; +import com.ninja_squad.dbsetup.Operations; +import com.ninja_squad.dbsetup.destination.DataSourceDestination; +import com.ninja_squad.dbsetup.operation.Operation; +import com.webank.wedatasphere.dss.apiservice.core.bo.ApiServiceToken; +import com.webank.wedatasphere.dss.apiservice.core.dao.ApiServiceAccessDao; +import com.webank.wedatasphere.dss.apiservice.core.dao.ApiServiceDao; +import com.webank.wedatasphere.dss.apiservice.core.service.ApiService; +import com.webank.wedatasphere.dss.apiservice.core.token.JwtManager; +import com.webank.wedatasphere.dss.apiservice.core.util.DateUtil; +import com.webank.wedatasphere.dss.apiservice.core.vo.ApiAccessVo; +import com.webank.wedatasphere.dss.apiservice.core.vo.ApiServiceVo; +import com.webank.wedatasphere.dss.apiservice.core.vo.ApiVersionVo; +import com.webank.wedatasphere.dss.apiservice.core.vo.ApprovalVo; +import com.webank.wedatasphere.linkis.DataWorkCloudApplication; +import org.junit.Before; +import org.junit.Test; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.DisplayName; +import org.junit.runner.RunWith; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.test.context.junit4.SpringJUnit4ClassRunner; + +import javax.sql.DataSource; +import java.util.Date; +import java.util.List; + +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.equalTo; + +/** + * @author allenlliu + * @date 2020/10/19 03:29 PM + */ + +@RunWith(SpringJUnit4ClassRunner.class) +//@MapperScan(annotationClass = Repository.class, basePackages = "com.webank.wedatasphere.dss.apiservice.dao" ) +@SpringBootTest(classes = {DataWorkCloudApplication.class}) +public class TestApiServiceDBOperation { + private static DbSetupTracker dbSetupTracker = new DbSetupTracker(); + @Autowired + ApiService apiService; + @Autowired + DataSource dataSource; + @Autowired + ApiServiceDao apiServiceDao; + + + @Autowired + ApiServiceAccessDao apiServiceAccessDao; + + @Before + public void setUp() { + Operation operation = Operations.sequenceOf( + Operations.deleteAllFrom("dss_apiservice_api", "dss_apiservice_param"), + Operations.insertInto("dss_apiservice_api") + .columns("name", "alias_name", "path", "protocol", "method", "tag", "scope", "description", "status", "type", "run_type", "create_time", "modify_time", "creator", "modifier", "script_path", "workspaceID", "api_comment") + .values("test01", "测试", "/test01", 0, "get", "new test", "shouquankejian", "miaoshu", "1", "sql", "spark", "2020-10-19 11:00:00", "2020-11-19 12:00:00", "allenlliu", "allenlliu", "123.sql", "180", "comment test") + .build(), + Operations.insertInto("dss_apiservice_param") + .columns("api_version_id", "name", "display_name", "type", "required", "default_value", "description", "details") + .values(1, "idcard", "身份证", "String", 1, "430124000000000", "test param", "身份标识") + .build() +// Operations.insertInto("dss_apiservice_access_info") +// .columns("api_version_id", "login_user", "display_name", "type", "required", "default_value", "description", "details").build(), + ); + DbSetup dbSetup = new DbSetup(new DataSourceDestination(dataSource), operation); + dbSetupTracker.launchIfNecessary(dbSetup); + } + + @DisplayName("ApiService库表验证") + @Test + public void testApiServiceDBOperate() { + dbSetupTracker.skipNextLaunch(); + + Integer actual = apiServiceDao.enableApi(0L); + ApiServiceVo apiServiceVo = apiServiceDao.queryByPath("/test01"); + Integer count = apiService.queryCountByName("test01"); + + Assertions.assertEquals(apiServiceVo.getName(), "test01"); + assertThat(actual, equalTo(0)); + assertThat(count, equalTo(1)); + } + + @DisplayName("ApiService备注更新测试") + @Test + public void testApiServiceCommentUpdate() { + dbSetupTracker.skipNextLaunch(); + + ApiServiceVo apiServiceVo01 = apiServiceDao.queryByPath("/test01"); + Integer actual = apiServiceDao.updateApiServiceComment(apiServiceVo01.getId(), "new comment"); + ApiServiceVo apiServiceVo02 = apiServiceDao.queryByPath("/test01"); + + + Assertions.assertEquals(apiServiceVo02.getComment(), "new comment"); + } + + @DisplayName("ApiService保存和版本验证") + @Test + public void testApiServiceVersion() { + dbSetupTracker.skipNextLaunch(); + + + ApiServiceVo apiServiceVo = apiServiceDao.queryByPath("/test01"); + apiServiceVo.setName("test02"); + apiServiceVo.setPath("/test02"); + //BML会上传失败，没有配置实际环境 + Assertions.assertThrows(NullPointerException.class, () -> { + apiService.save(apiServiceVo); + ; + }); + + } + +// +// @DisplayName("DataMap单库表解析验证") +// @Test +// public void testDataMapApprovalTableParse() { +// System.out.println("DataMap单库表解析验证"); +// ApprovalVo approvalVo = new ApprovalVo(); +// approvalVo.setApiId(1L); +// ApiServiceVo apiServiceVo = new ApiServiceVo(); +// apiServiceVo.setApprovalVo(approvalVo); +// ApiVersionVo apiVersionVo = new ApiVersionVo(); +// String metaDtaInfo = "[default1.a1,default2.b]"; +// List dataMapApplyContentDataList = apiService.genDataMapApplyContentDatas(apiServiceVo, apiVersionVo, metaDtaInfo); +// Assertions.assertAll("tableNames", +// () -> Assertions.assertEquals(dataMapApplyContentDataList.get(0).getDbName(), "default1"), +// () -> Assertions.assertEquals(dataMapApplyContentDataList.get(1).getTableName(), "b") +// +// ); +// +// } + + @DisplayName("数据服务访问记录验证") + @Test + public void testApiServiceAccessInfo() { + System.out.println("数据服务访问记录验证"); + ApiAccessVo apiAccessVo = new ApiAccessVo(); + apiAccessVo.setUser("allenlliu"); + apiAccessVo.setApiPublisher("testUser"); + apiAccessVo.setApiServiceName("testApi"); + apiAccessVo.setApiServiceId(102L); + apiAccessVo.setApiServiceVersionId(10L); + apiAccessVo.setProxyUser("hadoop"); + apiAccessVo.setAccessTime(DateUtil.getNow()); + apiServiceAccessDao.addAccessRecord(apiAccessVo); + + ApiAccessVo targetAccessVo = apiServiceAccessDao.queryByVersionId(10L); + Assertions.assertAll("accessInfos", + () -> Assertions.assertEquals(targetAccessVo.getProxyUser(), "hadoop"), + () -> Assertions.assertEquals(targetAccessVo.getApiServiceName(), "testApi"), + () -> Assertions.assertEquals(targetAccessVo.getApiPublisher(), "testUser") + + ); + + } + + @DisplayName("Token 解析验证") + @Test + public void testTokenParse() { + System.out.println("Token解析验证"); + final String applyUser = "allenlliu"; + ApiServiceToken apiServiceToken = new ApiServiceToken(); + apiServiceToken.setPublisher("allenlliu"); + apiServiceToken.setApiServiceId(150L); + apiServiceToken.setApplyUser("testUser1"); + apiServiceToken.setApplyTime(new Date()); + Long duration = 365L; + String token = JwtManager.createToken(applyUser, apiServiceToken, duration); + + ApiServiceToken parseToken = JwtManager.parseToken(token); + + Assertions.assertEquals(parseToken.getApiServiceId(), 150L); + Assertions.assertEquals(parseToken.getApplyUser(), "testUser1"); + + } +} \ No newline at end of file diff --git a/dss-apps/dss-apiservice-server/src/test/resources/linkis.properties b/dss-apps/dss-apiservice-server/src/test/resources/linkis.properties new file mode 100644 index 0000000000..e68b6f9632 --- /dev/null +++ b/dss-apps/dss-apiservice-server/src/test/resources/linkis.properties @@ -0,0 +1,57 @@ +# +# Copyright 2019 WeBank +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# + +# mysql 驱动: h2 +spring.datasource.driver-class-name=org.h2.Driver +# h2 内存数据库 库名: test +spring.datasource.url=jdbc:h2:mem:test +# 初始化数据表 +spring.datasource.schema=classpath:schema.sql +spring.datasource.username= +spring.datasource.password= +# 打印 SQL语句, Mapper所处的包 +logging.level.com.hawkingfoo.dao=debug + + + +wds.linkis.server.mybatis.mapperLocations=classpath:com/webank/wedatasphere/dss/apiservice/core/dao/mapper/*.xml +wds.linkis.server.mybatis.typeAliasesPackage=com.webank.wedatasphere.dss.apiservice.core +wds.linkis.server.mybatis.BasePackage=com.webank.wedatasphere.dss.apiservice.core.dao + +wds.linkis.server.restful.scan.packages=com.webank.wedatasphere.dss.apiservice.core.restful + +#sit +wds.linkis.server.version=v1 +wds.linkis.server.url= + +#test +wds.linkis.test.mode=false +wds.linkis.test.user= +wds.linkis.server.mybatis.datasource.url=jdbc:h2:mem:test +wds.linkis.server.mybatis.datasource.username= +***REMOVED*** + +#dsm +wds.linkis.server.dsm.admin.users= + +#bml +wds.linkis.gateway.url= +wds.linkis.gateway.ip= +wds.linkis.gateway.port= + +#用于执行的datasource配置 +wds.linkis.datasource.hikari.maximumPoolSize=100 +wds.linkis.datasource.hikari.minimumIdle=10 diff --git a/dss-apps/dss-apiservice-server/src/test/resources/schema.sql b/dss-apps/dss-apiservice-server/src/test/resources/schema.sql new file mode 100644 index 0000000000..905024b936 --- /dev/null +++ b/dss-apps/dss-apiservice-server/src/test/resources/schema.sql @@ -0,0 +1,66 @@ +CREATE TABLE dss_apiservice_api ( + id bigint(20) NOT NULL AUTO_INCREMENT COMMENT '主键', + name varchar(200) NOT NULL COMMENT '服务名称', + alias_name varchar(200) NOT NULL COMMENT '服务中文名称', + path varchar(200) NOT NULL COMMENT '服务路径', + protocol int(11) NOT NULL COMMENT '协议: http, https', + method varchar(10) NOT NULL COMMENT '方法： post, put, delete', + tag varchar(200) DEFAULT NULL COMMENT '标签', + scope varchar(50) DEFAULT NULL COMMENT '范围', + description varchar(200) DEFAULT NULL COMMENT '服务描述', + status int(11) DEFAULT '0' COMMENT '服务状态，默认0是停止，1是运行中', + type varchar(50) DEFAULT NULL COMMENT '服务引擎类型', + run_type varchar(50) DEFAULT NULL COMMENT '脚本类型', + create_time timestamp NOT NULL COMMENT '创建时间', + modify_time timestamp NOT NULL COMMENT '修改时间', + creator varchar(50) DEFAULT NULL COMMENT '创建者', + modifier varchar(50) DEFAULT NULL COMMENT '修改者', + script_path varchar(200) NOT NULL COMMENT '脚本路径', + workspaceID int(11) NOT NULL COMMENT '工作空间ID', + api_comment varchar(1024) DEFAULT NULL COMMENT '服务备注', + PRIMARY KEY ( id ) +) COMMENT='服务api配置表'; + + +CREATE TABLE dss_apiservice_param ( + id bigint(20) NOT NULL AUTO_INCREMENT COMMENT '主键', + api_version_id bigint(20) NOT NULL COMMENT '服务api版本id', + name varchar(200) NOT NULL COMMENT '名称', + display_name varchar(50) DEFAULT NULL COMMENT '展示名', + type varchar(50) DEFAULT NULL COMMENT '类型', + required tinyint(1) DEFAULT '1' COMMENT '是否必须: 0否, 1是', + default_value varchar(200) DEFAULT NULL COMMENT '参数的默认值', + description varchar(200) DEFAULT NULL COMMENT '描述', + details varchar(500) DEFAULT NULL COMMENT '变量的详细说明', + PRIMARY KEY ( id ) +) COMMENT='apiservice 参数表'; + + +CREATE TABLE `dss_apiservice_api_version` ( + `id` bigint(20) NOT NULL AUTO_INCREMENT COMMENT '主键', + `api_id` bigint(20) NOT NULL COMMENT '服务的ID', + `version` varchar(50) NOT NULL COMMENT '服务对应的版本信息', + `bml_resource_id` varchar(50) NOT NULL COMMENT 'bml资源id', + `bml_version` varchar(20) NOT NULL COMMENT 'bml版本', + `source` varchar(200) DEFAULT NULL COMMENT '来源', + `creator` varchar(50) DEFAULT NULL COMMENT '创建者', + `create_time`timestamp NOT NULL COMMENT '创建时间', + `status` tinyint(1) default '1' COMMENT '0表示被禁用，1表示正在运行', + `metadata_info` varchar(5000) NOT NULL COMMENT '发布者库表信息', + `auth_id` varchar(200) NOT NULL COMMENT '用于与datamap交互的UUID', + `datamap_order_no` varchar(200) DEFAULT NULL COMMENT 'datamap审批单号码', + PRIMARY KEY(`id`) +) COMMENT='服务api版本表'; + + +CREATE TABLE `dss_apiservice_access_info` ( + `id` bigint(20) NOT NULL AUTO_INCREMENT COMMENT '主键', + `api_id` bigint(20) NOT NULL COMMENT '服务id', + `api_version_id` bigint(20) NOT NULL COMMENT '版本id', + `api_name` varchar(50) NOT NULL COMMENT '服务名称', + `login_user` varchar(50) NOT NULL COMMENT '提交用户', + `execute_user` varchar(50) DEFAULT NULL COMMENT '代理执行用户', + `api_publisher` varchar(50) NOT NULL COMMENT 'api创建者', + `access_time` timestamp NOT null COMMENT '访问时间', + PRIMARY KEY(`id`) +) COMMENT='apiservice 访问信息表'; \ No newline at end of file diff --git a/dss-commons/dss-common/pom.xml b/dss-commons/dss-common/pom.xml new file mode 100644 index 0000000000..1a91fcd17a --- /dev/null +++ b/dss-commons/dss-common/pom.xml @@ -0,0 +1,116 @@ + + + + + 4.0.0 + + dss-commons + com.webank.wedatasphere.dss + 1.0.0 + + dss-common + + + + com.webank.wedatasphere.linkis + linkis-common + + + com.google.code.gson + gson + + + com.fasterxml.jackson.core + jackson-databind + + + org.codehaus.jackson + jackson-mapper-asl + + + org.scala-lang + scala-library + + + org.scala-lang + scala-compiler + + + org.scala-lang + scala-reflect + + + org.scala-lang + scalap + + + commons-lang + commons-lang + + + com.webank.wedatasphere.linkis + linkis-label-common + ${linkis.version} + + + com.webank.wedatasphere.linkis + linkis-protocol + ${linkis.version} + + + + + + + org.apache.maven.plugins + maven-compiler-plugin + 3.3 + + 1.8 + 1.8 + UTF-8 + + + + org.apache.maven.plugins + maven-deploy-plugin + + + + net.alchim31.maven + scala-maven-plugin + + + org.apache.maven.plugins + maven-jar-plugin + + + + + + product + + 1.0.0-SNAPSHOT + + + + + + + \ No newline at end of file diff --git a/dss-commons/dss-common/src/main/java/com/webank/wedatasphere/dss/common/conf/DSSCommonConf.java b/dss-commons/dss-common/src/main/java/com/webank/wedatasphere/dss/common/conf/DSSCommonConf.java new file mode 100644 index 0000000000..161bfb31c7 --- /dev/null +++ b/dss-commons/dss-common/src/main/java/com/webank/wedatasphere/dss/common/conf/DSSCommonConf.java @@ -0,0 +1,37 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.common.conf; + +import com.webank.wedatasphere.linkis.common.conf.CommonVars; + +/** + * @Author alexyang + * @Date 2020/3/27 + */ +public class DSSCommonConf { + + public static final CommonVars DSS_IO_ENV = CommonVars.apply("wds.dss.server.io.env", "BDAP_DEV"); + public static final CommonVars DSS_EXPORT_ENV = CommonVars.apply("wds.dss.server.export.env", "生产中心"); + + public static final CommonVars DSS_EXPORT_URL = CommonVars.apply("wds.dss.server.export.url", "/appcom/tmp/dss"); + public static final CommonVars DSS_INPUT_TOKEN = CommonVars.apply("wds.dss.server.input.token", "QML-AUTH"); + + public static final CommonVars DSS_UPLOAD_PATH = CommonVars.apply("wds.dss.file.upload.dir", "/appcom/tmp/uploads"); + public static final CommonVars DSS_INPUT_RESTFUL_URL = CommonVars.apply("wds.dss.server.input.url", "http://localhost:9004/api/rest_j/v1/dss/input"); + + +} diff --git a/dss-commons/dss-common/src/main/java/com/webank/wedatasphere/dss/common/entity/DSSWorkspace.java b/dss-commons/dss-common/src/main/java/com/webank/wedatasphere/dss/common/entity/DSSWorkspace.java new file mode 100644 index 0000000000..c5d85d61d8 --- /dev/null +++ b/dss-commons/dss-common/src/main/java/com/webank/wedatasphere/dss/common/entity/DSSWorkspace.java @@ -0,0 +1,26 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.common.entity; + +/** + * Created by enjoyyin on 2021/7/1. + */ +public interface DSSWorkspace { + + String getWorkspaceName(); + +} diff --git a/dss-commons/dss-common/src/main/java/com/webank/wedatasphere/dss/common/entity/IOEnv.java b/dss-commons/dss-common/src/main/java/com/webank/wedatasphere/dss/common/entity/IOEnv.java new file mode 100644 index 0000000000..a4f8e248ae --- /dev/null +++ b/dss-commons/dss-common/src/main/java/com/webank/wedatasphere/dss/common/entity/IOEnv.java @@ -0,0 +1,24 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.common.entity; + +public enum IOEnv { + /** + * + */ + BDP_DEV,BDP_PRODUCTION,BDAP_DEV,BDAP_PRODUCTION +} diff --git a/dss-commons/dss-common/src/main/java/com/webank/wedatasphere/dss/common/entity/IOType.java b/dss-commons/dss-common/src/main/java/com/webank/wedatasphere/dss/common/entity/IOType.java new file mode 100644 index 0000000000..5d7f235360 --- /dev/null +++ b/dss-commons/dss-common/src/main/java/com/webank/wedatasphere/dss/common/entity/IOType.java @@ -0,0 +1,26 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.common.entity; + +public enum IOType { + /** + * 区分工程导出和工作流导出 + * basepath/IO.properties + * type=PROJECT 或者 type=FLOE + */ + PROJECT,FLOW,ORCHESTRATOR +} diff --git a/dss-commons/dss-common/src/main/java/com/webank/wedatasphere/dss/common/entity/InputRelation.java b/dss-commons/dss-common/src/main/java/com/webank/wedatasphere/dss/common/entity/InputRelation.java new file mode 100644 index 0000000000..a6a02a4916 --- /dev/null +++ b/dss-commons/dss-common/src/main/java/com/webank/wedatasphere/dss/common/entity/InputRelation.java @@ -0,0 +1,75 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.common.entity; + +public class InputRelation { + + private Long id; + private String type; + private IOEnv sourceEnv; + private Long sourceID; + private IOEnv targetEnv; + private Long targetID; + + public Long getId() { + return id; + } + + public void setId(Long id) { + this.id = id; + } + + public String getType() { + return type; + } + + public void setType(String type) { + this.type = type; + } + + public IOEnv getSourceEnv() { + return sourceEnv; + } + + public void setSourceEnv(IOEnv sourceEnv) { + this.sourceEnv = sourceEnv; + } + + public Long getSourceID() { + return sourceID; + } + + public void setSourceID(Long sourceID) { + this.sourceID = sourceID; + } + + public IOEnv getTargetEnv() { + return targetEnv; + } + + public void setTargetEnv(IOEnv targetEnv) { + this.targetEnv = targetEnv; + } + + public Long getTargetID() { + return targetID; + } + + public void setTargetID(Long targetID) { + this.targetID = targetID; + } +} diff --git a/dss-commons/dss-common/src/main/java/com/webank/wedatasphere/dss/common/entity/PublishType.java b/dss-commons/dss-common/src/main/java/com/webank/wedatasphere/dss/common/entity/PublishType.java new file mode 100644 index 0000000000..cdf14a1ab1 --- /dev/null +++ b/dss-commons/dss-common/src/main/java/com/webank/wedatasphere/dss/common/entity/PublishType.java @@ -0,0 +1,25 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.common.entity; + +/** + * @author allenlliu + * @date 2020/12/16 11:53 + */ +public enum PublishType { + FULL,DELTA +} diff --git a/dss-commons/dss-common/src/main/java/com/webank/wedatasphere/dss/common/entity/Resource.java b/dss-commons/dss-common/src/main/java/com/webank/wedatasphere/dss/common/entity/Resource.java new file mode 100644 index 0000000000..0dbc3d09d5 --- /dev/null +++ b/dss-commons/dss-common/src/main/java/com/webank/wedatasphere/dss/common/entity/Resource.java @@ -0,0 +1,58 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.common.entity; + +import java.io.Serializable; + +public class Resource implements Serializable { + private String fileName; + private String resourceId; + private String version; + + public String getFileName() { + return fileName; + } + + public void setFileName(String fileName) { + this.fileName = fileName; + } + + public String getResourceId() { + return resourceId; + } + + public void setResourceId(String resourceId) { + this.resourceId = resourceId; + } + + public String getVersion() { + return version; + } + + public void setVersion(String version) { + this.version = version; + } + +/* @Override + public String toString() { + return "{" + + "fileName='" + fileName + '\'' + + ", resourceId='" + resourceId + '\'' + + ", version='" + version + '\'' + + '}'; + }*/ +} diff --git a/dss-commons/dss-common/src/main/java/com/webank/wedatasphere/dss/common/entity/node/DSSEdge.java b/dss-commons/dss-common/src/main/java/com/webank/wedatasphere/dss/common/entity/node/DSSEdge.java new file mode 100644 index 0000000000..1d64eb7595 --- /dev/null +++ b/dss-commons/dss-common/src/main/java/com/webank/wedatasphere/dss/common/entity/node/DSSEdge.java @@ -0,0 +1,36 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.common.entity.node; + +public interface DSSEdge { + + String getSource(); + + void setSource(); + + String getTarget(); + + void setTarget(String target); + + String getSourceLocation(); + + void setSourceLocation(String sourceLocation); + + String getTargetLocation(); + + void setTargetLocation(String targetLocation); +} diff --git a/dss-commons/dss-common/src/main/java/com/webank/wedatasphere/dss/common/entity/node/DSSEdgeDefault.java b/dss-commons/dss-common/src/main/java/com/webank/wedatasphere/dss/common/entity/node/DSSEdgeDefault.java new file mode 100644 index 0000000000..c6c6389162 --- /dev/null +++ b/dss-commons/dss-common/src/main/java/com/webank/wedatasphere/dss/common/entity/node/DSSEdgeDefault.java @@ -0,0 +1,74 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.common.entity.node; + +public class DSSEdgeDefault implements DSSEdge { + private String source; + private String target; + private String sourceLocation; + private String targetLocation; + + @Override + public String getSource() { + return source; + } + + @Override + public void setSource() { + this.source = source; + } + + @Override + public String getTarget() { + return target; + } + + @Override + public void setTarget(String target) { + this.target = target; + } + + @Override + public String getSourceLocation() { + return sourceLocation; + } + + @Override + public void setSourceLocation(String sourceLocation) { + this.sourceLocation = sourceLocation; + } + + @Override + public String getTargetLocation() { + return targetLocation; + } + + @Override + public void setTargetLocation(String targetLocation) { + this.targetLocation = targetLocation; + } + + @Override + public String toString() { + return "DWSEdge{" + + "source='" + source + '\'' + + ", target='" + target + '\'' + + ", sourceLocation='" + sourceLocation + '\'' + + ", targetLocation='" + targetLocation + '\'' + + '}'; + } +} diff --git a/dss-commons/dss-common/src/main/java/com/webank/wedatasphere/dss/common/entity/node/DSSNode.java b/dss-commons/dss-common/src/main/java/com/webank/wedatasphere/dss/common/entity/node/DSSNode.java new file mode 100644 index 0000000000..c37438e53f --- /dev/null +++ b/dss-commons/dss-common/src/main/java/com/webank/wedatasphere/dss/common/entity/node/DSSNode.java @@ -0,0 +1,45 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.common.entity.node; + +import com.webank.wedatasphere.dss.common.entity.Resource; + +import java.util.List; +import java.util.Map; + +public interface DSSNode extends Node { + + Layout getLayout(); + + void setLayout(Layout layout); + + Map getParams(); + + void setParams(Map params); + + List getResources(); + + void setResources(List resources); + + Map getJobContent(); + + void setJobContent(Map jobContent); + + String getUserProxy(); + + void setUserProxy(String userProxy); +} diff --git a/dss-commons/dss-common/src/main/java/com/webank/wedatasphere/dss/common/entity/node/DSSNodeDefault.java b/dss-commons/dss-common/src/main/java/com/webank/wedatasphere/dss/common/entity/node/DSSNodeDefault.java new file mode 100644 index 0000000000..72d24fefab --- /dev/null +++ b/dss-commons/dss-common/src/main/java/com/webank/wedatasphere/dss/common/entity/node/DSSNodeDefault.java @@ -0,0 +1,158 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.common.entity.node; + +import com.webank.wedatasphere.dss.common.entity.Resource; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +public class DSSNodeDefault implements DSSNode { + private Layout layout; + private String id; + private String jobType; + private String lastUpdateTime; + private Map params; + private List resources; + private String title; + private String desc; + private String createTime; + private String userProxy; + /** + * dependencys 是该Node的依赖节点 + */ + private List dependencys = new ArrayList(); + /** + * jobContent 是node的内容 + */ + private Map jobContent; + + + @Override + public Layout getLayout() { + return layout; + } + + + @Override + public void setLayout(Layout layout) { + this.layout = layout; + } + + @Override + public Map getParams() { + return params; + } + + @Override + public void setParams(Map params) { + this.params = params; + } + + @Override + public List getResources() { + return resources; + } + + @Override + public void setResources(List resources) { + this.resources = resources; + } + + @Override + public Map getJobContent() { + return jobContent; + } + + @Override + public void setJobContent(Map jobContent) { + this.jobContent = jobContent; + } + + @Override + public String getUserProxy() { + return userProxy; + } + + @Override + public void setUserProxy(String userProxy) { + this.userProxy = userProxy; + } + + @Override + public String getId() { + return id; + } + + @Override + public void setId(String id) { + this.id = id; + } + + @Override + public String getNodeType() { + return jobType; + } + + @Override + public void setNodeType(String nodeType) { + this.jobType = nodeType; + } + + @Override + public String getName() { + return title; + } + + @Override + public void setName(String name) { + this.title = name; + } + + @Override + public void addDependency(String nodeName) { + dependencys.add(nodeName); + } + + @Override + public void setDependency(List dependency) { + this.dependencys = dependency; + } + + @Override + public void removeDependency(String nodeName) { + dependencys.remove(nodeName); + } + + @Override + public List getDependencys() { + return dependencys; + } + + @Override + public String toString() { + return "dwsNode{" + + "layout=" + layout + + ", id='" + id + '\'' + + ", jobType='" + jobType + '\'' + + ", lastUpdateTime=" + lastUpdateTime + + ", params=" + params + + ", resources=" + resources + + '}'; + } +} diff --git a/dss-commons/dss-common/src/main/java/com/webank/wedatasphere/dss/common/entity/node/Layout.java b/dss-commons/dss-common/src/main/java/com/webank/wedatasphere/dss/common/entity/node/Layout.java new file mode 100644 index 0000000000..3d70f93efe --- /dev/null +++ b/dss-commons/dss-common/src/main/java/com/webank/wedatasphere/dss/common/entity/node/Layout.java @@ -0,0 +1,69 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.common.entity.node; + +/** + * Created by v_wbjftang on 2019/11/12. + */ +public class Layout { + private float x; + private float y; + private float width; + private float height; + + public float getX() { + return x; + } + + public void setX(float x) { + this.x = x; + } + + public float getY() { + return y; + } + + public void setY(float y) { + this.y = y; + } + + public float getWidth() { + return width; + } + + public void setWidth(float width) { + this.width = width; + } + + public float getHeight() { + return height; + } + + public void setHeight(float height) { + this.height = height; + } + + @Override + public String toString() { + return "Layout{" + + "x=" + x + + ", y=" + y + + ", width=" + width + + ", height=" + height + + '}'; + } +} diff --git a/dss-commons/dss-common/src/main/java/com/webank/wedatasphere/dss/common/entity/node/Node.java b/dss-commons/dss-common/src/main/java/com/webank/wedatasphere/dss/common/entity/node/Node.java new file mode 100644 index 0000000000..c3379329b0 --- /dev/null +++ b/dss-commons/dss-common/src/main/java/com/webank/wedatasphere/dss/common/entity/node/Node.java @@ -0,0 +1,45 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.common.entity.node; + + +import java.util.List; + +/* + Node 接口 + */ +public interface Node { + String getId(); + + void setId(String id); + + String getNodeType(); + + void setNodeType(String nodeType); + + String getName(); + + void setName(String name); + + void addDependency(String nodeName); + + void setDependency(List dependency); + + void removeDependency(String nodeName); + + List getDependencys(); +} diff --git a/dss-commons/dss-common/src/main/java/com/webank/wedatasphere/dss/common/entity/project/DSSProject.java b/dss-commons/dss-common/src/main/java/com/webank/wedatasphere/dss/common/entity/project/DSSProject.java new file mode 100644 index 0000000000..1beb715287 --- /dev/null +++ b/dss-commons/dss-common/src/main/java/com/webank/wedatasphere/dss/common/entity/project/DSSProject.java @@ -0,0 +1,193 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.common.entity.project; + +import java.util.Date; +import java.util.List; + +public class DSSProject implements Project { + + private Long id; + private String name; + private String description; + private String createBy; + + private String username; + private Date createTime; + private Date updateTime; + + private String product; + private Integer applicationArea; + private String business; + private Integer workspaceId; + private String workspaceName; + + //stitched + private List viewUsers; + private List editUsers; + private List executeUsers; + + + @Override + public Long getId() { + return id; + } + + @Override + public void setId(Long id) { + this.id = id; + } + + @Override + public String getName() { + return name; + } + + @Override + public void setName(String name) { + this.name = name; + } + + @Override + public String getDescription() { + return description; + } + + @Override + public void setDescription(String description) { + this.description = description; + } + + @Override + public String getCreateBy() { + return createBy; + } + + @Override + public void setCreateBy(String createBy) { + this.createBy = createBy; + } + + public String getUsername() { + return username; + } + + public void setUsername(String username) { + this.username = username; + } + + public Date getCreateTime() { + return createTime; + } + + public void setCreateTime(Date createTime) { + this.createTime = createTime; + } + + public Date getUpdateTime() { + return updateTime; + } + + public void setUpdateTime(Date updateTime) { + this.updateTime = updateTime; + } + + public String getProduct() { + return product; + } + + public void setProduct(String product) { + this.product = product; + } + + public Integer getApplicationArea() { + return applicationArea; + } + + public void setApplicationArea(Integer applicationArea) { + this.applicationArea = applicationArea; + } + + public String getBusiness() { + return business; + } + + public void setBusiness(String business) { + this.business = business; + } + + public Integer getWorkspaceId() { + return workspaceId; + } + + public void setWorkspaceId(Integer workspaceId) { + this.workspaceId = workspaceId; + } + + public List getViewUsers() { + return viewUsers; + } + + public void setViewUsers(List viewUsers) { + this.viewUsers = viewUsers; + } + + public List getEditUsers() { + return editUsers; + } + + public void setEditUsers(List editUsers) { + this.editUsers = editUsers; + } + + public List getExecuteUsers() { + return executeUsers; + } + + public void setExecuteUsers(List executeUsers) { + this.executeUsers = executeUsers; + } + + public String getWorkspaceName() { + return workspaceName; + } + + public void setWorkspaceName(String workspaceName) { + this.workspaceName = workspaceName; + } + + @Override + public String toString() { + return "DSSProject{" + + "id=" + id + + ", name='" + name + '\'' + + ", description='" + description + '\'' + + ", createBy='" + createBy + '\'' + + ", username='" + username + '\'' + + ", createTime=" + createTime + + ", updateTime=" + updateTime + + ", product='" + product + '\'' + + ", applicationArea=" + applicationArea + + ", business='" + business + '\'' + + ", workspaceId=" + workspaceId + + ", workspaceName='" + workspaceName + '\'' + + ", viewUsers=" + viewUsers + + ", editUsers=" + editUsers + + ", executeUsers=" + executeUsers + + '}'; + } +} diff --git a/dss-commons/dss-common/src/main/java/com/webank/wedatasphere/dss/common/entity/project/Project.java b/dss-commons/dss-common/src/main/java/com/webank/wedatasphere/dss/common/entity/project/Project.java new file mode 100644 index 0000000000..4043ce8a8b --- /dev/null +++ b/dss-commons/dss-common/src/main/java/com/webank/wedatasphere/dss/common/entity/project/Project.java @@ -0,0 +1,40 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.common.entity.project; + +/** + * Created by v_wbjftang on 2019/9/16. + */ +public interface Project { + Long getId(); + + void setId(Long id); + + String getName(); + + void setName(String name); + + String getDescription(); + + void setDescription(String description); + + String getCreateBy(); + + void setCreateBy(String createBy); + + +} diff --git a/dss-commons/dss-common/src/main/java/com/webank/wedatasphere/dss/common/exception/DSSErrorException.java b/dss-commons/dss-common/src/main/java/com/webank/wedatasphere/dss/common/exception/DSSErrorException.java new file mode 100644 index 0000000000..9eeafada5f --- /dev/null +++ b/dss-commons/dss-common/src/main/java/com/webank/wedatasphere/dss/common/exception/DSSErrorException.java @@ -0,0 +1,33 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.common.exception; + +import com.webank.wedatasphere.linkis.common.exception.ErrorException; + +/** + * Created by v_wbjftang on 2019/11/6. + */ +public class DSSErrorException extends ErrorException { + + public DSSErrorException(int errCode, String desc) { + super(errCode, desc); + } + + public DSSErrorException(int errCode, String desc, String ip, int port, String serviceKind) { + super(errCode, desc, ip, port, serviceKind); + } +} diff --git a/dss-commons/dss-common/src/main/java/com/webank/wedatasphere/dss/common/exception/DSSRuntimeException.java b/dss-commons/dss-common/src/main/java/com/webank/wedatasphere/dss/common/exception/DSSRuntimeException.java new file mode 100644 index 0000000000..f3b82edfa8 --- /dev/null +++ b/dss-commons/dss-common/src/main/java/com/webank/wedatasphere/dss/common/exception/DSSRuntimeException.java @@ -0,0 +1,38 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.common.exception; + +import com.webank.wedatasphere.linkis.common.exception.WarnException; + +/** + * Created by v_wbjftang on 2019/9/24. + */ +public class DSSRuntimeException extends WarnException { + + public DSSRuntimeException(String msg){ + super(100000, msg); + } + + public DSSRuntimeException(int errorCode, String msg) { + super(errorCode, msg); + } + + public DSSRuntimeException(int errorCode, String msg, Throwable e) { + super(errorCode, msg); + initCause(e); + } +} diff --git a/dss-commons/dss-common/src/main/java/com/webank/wedatasphere/dss/common/exception/ErrorCode.java b/dss-commons/dss-common/src/main/java/com/webank/wedatasphere/dss/common/exception/ErrorCode.java new file mode 100644 index 0000000000..1e20b247c8 --- /dev/null +++ b/dss-commons/dss-common/src/main/java/com/webank/wedatasphere/dss/common/exception/ErrorCode.java @@ -0,0 +1,28 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.common.exception; + +/** + * @Author alexyang + * @Date 2020/3/20 + */ +public class ErrorCode { + + public static final int INVALID_PARAMS = 71001; + + +} diff --git a/dss-commons/dss-common/src/main/java/com/webank/wedatasphere/dss/common/exception/ThrowingConsumer.java b/dss-commons/dss-common/src/main/java/com/webank/wedatasphere/dss/common/exception/ThrowingConsumer.java new file mode 100644 index 0000000000..9c69b1c13c --- /dev/null +++ b/dss-commons/dss-common/src/main/java/com/webank/wedatasphere/dss/common/exception/ThrowingConsumer.java @@ -0,0 +1,25 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.common.exception; + +/** + * Created by v_wbjftang on 2019/9/20. + */ +@FunctionalInterface +public interface ThrowingConsumer { + void accept(T t) throws E; +} diff --git a/dss-commons/dss-common/src/main/java/com/webank/wedatasphere/dss/common/exception/ThrowingFunction.java b/dss-commons/dss-common/src/main/java/com/webank/wedatasphere/dss/common/exception/ThrowingFunction.java new file mode 100644 index 0000000000..a89056aa6f --- /dev/null +++ b/dss-commons/dss-common/src/main/java/com/webank/wedatasphere/dss/common/exception/ThrowingFunction.java @@ -0,0 +1,25 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.common.exception; + +/** + * Created by v_wbjftang on 2019/11/13. + */ +@FunctionalInterface +public interface ThrowingFunction { + R accept(T t) throws E; +} diff --git a/dss-commons/dss-common/src/main/java/com/webank/wedatasphere/dss/common/label/DSSLabel.java b/dss-commons/dss-common/src/main/java/com/webank/wedatasphere/dss/common/label/DSSLabel.java new file mode 100644 index 0000000000..0a5eee9f25 --- /dev/null +++ b/dss-commons/dss-common/src/main/java/com/webank/wedatasphere/dss/common/label/DSSLabel.java @@ -0,0 +1,25 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.common.label; + +import com.webank.wedatasphere.linkis.manager.label.entity.GenericLabel; + +/** + * The basic label class of DSS. All other labels of DSS must extends this class. + */ +public class DSSLabel extends GenericLabel { +} diff --git a/dss-commons/dss-common/src/main/java/com/webank/wedatasphere/dss/common/label/DSSLabelUtil.java b/dss-commons/dss-common/src/main/java/com/webank/wedatasphere/dss/common/label/DSSLabelUtil.java new file mode 100644 index 0000000000..27cb5900fd --- /dev/null +++ b/dss-commons/dss-common/src/main/java/com/webank/wedatasphere/dss/common/label/DSSLabelUtil.java @@ -0,0 +1,65 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.common.label; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +/** + * 生成DSSLabel工具类 + */ +public class DSSLabelUtil { + /** + * 临时构建标准labe map + */ + private static Map consltructLabelMap(String label) { + HashMap labelMap = new HashMap<>(); + labelMap.put(LabelKeyConvertor.ROUTE_LABEL_KEY, label); + return labelMap; + } + + /** + * 生成DSSLabel list + * @param label + * @return + */ + public static List createLabelList(String label){ + Map labelParam = consltructLabelMap(label); + LabelInstanceFactory labelInstanceFactory = new LabelInstanceFactory(labelParam); + List dssLabelList = labelInstanceFactory.getLabelList(); + return dssLabelList; + } + + /** + * 生成单一 DSSLabel + * @param label + * @return + */ + public static EnvDSSLabel createLabel(String label){ + Map labelParam = consltructLabelMap(label); + LabelInstanceFactory labelInstanceFactory = new LabelInstanceFactory(labelParam); + EnvDSSLabel envDSSLabel = labelInstanceFactory.getEnvDssLabel(); + return envDSSLabel; + } + + public static void main(String[] args) { + List labelList = DSSLabelUtil.createLabelList("dev"); + System.out.println(labelList.size()); + } + +} diff --git a/dss-commons/dss-common/src/main/java/com/webank/wedatasphere/dss/common/label/EnvDSSLabel.java b/dss-commons/dss-common/src/main/java/com/webank/wedatasphere/dss/common/label/EnvDSSLabel.java new file mode 100644 index 0000000000..1595101ea6 --- /dev/null +++ b/dss-commons/dss-common/src/main/java/com/webank/wedatasphere/dss/common/label/EnvDSSLabel.java @@ -0,0 +1,68 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.common.label; + +import com.webank.wedatasphere.linkis.manager.label.builder.factory.LabelBuilderFactoryContext; +import com.webank.wedatasphere.linkis.manager.label.entity.Feature; +import com.webank.wedatasphere.linkis.manager.label.entity.annon.ValueSerialNum; +import java.util.HashMap; +import java.util.Map; + +/** + * Used to mark the env of this micro-services. + */ +public class EnvDSSLabel extends DSSLabel { + + public static final String DSS_ENV_LABEL_KEY = "DSSEnv"; + + public EnvDSSLabel(String env) { + setLabelKey(DSS_ENV_LABEL_KEY); + setEnv(env); + } + + public EnvDSSLabel() { + setLabelKey(DSS_ENV_LABEL_KEY); + } + + @Override + public Feature getFeature() { + return Feature.CORE; + } + + public String getEnv() { + if (null == getValue()) { + return null; + } + return getValue().get(DSS_ENV_LABEL_KEY); + } + + @ValueSerialNum(0) + public void setEnv(String env) { + if (null == getValue()) { + setValue(new HashMap<>(1)); + } + getValue().put(DSS_ENV_LABEL_KEY, env); + } + + public static void main(String[] args) { + Map map = new HashMap<>(); + map.put(DSS_ENV_LABEL_KEY, "dev"); + EnvDSSLabel label = (EnvDSSLabel) LabelBuilderFactoryContext.getLabelBuilderFactory().getLabels(map).get(0); + System.out.println("label: " + label.getEnv()); + } + +} diff --git a/dss-commons/dss-common/src/main/java/com/webank/wedatasphere/dss/common/label/LabelInstanceFactory.java b/dss-commons/dss-common/src/main/java/com/webank/wedatasphere/dss/common/label/LabelInstanceFactory.java new file mode 100644 index 0000000000..3c0ddbe65c --- /dev/null +++ b/dss-commons/dss-common/src/main/java/com/webank/wedatasphere/dss/common/label/LabelInstanceFactory.java @@ -0,0 +1,60 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.common.label; + +import com.webank.wedatasphere.linkis.manager.label.entity.Label; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.*; +import java.util.stream.Collectors; + +/** + * @author: jinyangrao on 2021/7/16 + * @description: + */ +public class LabelInstanceFactory { + + private static final Logger log = LoggerFactory.getLogger(LabelInstanceFactory.class); + + private Map> labels = null; + + public LabelInstanceFactory(Map labelParam) { + labels = getLabels(labelParam); + } + + private Map> getLabels(Map originalLabels) { + Map originalLabelsMap = LabelKeyConvertor.labelKeyConvert(originalLabels); + return LabelBuilder.buildLabel(originalLabelsMap); + } + + public Map getLabelsMap() { + HashMap resultLabels = new HashMap<>(); + labels.entrySet().stream().map(item -> resultLabels.put(item.getKey(), (DSSLabel) item.getValue())); + return resultLabels; + } + + public List getLabelList() { + List dssLabelList = new ArrayList<>(); + labels.entrySet().stream().map(item -> dssLabelList.add((DSSLabel) (item.getValue()))); + return dssLabelList; + } + + public EnvDSSLabel getEnvDssLabel() { + return (EnvDSSLabel) labels.get(EnvDSSLabel.DSS_ENV_LABEL_KEY); + } +} diff --git a/dss-commons/dss-common/src/main/java/com/webank/wedatasphere/dss/common/label/LabelKeyConvertor.java b/dss-commons/dss-common/src/main/java/com/webank/wedatasphere/dss/common/label/LabelKeyConvertor.java new file mode 100644 index 0000000000..e647a45d1b --- /dev/null +++ b/dss-commons/dss-common/src/main/java/com/webank/wedatasphere/dss/common/label/LabelKeyConvertor.java @@ -0,0 +1,55 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.common.label; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.HashMap; +import java.util.Map; + +/** + * @author: jinyangrao on 2021/7/16 + * @description: + */ +public class LabelKeyConvertor { + + private static final Logger log = LoggerFactory.getLogger(LabelKeyConvertor.class); + + + public static String ROUTE_LABEL_KEY = "route"; + + private static final Map convertedlabels = new HashMap<>(); + + static { + convertedlabels.put(ROUTE_LABEL_KEY, EnvDSSLabel.DSS_ENV_LABEL_KEY); + } + + private static String getRealLabelKey(String originalLabelKey) { + return convertedlabels.get(originalLabelKey); + } + + public static Map labelKeyConvert(Map labels) { + Map finalLabels = new HashMap<>(); + for (Map.Entry item : labels.entrySet()) { + String realLabelKey = LabelKeyConvertor.getRealLabelKey(item.getKey()); + finalLabels.put(realLabelKey, item.getValue()); + } + log.info("Get labels map: {}", finalLabels); + return finalLabels; + } +} diff --git a/dss-commons/dss-common/src/main/java/com/webank/wedatasphere/dss/common/protocol/JobStatus.java b/dss-commons/dss-common/src/main/java/com/webank/wedatasphere/dss/common/protocol/JobStatus.java new file mode 100644 index 0000000000..9724233e89 --- /dev/null +++ b/dss-commons/dss-common/src/main/java/com/webank/wedatasphere/dss/common/protocol/JobStatus.java @@ -0,0 +1,47 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.common.protocol; + +/** + * created by cooperyang on 2020/11/17 + * Description: + */ +public enum JobStatus { + + /** + * JobStatus是用来进行发布任务的状态的追踪 + */ + + Inited("Inited", 1), + Running("Running", 2), + Success("Success", 3), + Failed("Failed", 4); + + private String status; + private int index; + + private JobStatus(String status, int index){ + this.status = status; + this.index = index; + } + + public static boolean isSuccess(String status){ + return Success.status.equals(status); + } + + +} diff --git a/dss-commons/dss-common/src/main/java/com/webank/wedatasphere/dss/common/protocol/project/ProjectInfoRequest.java b/dss-commons/dss-common/src/main/java/com/webank/wedatasphere/dss/common/protocol/project/ProjectInfoRequest.java new file mode 100644 index 0000000000..03a2a81708 --- /dev/null +++ b/dss-commons/dss-common/src/main/java/com/webank/wedatasphere/dss/common/protocol/project/ProjectInfoRequest.java @@ -0,0 +1,33 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.common.protocol.project; + +/** + * Created by enjoyyin on 2021/7/2. + */ +public class ProjectInfoRequest { + + private long projectId; + + public long getProjectId() { + return projectId; + } + + public void setProjectId(long projectId) { + this.projectId = projectId; + } +} diff --git a/dss-commons/dss-common/src/main/java/com/webank/wedatasphere/dss/common/protocol/project/ProjectRelationRequest.java b/dss-commons/dss-common/src/main/java/com/webank/wedatasphere/dss/common/protocol/project/ProjectRelationRequest.java new file mode 100644 index 0000000000..71bf0a0d5b --- /dev/null +++ b/dss-commons/dss-common/src/main/java/com/webank/wedatasphere/dss/common/protocol/project/ProjectRelationRequest.java @@ -0,0 +1,65 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.common.protocol.project; + + +import com.webank.wedatasphere.dss.common.label.DSSLabel; + +import java.util.List; + +/** + * created by cooperyang on 2021/1/18 + * Description: + */ +public class ProjectRelationRequest { + + private Long dssProjectId; + + private String appConnName; + + private List dssLabels; + + public ProjectRelationRequest(Long dssProjectId, String appconnName, List dssLabels) { + this.dssProjectId = dssProjectId; + this.appConnName = appconnName; + this.dssLabels = dssLabels; + } + + public Long getDssProjectId() { + return dssProjectId; + } + + public void setDssProjectId(Long dssProjectId) { + this.dssProjectId = dssProjectId; + } + + public String getAppconnName() { + return appConnName; + } + + public void setAppconnName(String appconnName) { + this.appConnName = appconnName; + } + + public List getDssLabels() { + return dssLabels; + } + + public void setDssLabels(List dssLabels) { + this.dssLabels = dssLabels; + } +} diff --git a/dss-commons/dss-common/src/main/java/com/webank/wedatasphere/dss/common/protocol/project/ProjectRelationResponse.java b/dss-commons/dss-common/src/main/java/com/webank/wedatasphere/dss/common/protocol/project/ProjectRelationResponse.java new file mode 100644 index 0000000000..b1072e54b0 --- /dev/null +++ b/dss-commons/dss-common/src/main/java/com/webank/wedatasphere/dss/common/protocol/project/ProjectRelationResponse.java @@ -0,0 +1,76 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.common.protocol.project; + + +import com.webank.wedatasphere.dss.common.label.DSSLabel; + +import java.util.List; + +/** + * created by cooperyang on 2021/1/18 + * Description: + */ +public class ProjectRelationResponse { + + private Long dssProjectId; + + private String appConnName; + + private List dssLabels; + + private Long appInstanceProjectId; + + public ProjectRelationResponse(Long dssProjectId, String appConnName, List dssLabels, Long appInstanceProjectId) { + this.dssProjectId = dssProjectId; + this.appConnName = appConnName; + this.dssLabels = dssLabels; + this.appInstanceProjectId = appInstanceProjectId; + } + + public Long getDssProjectId() { + return dssProjectId; + } + + public void setDssProjectId(Long dssProjectId) { + this.dssProjectId = dssProjectId; + } + + public String getAppConnName() { + return appConnName; + } + + public void setAppConnName(String appConnName) { + this.appConnName = appConnName; + } + + public List getDssLabels() { + return dssLabels; + } + + public void setDssLabels(List dssLabels) { + this.dssLabels = dssLabels; + } + + public Long getAppInstanceProjectId() { + return appInstanceProjectId; + } + + public void setAppInstanceProjectId(Long appInstanceProjectId) { + this.appInstanceProjectId = appInstanceProjectId; + } +} diff --git a/dss-commons/dss-common/src/main/java/com/webank/wedatasphere/dss/common/utils/ClassUtils.java b/dss-commons/dss-common/src/main/java/com/webank/wedatasphere/dss/common/utils/ClassUtils.java new file mode 100644 index 0000000000..602613d9ca --- /dev/null +++ b/dss-commons/dss-common/src/main/java/com/webank/wedatasphere/dss/common/utils/ClassUtils.java @@ -0,0 +1,123 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.common.utils; + +import com.webank.wedatasphere.dss.common.exception.DSSErrorException; +import com.webank.wedatasphere.dss.common.exception.DSSRuntimeException; +import java.util.List; +import java.util.Optional; +import java.util.function.Predicate; +import java.util.stream.Collectors; +import org.apache.commons.lang.exception.ExceptionUtils; +import org.reflections.Reflections; + +/** + * Created by enjoyyin on 2021/6/24. + */ +public class ClassUtils { + + private static final ClassHelper CLASS_HELPER = new ClassHelper() { + @Override + protected Reflections getReflections(Class clazz) { + return com.webank.wedatasphere.linkis.common.utils.ClassUtils.reflections(); + } + }; + + public static T getInstance(Class clazz) throws DSSErrorException { + return CLASS_HELPER.getInstance(clazz); + } + + public static T getInstanceOrWarn(Class clazz) { + return CLASS_HELPER.getInstanceOrWarn(clazz); + } + + public static T getInstanceOrDefault(Class clazz, T defaultValue) { + return CLASS_HELPER.getInstanceOrDefault(clazz, defaultValue); + } + + public static T getInstanceOrDefault(Class clazz, Predicate> filterOp, T defaultValue) { + return CLASS_HELPER.getInstanceOrDefault(clazz, filterOp, defaultValue); + } + + public static List getInstances(Class clazz) { + return CLASS_HELPER.getInstances(clazz); + } + + public static List> getClasses(Class clazz) { + return CLASS_HELPER.getClasses(clazz); + } + + public abstract static class ClassHelper { + + protected abstract Reflections getReflections(Class clazz); + + public T getInstance(Class clazz) throws DSSErrorException { + List> factoryClasses = getReflections(clazz).getSubTypesOf(clazz) + .stream().filter(c -> !com.webank.wedatasphere.linkis.common.utils.ClassUtils.isInterfaceOrAbstract(c)).collect(Collectors.toList()); + if(factoryClasses.isEmpty()) { + DSSExceptionUtils.dealErrorException(60053, "Cannot find instance for " + clazz.getSimpleName(), DSSErrorException.class); + } else if(factoryClasses.size() > 1) { + DSSExceptionUtils.dealErrorException(60053, "Too many instances for " + clazz.getSimpleName() + ", exists: " + factoryClasses, + DSSErrorException.class); + } + T t = null; + try { + t = factoryClasses.get(0).newInstance(); + } catch (Exception e) { + DSSExceptionUtils.dealErrorException(60053, "Instance " + clazz.getSimpleName() + " failed", e, DSSErrorException.class); + } + return t; + } + + public T getInstanceOrWarn(Class clazz) { + try { + return getInstance(clazz); + } catch (DSSErrorException e) { + DSSExceptionUtils.dealWarnException(60053, e.getDesc(), ExceptionUtils.getCause(e), DSSRuntimeException.class); + } + return null; + } + + public T getInstanceOrDefault(Class clazz, T defaultValue) { + Optional optional = getReflections(clazz).getSubTypesOf(clazz) + .stream().filter(c -> !com.webank.wedatasphere.linkis.common.utils.ClassUtils.isInterfaceOrAbstract(c) && + !c.isInstance(defaultValue)).findFirst().map(DSSExceptionUtils.map(Class::newInstance)); + return optional.orElse(defaultValue); + } + + public T getInstanceOrDefault(Class clazz, Predicate> filterOp, T defaultValue) { + Optional optional = getReflections(clazz).getSubTypesOf(clazz) + .stream().filter(c -> !com.webank.wedatasphere.linkis.common.utils.ClassUtils.isInterfaceOrAbstract(c) && + filterOp.test(c)).findFirst().map(DSSExceptionUtils.map(Class::newInstance)); + return optional.orElse(defaultValue); + } + + public List getInstances(Class clazz) { + return getReflections(clazz).getSubTypesOf(clazz) + .stream().filter(c -> !com.webank.wedatasphere.linkis.common.utils.ClassUtils.isInterfaceOrAbstract(c)) + .map(DSSExceptionUtils.map(Class::newInstance)).collect(Collectors.toList()); + } + + public List> getClasses(Class clazz) { + return getReflections(clazz).getSubTypesOf(clazz) + .stream().filter(c -> !com.webank.wedatasphere.linkis.common.utils.ClassUtils.isInterfaceOrAbstract(c)) + .collect(Collectors.toList()); + } + + } + +} diff --git a/dss-commons/dss-common/src/main/java/com/webank/wedatasphere/dss/common/utils/DSSCommonUtils.java b/dss-commons/dss-common/src/main/java/com/webank/wedatasphere/dss/common/utils/DSSCommonUtils.java new file mode 100644 index 0000000000..37404486d7 --- /dev/null +++ b/dss-commons/dss-common/src/main/java/com/webank/wedatasphere/dss/common/utils/DSSCommonUtils.java @@ -0,0 +1,60 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.common.utils; + +import com.google.gson.Gson; +import com.google.gson.GsonBuilder; +import com.webank.wedatasphere.linkis.common.conf.CommonVars; + +/** + * @Author alexyang + * @Date 2020/3/16 + */ +public class DSSCommonUtils { + + public static final String FLOW_RESOURCE_NAME = "resources"; + + public static final String FLOW_EDGES_NAME = "edges"; + + public static final String FLOW_PARENT_NAME = "parent"; + + + public static final String NODE_RESOURCE_NAME = "resources"; + + public static final String FLOW_NODE_NAME = "nodes"; + + public static final String FLOW_PROP_NAME = "props"; + + public static final String NODE_PROP_NAME = "params"; + + public static final String NODE_PROP_VARIABLE_NAME = "variable"; + + public static final String NODE_ID_NAME = "id"; + + public static final String NODE_NAME_NAME = "title"; + + public static final String FLOW_CONTEXT_ID_PREFIX = "dss.context.id."; + + public static final Gson COMMON_GSON = new GsonBuilder().disableHtmlEscaping().setDateFormat("yyyy-MM-dd'T'HH:mm:ssZ").create(); + + public static final String ENV_LABEL_VALUE_DEV = "dev"; + + public static final String DSS_LABELS_KEY = "labels"; + + public static final CommonVars DSS_HOME = CommonVars.apply("DSS_HOME", ""); + +} diff --git a/dss-commons/dss-common/src/main/java/com/webank/wedatasphere/dss/common/utils/DSSExceptionUtils.java b/dss-commons/dss-common/src/main/java/com/webank/wedatasphere/dss/common/utils/DSSExceptionUtils.java new file mode 100644 index 0000000000..8cac36054a --- /dev/null +++ b/dss-commons/dss-common/src/main/java/com/webank/wedatasphere/dss/common/utils/DSSExceptionUtils.java @@ -0,0 +1,122 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.common.utils; + +import com.webank.wedatasphere.dss.common.exception.DSSErrorException; +import com.webank.wedatasphere.dss.common.exception.DSSRuntimeException; +import com.webank.wedatasphere.dss.common.exception.ThrowingConsumer; +import com.webank.wedatasphere.dss.common.exception.ThrowingFunction; +import com.webank.wedatasphere.linkis.common.exception.ErrorException; +import com.webank.wedatasphere.linkis.common.exception.WarnException; +import java.lang.reflect.Constructor; +import java.util.function.Consumer; +import java.util.function.Function; +import org.apache.commons.lang.exception.ExceptionUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Created by v_wbjftang on 2019/9/24. + */ +public class DSSExceptionUtils { + + private static final Logger LOGGER = LoggerFactory.getLogger(DSSExceptionUtils.class); + + public static Consumer handling( + ThrowingConsumer throwingConsumer) { + return i -> { + try { + throwingConsumer.accept(i); + } catch (Exception e) { + LOGGER.error("execute failed,reason:",e); + throw new DSSRuntimeException(e.getMessage()); + } + }; + } + + public static Function map( + ThrowingFunction throwingFunction) { + return i -> { + try { + return throwingFunction.accept(i); + } catch (Exception e) { + LOGGER.error("execute failed,reason:",e); + throw new DSSRuntimeException(e.getMessage()); + } + }; + } + + public static R tryAndWarn( + ThrowingFunction throwingFunction) { + Function function = i -> { + try { + return throwingFunction.accept(i); + } catch (Exception e) { + LOGGER.error("execute failed,reason:",e); + throw new DSSRuntimeException(53320, ExceptionUtils.getRootCauseMessage(e)); + } + }; + return function.apply(null); + } + + public static void dealErrorException(int errorCode, String errorDesc, Throwable throwable, + Class clazz) throws T{ + T errorException; + try { + Constructor constructor = clazz.getConstructor(int.class, String.class); + errorException = constructor.newInstance(errorCode, errorDesc); + errorException.setErrCode(errorCode); + errorException.setDesc(errorDesc); + } catch (Exception e) { + throw new DSSRuntimeException(errorCode, errorDesc, throwable); + } + errorException.initCause(throwable); + throw errorException; + } + + + public static void dealErrorException(int errorCode, String errorDesc, + Class clazz) throws T { + T errorException; + try { + Constructor constructor = clazz.getConstructor(int.class, String.class); + errorException = constructor.newInstance(errorCode, errorDesc); + errorException.setErrCode(errorCode); + errorException.setDesc(errorDesc); + } catch (Exception e) { + throw new DSSRuntimeException(errorCode, errorDesc, e); + } + throw errorException; + } + + + public static void dealWarnException(int errorCode, String errorDesc, Throwable throwable, + Class clazz) { + T warnException; + try { + Constructor constructor = clazz.getConstructor(int.class, String.class); + warnException = constructor.newInstance(errorCode, errorDesc); + warnException.setErrCode(errorCode); + warnException.setDesc(errorDesc); + warnException.initCause(throwable); + } catch (Exception e) { + throw new DSSRuntimeException(errorCode, errorDesc, throwable); + } + throw warnException; + } + +} diff --git a/dss-commons/dss-common/src/main/java/com/webank/wedatasphere/dss/common/utils/FileHelper.java b/dss-commons/dss-common/src/main/java/com/webank/wedatasphere/dss/common/utils/FileHelper.java new file mode 100644 index 0000000000..820c99cf03 --- /dev/null +++ b/dss-commons/dss-common/src/main/java/com/webank/wedatasphere/dss/common/utils/FileHelper.java @@ -0,0 +1,32 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.common.utils; + +import java.io.File; + +/** + * created by cooperyang on 2019/6/14 + * Description: FileHelper 是 linux 文件系统的帮助类，检查目录是否存在等 + */ +public class FileHelper { + + public static boolean checkDirExists(String dir){ + File file = new File(dir); + return file.exists() && file.isDirectory(); + } + +} diff --git a/dss-commons/dss-common/src/main/java/com/webank/wedatasphere/dss/common/utils/IoUtils.java b/dss-commons/dss-common/src/main/java/com/webank/wedatasphere/dss/common/utils/IoUtils.java new file mode 100644 index 0000000000..41171a7909 --- /dev/null +++ b/dss-commons/dss-common/src/main/java/com/webank/wedatasphere/dss/common/utils/IoUtils.java @@ -0,0 +1,129 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.common.utils; + +import com.webank.wedatasphere.dss.common.conf.DSSCommonConf; +import com.webank.wedatasphere.dss.common.entity.IOEnv; +import com.webank.wedatasphere.dss.common.entity.IOType; +import org.apache.commons.io.FileUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.*; +import java.text.SimpleDateFormat; +import java.util.Arrays; +import java.util.Date; +import java.util.Properties; + +/** + * @author allenlliu + * @version 2.0.0 + * @date 2020/03/11 11:18 AM + */ +public class IoUtils { + + private static Logger logger = LoggerFactory.getLogger(IoUtils.class); + private static final String DATE_FORMAT = "yyyyMMddHHmmssSSS"; + private static final String DEFAULT_IO_FILE_NAME = "IO.properties"; + + public static String generateIOPath(String userName,String projectName,String subDir) { + String baseUrl = DSSCommonConf.DSS_EXPORT_URL.getValue(); + String dataStr = new SimpleDateFormat(DATE_FORMAT).format(new Date()); + return addFileSeparator(baseUrl, dataStr, userName, projectName, subDir); + } + + private static String addFileSeparator(String... str) { + return Arrays.stream(str).reduce((a, b) -> a + File.separator + b).orElse(""); + } + + public static OutputStream generateExportOutputStream(String path) throws IOException { + File file = new File(path); + if (!file.getParentFile().exists()) { + FileUtils.forceMkdir(file.getParentFile()); + } + if (file.exists()) { + logger.warn(String.format("%s is exist,delete it", path)); + file.delete(); + } + file.createNewFile(); + return FileUtils.openOutputStream(file, true); + } + + public static InputStream generateInputInputStream(String path)throws IOException{ + return new FileInputStream(path); + } + + public static void generateIOType(IOType ioType, String basePath) throws IOException { + generateIOProperties("type",ioType.name(),basePath); + } + + public static void generateIOProperties(String key,String value,String basePath) throws IOException { + Properties properties = new Properties(); + properties.setProperty(key,value); + + File file = new File(basePath + File.separator + DEFAULT_IO_FILE_NAME); + if (!file.getParentFile().exists()) { + FileUtils.forceMkdir(file.getParentFile()); + } + if(!file.exists()){ + file.createNewFile(); + } + try(FileOutputStream fileOutputStream = FileUtils.openOutputStream(file, true)) { + properties.store(fileOutputStream,""); + } + + } + + public static String readIOProperties(String key,String basepath) throws IOException { + try(FileInputStream inputStream = new FileInputStream(basepath + File.separator + DEFAULT_IO_FILE_NAME)){ + Properties properties = new Properties(); + properties.load(inputStream); + return properties.get(key).toString(); + } + + } + + public static IOType readIOType(String basepath)throws IOException{ + return IOType.valueOf(readIOProperties("type",basepath)); + } + + public static void generateIOEnv(String basePath) throws IOException { + generateIOProperties("env",getDSSServerEnv().name(),basePath); + } + + public static IOEnv readIOEnv(String basePath) throws IOException { + return IOEnv.valueOf(readIOProperties("env",basePath)); + } + + public static IOEnv getDSSServerEnv(){ + //dssserverEnv 是当前dss服务启动的env环境 + return IOEnv.valueOf(DSSCommonConf.DSS_IO_ENV.getValue()); + } + + public static String addVersion(String version){ + String num = String.valueOf(Integer.valueOf(version.substring(1)) + 1); + int length = num.length(); + return version.substring(0,version.length() - length) + num; + } + + public static String subVersion(String version){ + String num = String.valueOf(Integer.valueOf(version.substring(1)) - 1); + int length = num.length(); + return version.substring(0,version.length() - length) + num; + } + +} diff --git a/dss-commons/dss-common/src/main/java/com/webank/wedatasphere/dss/common/utils/ZipHelper.java b/dss-commons/dss-common/src/main/java/com/webank/wedatasphere/dss/common/utils/ZipHelper.java new file mode 100644 index 0000000000..326d2e14b4 --- /dev/null +++ b/dss-commons/dss-common/src/main/java/com/webank/wedatasphere/dss/common/utils/ZipHelper.java @@ -0,0 +1,196 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.common.utils; + + + +import com.webank.wedatasphere.dss.common.exception.DSSErrorException; +import org.apache.commons.io.IOUtils; +import org.apache.commons.lang.StringUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.BufferedReader; +import java.io.File; +import java.io.InputStreamReader; +import java.util.ArrayList; +import java.util.List; + +/** + * created by cooperyang on 2019/6/13 + * Description: + */ +public class ZipHelper { + + private static final Logger logger = LoggerFactory.getLogger(ZipHelper.class); + + private static final String ZIP_CMD = "zip"; + private static final String UN_ZIP_CMD = "unzip"; + private static final String RECURSIVE = "-r"; + private static final String ZIP_TYPE = ".zip"; + + public static String zip(String dirPath) throws DSSErrorException { + return zip(dirPath, true); + } + + /** + * ZipHelper可以将传入的path进行打包 + * @param dirPath 需要打包的project路径,绝对路径 + * @return 打包之后的zip包全路径 + */ + public static String zip(String dirPath, boolean deleteOriginDir)throws DSSErrorException { + if(!FileHelper.checkDirExists(dirPath)){ + logger.error("{} 不存在, 不能创建zip文件", dirPath); + throw new DSSErrorException(90001,dirPath + " does not exist, can not zip"); + } + //先用简单的方法，调用新进程进行压缩 + String[] strArr = dirPath.split(File.separator); + String shortPath = strArr[strArr.length - 1]; + String workPath = dirPath.substring(0, dirPath.length() - shortPath.length() - 1); + List list = new ArrayList<>(); + list.add(ZIP_CMD); + list.add(RECURSIVE); + String zipFilePath = shortPath + ZIP_TYPE; + String longZipFilePath = dirPath + ZIP_TYPE; + list.add(zipFilePath); + list.add(shortPath); + ProcessBuilder processBuilder = new ProcessBuilder(list); + processBuilder.redirectErrorStream(true); + processBuilder.directory(new File(workPath)); + BufferedReader infoReader = null; + BufferedReader errorReader = null; + try{ + Process process = processBuilder.start(); + infoReader = new BufferedReader(new InputStreamReader(process.getInputStream())); + errorReader = new BufferedReader(new InputStreamReader(process.getErrorStream())); + String infoLine = null; + while((infoLine = infoReader.readLine()) != null){ + logger.info("process output: {} ", infoLine); + } + String errorLine = null; + StringBuilder errMsg = new StringBuilder(); + while((errorLine = errorReader.readLine()) != null){ + if (StringUtils.isNotEmpty(errorLine)){ + errMsg.append(errorLine).append("\n"); + } + logger.error("process error: {} ", errorLine); + } + int exitCode = process.waitFor(); + if (exitCode != 0){ + throw new DSSErrorException(90002,errMsg.toString()); + } + }catch(final Exception e){ + logger.error("{} 压缩成 zip 文件失败, reason: ", e); + DSSErrorException exception = new DSSErrorException(90003,dirPath + " to zip file failed"); + exception.initCause(e); + throw exception; + } finally { + //删掉整个目录 + IOUtils.closeQuietly(infoReader); + IOUtils.closeQuietly(errorReader); + if(deleteOriginDir) { + File file = new File(dirPath); + logger.info("生成zip文件{}",longZipFilePath); + logger.info("开始删除目录 {}", dirPath); + if (deleteDir(file)){ + logger.info("结束删除目录 {} 成功", dirPath); + }else{ + logger.info("删除目录 {} 失败", dirPath); + } + } + } + return longZipFilePath; + } + + + + public static String unzip(String dirPath)throws DSSErrorException { + File file = new File(dirPath); + if(!file.exists()){ + logger.error("{} 不存在, 不能解压zip文件", dirPath); + throw new DSSErrorException(90001,dirPath + " does not exist, can not unzip"); + } + //先用简单的方法，调用新进程进行压缩 + String[] strArr = dirPath.split(File.separator); + String shortPath = strArr[strArr.length - 1]; + String workPath = dirPath.substring(0, dirPath.length() - shortPath.length() - 1); + List list = new ArrayList<>(); + list.add(UN_ZIP_CMD); + String longZipFilePath = dirPath.replace(ZIP_TYPE,""); + list.add(shortPath); + ProcessBuilder processBuilder = new ProcessBuilder(list); + processBuilder.redirectErrorStream(true); + processBuilder.directory(new File(workPath)); + BufferedReader infoReader = null; + BufferedReader errorReader = null; + try{ + Process process = processBuilder.start(); + infoReader = new BufferedReader(new InputStreamReader(process.getInputStream())); + errorReader = new BufferedReader(new InputStreamReader(process.getErrorStream())); + String infoLine = null; + while((infoLine = infoReader.readLine()) != null){ + logger.info("process output: {} ", infoLine); + } + String errorLine = null; + StringBuilder errMsg = new StringBuilder(); + while((errorLine = errorReader.readLine()) != null){ + if (StringUtils.isNotEmpty(errorLine)){ + errMsg.append(errorLine).append("\n"); + } + logger.error("process error: {} ", errorLine); + } + int exitCode = process.waitFor(); + if (exitCode != 0){ + throw new DSSErrorException(90007,errMsg.toString()); + } + }catch(final Exception e){ + logger.error("{} 解压缩 zip 文件失败, reason: ", e); + DSSErrorException exception = new DSSErrorException(90009,dirPath + " to zip file failed"); + exception.initCause(e); + throw exception; + } finally { + + logger.info("生成解压目录{}", longZipFilePath); + IOUtils.closeQuietly(infoReader); + IOUtils.closeQuietly(errorReader); + } + return longZipFilePath; + } + + private static boolean deleteDir(File dir) { + if (dir.isDirectory()) { + String[] children = dir.list(); + if (children != null && children.length > 0){ + for (String s : children) { + boolean success = deleteDir(new File(dir, s)); + if (!success) { + return false; + } + } + } + } + // 目录此时为空，可以删除 + return dir.delete(); + } + + public static String zipExportProject(String projectPath) throws DSSErrorException { + if(projectPath.endsWith(File.separator)) { + projectPath = projectPath.substring(0, projectPath.lastIndexOf(File.separator)); + } + return ZipHelper.zip(projectPath); + } +} diff --git a/dss-commons/dss-common/src/main/scala/com/webank/wedatasphere/dss/common/label/LabelBuilder.scala b/dss-commons/dss-common/src/main/scala/com/webank/wedatasphere/dss/common/label/LabelBuilder.scala new file mode 100644 index 0000000000..e2a1b6762c --- /dev/null +++ b/dss-commons/dss-common/src/main/scala/com/webank/wedatasphere/dss/common/label/LabelBuilder.scala @@ -0,0 +1,49 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.common.label + +import com.webank.wedatasphere.linkis.manager.label.entity.Label + +import scala.collection.JavaConverters._ +import java.util + +import com.webank.wedatasphere.linkis.manager.label.builder.factory.{LabelBuilderFactory, LabelBuilderFactoryContext} + + +/** + * @author: jinyangrao on 2021/7/16 + * @description: + */ +object LabelBuilder { + + def buildLabel(labelMap: util.Map[String, Object]): util.Map[String, Label[_]] = { + val labelKeyValueMap = new util.HashMap[String, Label[_]]() + if (null != labelMap && !labelMap.isEmpty) { + val list: util.List[Label[_]] = labelBuilderFactoryProxyMethod.getLabels(labelMap.asInstanceOf[util.Map[String, AnyRef]]) + if (null != list) { + list.asScala.filter(_ != null).foreach { + label => labelKeyValueMap.put(label.getLabelKey, label) + } + } + } + labelKeyValueMap + } + + private def labelBuilderFactoryProxyMethod: LabelBuilderFactory = { + LabelBuilderFactoryContext.getLabelBuilderFactory + } +} diff --git a/dss-commons/dss-common/src/main/scala/com/webank/wedatasphere/dss/common/protocol/DSSProtocol.scala b/dss-commons/dss-common/src/main/scala/com/webank/wedatasphere/dss/common/protocol/DSSProtocol.scala new file mode 100644 index 0000000000..595edc5973 --- /dev/null +++ b/dss-commons/dss-common/src/main/scala/com/webank/wedatasphere/dss/common/protocol/DSSProtocol.scala @@ -0,0 +1,25 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.common.protocol + +/** + * created by cooperyang on 2020/11/17 + * Description: + */ +trait DSSProtocol { + +} diff --git a/dss-commons/dss-common/src/main/scala/com/webank/wedatasphere/dss/common/protocol/DSSProtocolObject.scala b/dss-commons/dss-common/src/main/scala/com/webank/wedatasphere/dss/common/protocol/DSSProtocolObject.scala new file mode 100644 index 0000000000..c197ac1c11 --- /dev/null +++ b/dss-commons/dss-common/src/main/scala/com/webank/wedatasphere/dss/common/protocol/DSSProtocolObject.scala @@ -0,0 +1,66 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.common.protocol + +import com.webank.wedatasphere.dss.common.label.DSSLabel + + +/** + * Created by v_wbjftang on 2019/11/8. + */ + + + +case class ResponseCreateOrchestrator(orchestratorId: Long, + orchestratorVersionId: Long) + +case class ResponseExportOrchestrator(resourceId: String, + version: String, + orcVersionId: Long) + +case class ResponseImportOrchestrator(orcId: Long) + +case class RequestUpdateWorkflow(userName: String, + flowID: Long, + flowName: String, + description: String, + uses: String) + +case class RequestDeleteWorkflow(userName: String, flowID: Long) + + +case class RequestExportWorkflow(userName: String, + flowID: Long, + projectId: Long, + projectName: String, + workspaceStr: String, + dssLabelList: java.util.List[DSSLabel] + ) + +case class ResponseExportWorkflow(resourceId: String, version: String, flowID: Long) + +case class RequestQueryWorkFlow(userName: String, rootFlowId: Long) + + + + + + + + + + diff --git a/dss-commons/dss-common/src/main/scala/com/webank/wedatasphere/dss/common/protocol/framework/ReleaseOrchestratorRequest.scala b/dss-commons/dss-common/src/main/scala/com/webank/wedatasphere/dss/common/protocol/framework/ReleaseOrchestratorRequest.scala new file mode 100644 index 0000000000..25963299fe --- /dev/null +++ b/dss-commons/dss-common/src/main/scala/com/webank/wedatasphere/dss/common/protocol/framework/ReleaseOrchestratorRequest.scala @@ -0,0 +1,48 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.common.protocol.framework + +import com.webank.wedatasphere.dss.common.protocol.DSSProtocol + + +/** + * created by cooperyang on 2020/11/17 + * Description: + * DSS框架RPC所需要用到RPC + */ + + + +trait FrameworkProtocol extends DSSProtocol + + +case class ReleaseOrchestratorRequest(releaseUser:String, orchestratorVersionId:Long, orchestratorId:Long, + dssLabel:String, workspaceName:String, workSpaceStr: String) extends FrameworkProtocol + +case class ReleaseOrchestratorResponse(jobId:Long) extends FrameworkProtocol + + + +case class ReleaseOrchestratorStatusRequest(jobId:Long) extends FrameworkProtocol + +case class ReleaseOrchestratorStatusResponse(status:String, jobId:Long, errorMsg:String) extends FrameworkProtocol + + + + + + diff --git a/dss-commons/dss-common/src/main/scala/com/webank/wedatasphere/dss/common/utils/DSSMainHelper.scala b/dss-commons/dss-common/src/main/scala/com/webank/wedatasphere/dss/common/utils/DSSMainHelper.scala new file mode 100644 index 0000000000..e6380cdf92 --- /dev/null +++ b/dss-commons/dss-common/src/main/scala/com/webank/wedatasphere/dss/common/utils/DSSMainHelper.scala @@ -0,0 +1,52 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.common.utils + +import com.webank.wedatasphere.linkis.common.conf.{BDPConfiguration, CommonVars, DSSConfiguration} + +import scala.collection.JavaConversions._ + +/** + * @author allenlliu + * @date 2021/3/9 17:18 + */ +object DSSMainHelper { + +// val DSS_DEFAULT_PROPERTY_FILE_NAME = CommonVars("wds.dss.default.property.file", "dss.properties") +// val DSS_SERVER_DEFAULT_PROPERTY_FILE_NAME = CommonVars("wds.dss.server.default.property.file", "%s.properties") + + def formatPropertyFiles(serviceName: String): Unit = { + sys.props.put("wds.linkis.configuration", "dss.properties") + sys.props.put("wds.linkis.server.conf", serviceName+".properties") + } + + //TODO wait for linkis re-written + @Deprecated + def addExtraPropertyFiles(filePaths: String *): Unit = { + sys.props.put("wds.linkis.server.confs", filePaths.mkString(",")) + } + + def getExtraSpringOptions: Array[String] = { + "--spring.profiles.active=dss" +: DSSConfiguration.getAllProperties.filter { case (k, v) => k != null && k.startsWith("spring.")} + .map{ case (k, v) => { + val realKey=k.substring(7) + s"--$realKey=$v" + } + }.toArray + } + +} diff --git a/dss-commons/dss-common/src/main/scala/com/webank/wedatasphere/linkis/common/conf/DSSConfiguration.scala b/dss-commons/dss-common/src/main/scala/com/webank/wedatasphere/linkis/common/conf/DSSConfiguration.scala new file mode 100644 index 0000000000..d3a2acd7e4 --- /dev/null +++ b/dss-commons/dss-common/src/main/scala/com/webank/wedatasphere/linkis/common/conf/DSSConfiguration.scala @@ -0,0 +1,29 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.linkis.common.conf + +import java.util.Properties + +/** + * @author allenlliu + * @date 2021/3/10 17:32 + */ +object DSSConfiguration { + + def getAllProperties: Properties = BDPConfiguration.properties + +} diff --git a/dss-commons/dss-contextservice/pom.xml b/dss-commons/dss-contextservice/pom.xml new file mode 100644 index 0000000000..950a9ea5a4 --- /dev/null +++ b/dss-commons/dss-contextservice/pom.xml @@ -0,0 +1,84 @@ + + + + 4.0.0 + + dss-commons + com.webank.wedatasphere.dss + 1.0.0 + + dss-contextservice + + + + com.webank.wedatasphere.linkis + linkis-common + ${linkis.version} + provided + + + + com.webank.wedatasphere.linkis + linkis-cs-client + ${linkis.version} + + + com.webank.wedatasphere.dss + dss-common + ${dss.version} + provided + + + com.webank.wedatasphere.dss + dss-scheduler-appconn + ${dss.version} + + + + + + + + + org.apache.maven.plugins + maven-compiler-plugin + 3.3 + + 1.8 + 1.8 + UTF-8 + + + + org.apache.maven.plugins + maven-deploy-plugin + + + + net.alchim31.maven + scala-maven-plugin + + + org.apache.maven.plugins + maven-jar-plugin + + + + + \ No newline at end of file diff --git a/dss-commons/dss-contextservice/src/main/java/com/webank/wedatasphere/dss/contextservice/service/ContextService.java b/dss-commons/dss-contextservice/src/main/java/com/webank/wedatasphere/dss/contextservice/service/ContextService.java new file mode 100644 index 0000000000..c646cea104 --- /dev/null +++ b/dss-commons/dss-contextservice/src/main/java/com/webank/wedatasphere/dss/contextservice/service/ContextService.java @@ -0,0 +1,80 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.contextservice.service; + +import com.webank.wedatasphere.dss.common.exception.DSSErrorException; + +/** + * Linkis ContextService访问接口工具类 + * 提供创建ContextID、存储状态信息等到CS服务的接口 + * @Author alexyang + * @Date 2020-0318 + */ +public interface ContextService { + + /** + * 创建ContextID，返回ContextID序列化后的字符串 + * @param flow + * @param user + * @param version + * @return + */ + String createContextID(String workspace, String projectName, String flow, String version, String user); + + /** + * 检查和创建ContextID，返回创建ID后的jsonFlow + * 检查ContextID信息：如果jsonFlow不包含ContextID信息，则创建新的； + * 如果已经有ContextID，判断传入flowVersion是否相同，不同则创建新的ContextID，并返回新创建的ContextID序列化内容；相同则不创建新ID; + * 新创建的ContextID会更新到flowJson里 + * @param jsonFlow + * @param flowVersion + * @param workspace + * @param project + * @param flow + * @param user + * @param fullCheck - true 检查所有参数； false 只检查jsonFlow中ContextID是否存在，存在则不再比较ContextID中version参数等是否相同，且不创建 + * @return + */ + String checkAndCreateContextID(String jsonFlow, String flowVersion, String workspace, String project, String flow, String user, boolean fullCheck); + + /** + * 解析jsonFlow、DWSProject内容，存储到CS里 + * 解析DWSProject、jsonFlow里面的资源、变量、节点依赖等信息，存储到CS服务 + * @param jsonFlow + * @param parentFlowID + */ + void checkAndSaveContext(String jsonFlow, String parentFlowID) throws DSSErrorException; + + /** + * 检查更新ContextID信息，解析传入的jsonFlow、DWSProject内容，存储到CS里 + * @param jsonFlow 必需 + * @param parentFlowId 必需 + * @param workspace + * @param flowVersion + * @param user + * @throws DSSErrorException + */ + String checkAndInitContext(String jsonFlow, String parentFlowId, String workspace, String projectName, String flowName, String flowVersion, String user) throws DSSErrorException; + + /** + * 解析SchedulerFlow信息，并更新ContextID和存储Context信息，调用checkAndInitContext信息 + * @param schedulerFlow + * @return + * @throws DSSErrorException + */ +// String checkAndInitContext(SchedulerFlow schedulerFlow) throws DSSErrorException; +} diff --git a/dss-commons/dss-contextservice/src/main/java/com/webank/wedatasphere/dss/contextservice/service/impl/ContextServiceImpl.java b/dss-commons/dss-contextservice/src/main/java/com/webank/wedatasphere/dss/contextservice/service/impl/ContextServiceImpl.java new file mode 100644 index 0000000000..9fe666baee --- /dev/null +++ b/dss-commons/dss-contextservice/src/main/java/com/webank/wedatasphere/dss/contextservice/service/impl/ContextServiceImpl.java @@ -0,0 +1,416 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.contextservice.service.impl; + +import com.google.gson.*; +import com.webank.wedatasphere.dss.common.conf.DSSCommonConf; +import com.webank.wedatasphere.dss.common.entity.Resource; +import com.webank.wedatasphere.dss.common.exception.DSSErrorException; +import com.webank.wedatasphere.dss.common.exception.ErrorCode; +import com.webank.wedatasphere.dss.common.utils.DSSCommonUtils; +import com.webank.wedatasphere.dss.contextservice.service.ContextService; +import com.webank.wedatasphere.linkis.common.exception.ErrorException; +import com.webank.wedatasphere.linkis.cs.client.ContextClient; +import com.webank.wedatasphere.linkis.cs.client.builder.ContextClientFactory; +import com.webank.wedatasphere.linkis.cs.client.service.CSWorkService; +import com.webank.wedatasphere.linkis.cs.client.service.CSWorkServiceImpl; +import com.webank.wedatasphere.linkis.cs.client.utils.SerializeHelper; +import com.webank.wedatasphere.linkis.cs.common.entity.enumeration.ContextScope; +import com.webank.wedatasphere.linkis.cs.common.entity.enumeration.ContextType; +import com.webank.wedatasphere.linkis.cs.common.entity.enumeration.WorkType; +import com.webank.wedatasphere.linkis.cs.common.entity.object.CSFlowInfos; +import com.webank.wedatasphere.linkis.cs.common.entity.object.LinkisVariable; +import com.webank.wedatasphere.linkis.cs.common.entity.resource.LinkisBMLResource; +import com.webank.wedatasphere.linkis.cs.common.entity.source.*; +import com.webank.wedatasphere.linkis.cs.common.utils.CSCommonUtils; +import org.apache.commons.lang.StringUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.*; + +/** + * @Author alexyang + * @Date 2020-0318 + */ +public class ContextServiceImpl implements ContextService { + + private static final Logger logger = LoggerFactory.getLogger(ContextServiceImpl.class); + private static ContextClient contextClient = ContextClientFactory.getOrCreateContextClient(); + private static ContextService contextService = null; + + private ContextServiceImpl() {} + + public static ContextService getInstance() { + if (null == contextService) { + synchronized (ContextServiceImpl.class) { + if (null == contextService) { + contextService = new ContextServiceImpl(); + } + } + } + return contextService; + } + + @Override + public String createContextID(String workspace, String project, String flow, String version, String user) { + LinkisHAWorkFlowContextID contextID = new LinkisHAWorkFlowContextID(); + contextID.setWorkSpace(workspace); + contextID.setProject(project); + contextID.setFlow(flow); + contextID.setUser(user); + contextID.setVersion(version); + contextID.setEnv(DSSCommonConf.DSS_IO_ENV.getValue()); + try { + contextClient.createContext(contextID); + return SerializeHelper.serializeContextID(contextID); + } catch (Exception e) { + logger.error("createContextID error. workspace : {}, project : {}, flow : {}, version : {}, user : {}", workspace, project, flow, version, user, e); + } + return null; + } + + @Override + public String checkAndCreateContextID(String jsonFlow, String flowVersion, String workspace, String project, String flow, String user, boolean fullCheck) { + JsonObject flowObject = null; + try { + flowObject = new Gson().fromJson(jsonFlow, JsonObject.class); + if (!flowObject.has(CSCommonUtils.CONTEXT_ID_STR) || + StringUtils.isBlank(flowObject.get(CSCommonUtils.CONTEXT_ID_STR).getAsString())|| + !flowObject.get(CSCommonUtils.CONTEXT_ID_STR).isJsonPrimitive()) { + String contextID = createContextID(workspace, + project, + flow, flowVersion, user); + flowObject.addProperty(CSCommonUtils.CONTEXT_ID_STR, contextID); + } else { + boolean updateContextId = false; + LinkisHAWorkFlowContextID contextID = null; + try { + Object contextIDObj = SerializeHelper.deserializeContextID(flowObject.get(CSCommonUtils.CONTEXT_ID_STR).getAsString()); + contextID = (LinkisHAWorkFlowContextID)contextIDObj; + if (fullCheck) { + if (!DSSCommonConf.DSS_IO_ENV.getValue().equalsIgnoreCase(contextID.getEnv())) { + updateContextId = true; + } else if (StringUtils.isBlank(contextID.getProject()) + || StringUtils.isBlank(contextID.getFlow()) + || StringUtils.isBlank(contextID.getVersion())) { + updateContextId = false; + } else if ((null != contextID.getWorkSpace() && !contextID.getWorkSpace().equalsIgnoreCase(workspace)) + || !contextID.getProject().equalsIgnoreCase(project) + || !contextID.getFlow().equalsIgnoreCase(flow) + || !contextID.getVersion().equalsIgnoreCase(flowVersion)) { + updateContextId = true; + } else { + updateContextId = false; + } + } + } catch (ErrorException e0) { + logger.error("Invalid contextID : {}, please contact with administrator", flowObject.get(CSCommonUtils.CONTEXT_ID_STR)); + } + if (updateContextId) { + String newContextID = createContextID(workspace, + project, + flow, flowVersion, user); + flowObject.addProperty(CSCommonUtils.CONTEXT_ID_STR, newContextID); + logger.info("UpdateContextId true, old contextID : {}, new contextID : {}", CSCommonUtils.gson.toJson(contextID), CSCommonUtils.gson.toJson(newContextID)); + } + } + } catch (Exception e) { + logger.error("Invalid json : {}", jsonFlow, e); + } + if (null != flowObject) { + return flowObject.toString(); + } else { + return jsonFlow; + } + } + + @Override + public void checkAndSaveContext(String jsonFlow,String parentFlowID) throws DSSErrorException{ + logger.info("jsonFlow => \n" + jsonFlow); + try { + JsonObject flowObject = new Gson().fromJson(jsonFlow, JsonObject.class); + if (!flowObject.has(CSCommonUtils.CONTEXT_ID_STR) || !flowObject.get(CSCommonUtils.CONTEXT_ID_STR).isJsonPrimitive()) { + logger.error("Did not have invalid contextID, save context failed."); + return; + } else { + String contextIDStr = flowObject.get(CSCommonUtils.CONTEXT_ID_STR).getAsString(); + // ①reset原有key 这里只清理 + CSWorkService csWorkService = CSWorkServiceImpl.getInstance(); + List workTypes = new ArrayList<>(); + workTypes.add(WorkType.WORKSPACE); + workTypes.add(WorkType.PROJECT); + workTypes.add(WorkType.FLOW); + csWorkService.initContextServiceInfo(contextIDStr, workTypes); + + // ②解析和保存新的 UDF、Resource、Variable + // 保存Workspace和Project的资源参数等 +// if (null != project.getProjectResources() && project.getProjectResources().size() > 0) { +// saveContextResource(contextIDStr, project.getProjectResources(), contextClient, CSCommonUtils.PROJECT_RESOURCE_PREFIX); +// } + + // 保存flow的资源 + if (flowObject.has(DSSCommonUtils.FLOW_RESOURCE_NAME)) { + JsonArray flowRes = flowObject.get(DSSCommonUtils.FLOW_RESOURCE_NAME).getAsJsonArray(); + saveContextResource(contextIDStr, flowRes, contextClient, CSCommonUtils.FLOW_RESOURCE_PREFIX, null); + } + if (flowObject.has(DSSCommonUtils.FLOW_PROP_NAME)) { + JsonElement flowProp = flowObject.get(DSSCommonUtils.FLOW_PROP_NAME); + saveContextVariable(contextIDStr, flowProp, contextClient, CSCommonUtils.FLOW_VARIABLE_PREFIX, null); + } + // todo udf + + // 保存节点的资源 + if (flowObject.has(DSSCommonUtils.FLOW_NODE_NAME)) { + JsonArray nodes = flowObject.get(DSSCommonUtils.FLOW_NODE_NAME).getAsJsonArray(); + for (JsonElement node : nodes) { + JsonObject json = node.getAsJsonObject(); + if (json.has(DSSCommonUtils.NODE_RESOURCE_NAME)) { + JsonArray nodeRes = json.get(DSSCommonUtils.NODE_RESOURCE_NAME).getAsJsonArray(); + saveContextResource(contextIDStr, nodeRes, contextClient, + CSCommonUtils.NODE_PREFIX, json.get(DSSCommonUtils.NODE_NAME_NAME).getAsString()); + } + if (json.has(DSSCommonUtils.NODE_PROP_NAME)) { + JsonObject nodePropObj = json.get(DSSCommonUtils.NODE_PROP_NAME).getAsJsonObject(); + if (nodePropObj.has(DSSCommonUtils.NODE_PROP_VARIABLE_NAME)) { + JsonElement nodeVariables = nodePropObj.get(DSSCommonUtils.NODE_PROP_VARIABLE_NAME); + saveContextVariable(contextIDStr, nodeVariables, contextClient, + CSCommonUtils.NODE_PREFIX, json.get(DSSCommonUtils.NODE_NAME_NAME).getAsString()); + } + } + } + } + // 保存info信息 + saveFlowInfo(contextIDStr, parentFlowID, jsonFlow); + } + } catch (Exception e) { + logger.error("CheckAndSaveContext error. jsonFlow : {}, parentFlowId : {}, e : ", jsonFlow, parentFlowID, e); + throw new DSSErrorException(ErrorCode.INVALID_PARAMS, "CheckAndSaveContext error : " + e.getMessage()); + } + } + + @Override + public String checkAndInitContext(String jsonFlow, String parentFlowId, String workspace, String projectName, String flowName, String flowVersion, String user) throws DSSErrorException { + if (StringUtils.isBlank(jsonFlow) ) { + logger.error("Invalid jsonFlow : {} or parentFlowId : {}.", jsonFlow, parentFlowId); + throw new DSSErrorException(ErrorCode.INVALID_PARAMS, "Invalid jsonFlow : " + jsonFlow + ", or parentFlowId : " + parentFlowId + "."); + } + jsonFlow = checkAndCreateContextID(jsonFlow, flowVersion, workspace, projectName, flowName, user, true); + checkAndSaveContext(jsonFlow,parentFlowId); + JsonObject flowObject = new Gson().fromJson(jsonFlow, JsonObject.class); + return flowObject.get(CSCommonUtils.CONTEXT_ID_STR).getAsString(); + } + + + + private void saveContextVariable(String contextIDStr, JsonElement variables, ContextClient contextClient, String variablePrefix, String nodeName) { + try { + if (variables.isJsonArray()) { + JsonArray flowProp = variables.getAsJsonArray(); + for (JsonElement prop : flowProp) { + Set> entrySet = prop.getAsJsonObject().entrySet(); + // assign entry num is 1 + if (null != entrySet) { + for (Map.Entry entry : entrySet) { + if (!entry.getValue().isJsonPrimitive()) { + continue; + } + saveContextVariableKeyValue(contextIDStr, contextClient, variablePrefix, entry, nodeName); + } + } + + } + } else if (variables.isJsonObject()) { + JsonObject variableJson = variables.getAsJsonObject(); + for (Map.Entry entry : variableJson.entrySet()) { + if (!entry.getValue().isJsonPrimitive()) { + continue; + } + saveContextVariableKeyValue(contextIDStr, contextClient, variablePrefix, entry, nodeName); + } + } else { + logger.error("Invalid JsonElement variables : {}", variables.toString()); + } + + } catch (ErrorException e) { + logger.error("SaveContextVariable failed. contextIDStr : {}, variables : {}, variablePrefix : {}, e : ", + contextIDStr, variables.toString(), variablePrefix, e); + } + + } + + private void saveContextVariableKeyValue(String contextIDStr, ContextClient contextClient, String uniKeyPrefix, Map.Entry entry, String nodeName) throws ErrorException { + String contextKeyPrefix = null; + switch (uniKeyPrefix) { + case CSCommonUtils.WORKSPACE_VARIABLE_PREFIX: + case CSCommonUtils.PROJECT_VARIABLE_PREFIX: + case CSCommonUtils.FLOW_VARIABLE_PREFIX: + contextKeyPrefix = uniKeyPrefix; + break; + case CSCommonUtils.NODE_PREFIX: + contextKeyPrefix = uniKeyPrefix + nodeName + "." + CSCommonUtils.RESOURCE_PREFIX; + break; + default: + logger.error("Invalid contextKeyPrefix : {}", uniKeyPrefix); + return; + } + LinkisVariable linkisVariable = new LinkisVariable(); + linkisVariable.setKey(entry.getKey()); + linkisVariable.setValue(entry.getValue().getAsString()); + ContextKey contextKey = new CommonContextKey(); + contextKey.setKey(contextKeyPrefix + linkisVariable.getKey()); + contextKey.setContextType(ContextType.OBJECT); + contextKey.setContextScope(ContextScope.PUBLIC); + ContextValue contextValue = new CommonContextValue(); + contextValue.setValue(linkisVariable); + ContextKeyValue contextKeyValue = new CommonContextKeyValue(contextKey, contextValue); + contextClient.setContextKeyValue(SerializeHelper.deserializeContextID(contextIDStr), contextKeyValue); + } + + private void saveContextResource(String contextIDStr, JsonArray flowRes, ContextClient contextClient, String uniKeyPrefix, String nodeName) throws ErrorException { + String contextKeyPrefix = null; + switch (uniKeyPrefix) { + case CSCommonUtils.WORKSPACE_RESOURCE_PREFIX: + case CSCommonUtils.PROJECT_RESOURCE_PREFIX: + case CSCommonUtils.FLOW_RESOURCE_PREFIX: + contextKeyPrefix = uniKeyPrefix; + break; + case CSCommonUtils.NODE_PREFIX: + contextKeyPrefix = uniKeyPrefix + nodeName + "." + CSCommonUtils.RESOURCE_PREFIX; + break; + default: + logger.error("Invalid contextKeyPrefix : {}", uniKeyPrefix); + return; + } + for (JsonElement res : flowRes) { + LinkisBMLResource bmlResource = new LinkisBMLResource(); + JsonObject json = res.getAsJsonObject(); + if (!json.has("fileName") || !json.has("resourceId") || !json.has("version")) { + logger.warn("Invalid resource: res : {}, contextidStr : {}, all res : {} ", CSCommonUtils.gson.toJson(json), contextIDStr, CSCommonUtils.gson.toJson(flowRes)); + continue; + } + if (json.get("fileName") instanceof JsonNull || json.get("resourceId") instanceof JsonNull || json.get("version") instanceof JsonNull) { + logger.warn("Invalid resource: res : {}, contextidStr : {}, all res : {} ", CSCommonUtils.gson.toJson(json), contextIDStr, CSCommonUtils.gson.toJson(flowRes)); + continue; + } + bmlResource.setDownloadedFileName(json.get("fileName").getAsString()); + bmlResource.setResourceId(json.get("resourceId").getAsString()); + bmlResource.setVersion(json.get("version").getAsString()); + ContextKey contextKey = new CommonContextKey(); + contextKey.setKey(contextKeyPrefix + bmlResource.getDownloadedFileName()); + contextKey.setContextScope(ContextScope.PUBLIC); + contextKey.setContextType(ContextType.RESOURCE); + ContextValue contextValue = new CommonContextValue(); + contextValue.setValue(bmlResource); + ContextKeyValue contextKeyValue = new CommonContextKeyValue(contextKey, contextValue); + contextClient.setContextKeyValue(SerializeHelper.deserializeContextID(contextIDStr), contextKeyValue); + //// todo test + logger.info("Debug: saved contextKeyValue : {}", CSCommonUtils.gson.toJson(contextKeyValue)); + } + } + + private void saveContextResource(String contextIDStr, List resourceList, ContextClient contextClient, String uniKeyPrefix) throws ErrorException { + + String contextKeyPrefix = null; + switch (uniKeyPrefix) { + case CSCommonUtils.WORKSPACE_RESOURCE_PREFIX: + case CSCommonUtils.PROJECT_RESOURCE_PREFIX: + case CSCommonUtils.FLOW_RESOURCE_PREFIX: + contextKeyPrefix = uniKeyPrefix; + break; + default: + logger.error("Invalid contextKeyPrefix : {}", uniKeyPrefix); + return; + } + for (Resource res : resourceList) { + LinkisBMLResource bmlResource = new LinkisBMLResource(); + bmlResource.setDownloadedFileName(res.getFileName()); + bmlResource.setResourceId(res.getResourceId()); + bmlResource.setVersion(res.getVersion()); + ContextKey contextKey = new CommonContextKey(); + contextKey.setKey(contextKeyPrefix + bmlResource.getDownloadedFileName()); + contextKey.setContextScope(ContextScope.PUBLIC); + contextKey.setContextType(ContextType.RESOURCE); + ContextValue contextValue = new CommonContextValue(); + contextValue.setValue(bmlResource); + ContextKeyValue contextKeyValue = new CommonContextKeyValue(contextKey, contextValue); + contextClient.setContextKeyValue(SerializeHelper.deserializeContextID(contextIDStr), contextKeyValue); + //// todo test + logger.info("Debug: saved contextKeyValue : {}", CSCommonUtils.gson.toJson(contextKeyValue)); + } + } + + + /** + * 保存节点间关系信息 + * @param contextIDStr + * @param parentFlowID + * @param parentFlowID + */ + private void saveFlowInfo(String contextIDStr, String parentFlowID, String flowJson) { + if (StringUtils.isBlank(contextIDStr)) { + return; + } + ContextClient contextClient = ContextClientFactory.getOrCreateContextClient(); + JsonObject flowJsonObject = new Gson().fromJson(flowJson, JsonObject.class); + CSFlowInfos flowInfos = new CSFlowInfos(); + Map flowInfoMap = new HashMap<>(); + // 保存边信息 + JsonArray flowEdges = new JsonArray(); + if (flowJsonObject.has(DSSCommonUtils.FLOW_EDGES_NAME)) { + flowEdges = flowJsonObject.getAsJsonArray(DSSCommonUtils.FLOW_EDGES_NAME); + } + flowInfoMap.put(DSSCommonUtils.FLOW_EDGES_NAME, flowEdges); + // 保存父节点 可以为空 + flowInfoMap.put(DSSCommonUtils.FLOW_PARENT_NAME, parentFlowID); + //保存节点名 + JsonObject idNodeNameJson = new JsonObject(); + if (flowJsonObject.has(DSSCommonUtils.FLOW_NODE_NAME)) { + JsonArray nodes = flowJsonObject.getAsJsonArray(DSSCommonUtils.FLOW_NODE_NAME); + for (JsonElement element : nodes) { + JsonObject jsonObject = element.getAsJsonObject(); + String id = jsonObject.get(DSSCommonUtils.NODE_ID_NAME).getAsString(); + String name = jsonObject.get(DSSCommonUtils.NODE_NAME_NAME).getAsString(); + idNodeNameJson.addProperty(id, name); + } + } + flowInfoMap.put(CSCommonUtils.ID_NODE_NAME, idNodeNameJson); + flowInfos.setInfos(flowInfoMap); + ContextKey contextKey = new CommonContextKey(); + contextKey.setContextType(ContextType.OBJECT); + contextKey.setContextScope(ContextScope.PUBLIC); + contextKey.setKey(CSCommonUtils.FLOW_INFOS); + ContextValue contextValue = new CommonContextValue(); + contextValue.setValue(flowInfos); + ContextKeyValue contextKeyValue = new CommonContextKeyValue(contextKey, contextValue); + try { + contextClient.setContextKeyValue(SerializeHelper.deserializeContextID(contextIDStr), contextKeyValue); + //// todo test + logger.info("ContextID : {}, \nContextKey : {}, \nContextValue : {}", + CSCommonUtils.gson.toJson(SerializeHelper.deserializeContextID(contextIDStr)), + CSCommonUtils.gson.toJson(contextKey), + CSCommonUtils.gson.toJson(contextValue)); + ContextValue contextValue1 = contextClient.getContextValue(SerializeHelper.deserializeContextID(contextIDStr), contextKey); + logger.info(CSCommonUtils.gson.toJson(contextValue1)); + } catch (ErrorException e) { + logger.error("Set ContextKeyValue error. contextIDStr "); + } + } + + + +} diff --git a/dss-commons/dss-sender-service/pom.xml b/dss-commons/dss-sender-service/pom.xml new file mode 100644 index 0000000000..41cd35829e --- /dev/null +++ b/dss-commons/dss-sender-service/pom.xml @@ -0,0 +1,57 @@ + + + + + + dss-commons + com.webank.wedatasphere.dss + 1.0.0 + + 4.0.0 + + dss-sender-service + + + + + com.webank.wedatasphere.dss + dss-common + ${dss.version} + + + + com.webank.wedatasphere.linkis + linkis-rpc + ${linkis.version} + + + com.webank.wedatasphere.dss + dss-development-process-standard + ${dss.version} + + + com.webank.wedatasphere.dss + dss-orchestrator-core + ${dss.version} + provided + + + + + \ No newline at end of file diff --git a/dss-commons/dss-sender-service/src/main/java/com/webank/wedatasphere/dss/sender/service/DSSSenderService.java b/dss-commons/dss-sender-service/src/main/java/com/webank/wedatasphere/dss/sender/service/DSSSenderService.java new file mode 100644 index 0000000000..c160f3c274 --- /dev/null +++ b/dss-commons/dss-sender-service/src/main/java/com/webank/wedatasphere/dss/sender/service/DSSSenderService.java @@ -0,0 +1,44 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.sender.service; + +import com.webank.wedatasphere.dss.common.label.DSSLabel; +import com.webank.wedatasphere.linkis.rpc.Sender; +import java.util.List; + +/** + * @author allenlliu + * @date 2021/6/24 17:09 + */ +public interface DSSSenderService { + + Sender getOrcSender(); + + Sender getOrcSender(List dssLabels); + + Sender getScheduleOrcSender(); + + Sender getWorkflowSender(List dssLabels); + + Sender getWorkflowSender(); + + Sender getSchedulerWorkflowSender(); + + Sender getProjectServerSender(); + +} + diff --git a/dss-commons/dss-sender-service/src/main/java/com/webank/wedatasphere/dss/sender/service/DSSSenderServiceFactory.java b/dss-commons/dss-sender-service/src/main/java/com/webank/wedatasphere/dss/sender/service/DSSSenderServiceFactory.java new file mode 100644 index 0000000000..5d36f92ed6 --- /dev/null +++ b/dss-commons/dss-sender-service/src/main/java/com/webank/wedatasphere/dss/sender/service/DSSSenderServiceFactory.java @@ -0,0 +1,44 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.sender.service; + +import com.webank.wedatasphere.dss.common.utils.ClassUtils; +import com.webank.wedatasphere.dss.sender.service.impl.DSSSenderServiceImpl; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * @author allenlliu + * @date 2021/6/28 11:41 + */ +public class DSSSenderServiceFactory { + + private static final Logger LOGGER = LoggerFactory.getLogger(DSSSenderServiceFactory.class); + private static final DSSSenderService service; + + static { + service = ClassUtils.getInstanceOrDefault(DSSSenderService.class, new DSSSenderServiceImpl()); + LOGGER.info("Use {} to instance a available DSSSenderService.", service.getClass().getName()); + } + + + public static DSSSenderService getOrCreateServiceInstance() { + return service; + } + + +} diff --git a/dss-commons/dss-sender-service/src/main/java/com/webank/wedatasphere/dss/sender/service/conf/DSSSenderServiceConf.java b/dss-commons/dss-sender-service/src/main/java/com/webank/wedatasphere/dss/sender/service/conf/DSSSenderServiceConf.java new file mode 100644 index 0000000000..41539abe6f --- /dev/null +++ b/dss-commons/dss-sender-service/src/main/java/com/webank/wedatasphere/dss/sender/service/conf/DSSSenderServiceConf.java @@ -0,0 +1,35 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.sender.service.conf; + +import com.webank.wedatasphere.linkis.common.conf.CommonVars; + +/** + * @author allenlliu + * @date 2021/6/24 17:23 + */ +public class DSSSenderServiceConf { + public static final CommonVars ORCHESTRATOR_SERVER_DEV_NAME = + CommonVars.apply("wds.dss.orc.server.dev.name", "DSS-Framework-Orchestrator-Server-Dev"); + + public static final CommonVars DSS_WORKFLOW_APPLICATION_NAME_DEV = + CommonVars.apply("wds.dss.workflow.name.dev", "dss-workflow-server-dev"); + + public static final CommonVars PROJECT_SERVER_NAME = + CommonVars.apply("wds.dss.project.sever.name", "dss-framework-project-server"); + +} diff --git a/dss-commons/dss-sender-service/src/main/java/com/webank/wedatasphere/dss/sender/service/impl/DSSSenderServiceImpl.java b/dss-commons/dss-sender-service/src/main/java/com/webank/wedatasphere/dss/sender/service/impl/DSSSenderServiceImpl.java new file mode 100644 index 0000000000..93ca057bcd --- /dev/null +++ b/dss-commons/dss-sender-service/src/main/java/com/webank/wedatasphere/dss/sender/service/impl/DSSSenderServiceImpl.java @@ -0,0 +1,71 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.sender.service.impl; + +import com.webank.wedatasphere.dss.common.label.DSSLabel; +import com.webank.wedatasphere.dss.sender.service.DSSSenderService; +import com.webank.wedatasphere.dss.sender.service.conf.DSSSenderServiceConf; +import com.webank.wedatasphere.linkis.rpc.Sender; +import java.util.List; + +/** + * @author allenlliu + * @date 2021/6/24 17:13 + */ +public class DSSSenderServiceImpl implements DSSSenderService { + + private final Sender orcSender = Sender.getSender(DSSSenderServiceConf.ORCHESTRATOR_SERVER_DEV_NAME.getValue()); + + private final Sender workflowSender = Sender.getSender(DSSSenderServiceConf.DSS_WORKFLOW_APPLICATION_NAME_DEV.getValue()); + + private final Sender projectSender = Sender.getSender(DSSSenderServiceConf.PROJECT_SERVER_NAME.getValue()); + @Override + public Sender getOrcSender() { + return orcSender; + } + + @Override + public Sender getOrcSender(List dssLabels) { + return orcSender; + } + + @Override + public Sender getScheduleOrcSender() { + return orcSender; + } + + @Override + public Sender getWorkflowSender(List dssLabels) { + return workflowSender; + } + + @Override + public Sender getWorkflowSender() { + return workflowSender; + } + + @Override + public Sender getSchedulerWorkflowSender() { + return workflowSender; + } + + @Override + public Sender getProjectServerSender() { + return projectSender; + } + +} diff --git a/dss-commons/pom.xml b/dss-commons/pom.xml new file mode 100644 index 0000000000..be492b6f45 --- /dev/null +++ b/dss-commons/pom.xml @@ -0,0 +1,37 @@ + + + + + + dss + com.webank.wedatasphere.dss + 1.0.0 + + 4.0.0 + + dss-commons + pom + + + dss-common + dss-contextservice + dss-sender-service + + + \ No newline at end of file