Skip to content

Commit

Permalink
[Feature-10495][Resource Center] Resource Center Refactor (#12076)
Browse files Browse the repository at this point in the history 
* resource center refactor - S3 services connection

Co-authored-by: caishunfeng <caishunfeng2021@gmail.com>
  • Loading branch information
@EricPyZhou @caishunfeng
EricPyZhou and caishunfeng authored Oct 26, 2022
1 parent cb70e89 commit 489e7fe
Show file tree
Hide file tree
Showing 73 changed files with 2,682 additions and 1,495 deletions.
151 changes: 76 additions & 75 deletions ...ler-api/src/main/java/org/apache/dolphinscheduler/api/controller/ResourcesController.java
View file

Large diffs are not rendered by default.

4 changes: 2 additions & 2 deletions ...er-api/src/main/java/org/apache/dolphinscheduler/api/dto/resources/ResourceComponent.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -35,7 +35,7 @@
@JsonPropertyOrder({"id", "pid", "name", "fullName", "description", "isDirctory", "children", "type"})
public abstract class ResourceComponent {

public ResourceComponent(int id, int pid, String name, String fullName, String description, boolean isDirctory) {
public ResourceComponent(int id, String pid, String name, String fullName, String description, boolean isDirctory) {
this.id = id;
this.pid = pid;
this.name = name;
Expand All @@ -53,7 +53,7 @@ public ResourceComponent(int id, int pid, String name, String fullName, String d
/**
* parent id
*/
protected int pid;
protected String pid;
/**
* name
*/
Expand Down
43 changes: 24 additions & 19 deletions .../main/java/org/apache/dolphinscheduler/api/dto/resources/visitor/ResourceTreeVisitor.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,10 +19,11 @@
import org.apache.dolphinscheduler.api.dto.resources.Directory;
import org.apache.dolphinscheduler.api.dto.resources.FileLeaf;
import org.apache.dolphinscheduler.api.dto.resources.ResourceComponent;
import org.apache.dolphinscheduler.dao.entity.Resource;
import org.apache.dolphinscheduler.service.storage.StorageEntity;

import java.util.ArrayList;
import java.util.List;
import java.util.Objects;

/**
* resource tree visitor
Expand All @@ -32,7 +33,7 @@ public class ResourceTreeVisitor implements Visitor {
/**
* resource list
*/
private List<Resource> resourceList;
private List<StorageEntity> resourceList;

public ResourceTreeVisitor() {
}
Expand All @@ -41,7 +42,7 @@ public ResourceTreeVisitor() {
* constructor
* @param resourceList resource list
*/
public ResourceTreeVisitor(List<Resource> resourceList) {
public ResourceTreeVisitor(List<StorageEntity> resourceList) {
this.resourceList = resourceList;
}

Expand All @@ -50,35 +51,37 @@ public ResourceTreeVisitor(List<Resource> resourceList) {
* @return resoruce component
*/
@Override
public ResourceComponent visit() {
public ResourceComponent visit(String rootPath) {
ResourceComponent rootDirectory = new Directory();
for (Resource resource : resourceList) {
for (StorageEntity resource : resourceList) {
// judge whether is root node
if (rootNode(resource)) {
if (rootNode(resource, rootPath)) {
// if it is a root node.
ResourceComponent tempResourceComponent = getResourceComponent(resource);
rootDirectory.add(tempResourceComponent);
tempResourceComponent.setChildren(setChildren(tempResourceComponent.getId(), resourceList));
tempResourceComponent.setChildren(setChildren(tempResourceComponent.getFullName(), resourceList));
}
}
return rootDirectory;
}

/**
* set children
* @param id id
* @param fullName unique path
* @param list resource list
* @return resource component list
*/
public static List<ResourceComponent> setChildren(int id, List<Resource> list) {
public static List<ResourceComponent> setChildren(String fullName, List<StorageEntity> list) {
// id is the unique value,
List<ResourceComponent> childList = new ArrayList<>();
for (Resource resource : list) {
if (id == resource.getPid()) {
for (StorageEntity resource : list) {
if (Objects.equals(fullName, resource.getPfullName())) {
ResourceComponent tempResourceComponent = getResourceComponent(resource);
childList.add(tempResourceComponent);
}
}
for (ResourceComponent resourceComponent : childList) {
resourceComponent.setChildren(setChildren(resourceComponent.getId(), list));
resourceComponent.setChildren(setChildren(resourceComponent.getFullName(), list));
}
if (childList.size() == 0) {
return new ArrayList<>();
Expand All @@ -91,17 +94,18 @@ public static List<ResourceComponent> setChildren(int id, List<Resource> list) {
* @param resource resource
* @return true if it is the root node
*/
public boolean rootNode(Resource resource) {
public boolean rootNode(StorageEntity resource, String rootPath) {

boolean isRootNode = true;
if (resource.getPid() != -1) {
for (Resource parent : resourceList) {
if (resource.getPid() == parent.getId()) {
if (!Objects.equals(resource.getPfullName(), rootPath)) {
for (StorageEntity parent : resourceList) {
if (Objects.equals(resource.getPfullName(), parent.getFullName())) {
isRootNode = false;
break;
}
}
}

return isRootNode;
}

Expand All @@ -110,7 +114,7 @@ public boolean rootNode(Resource resource) {
* @param resource resource
* @return resource component
*/
private static ResourceComponent getResourceComponent(Resource resource) {
private static ResourceComponent getResourceComponent(StorageEntity resource) {
ResourceComponent tempResourceComponent;
if (resource.isDirectory()) {
tempResourceComponent = new Directory();
Expand All @@ -119,9 +123,10 @@ private static ResourceComponent getResourceComponent(Resource resource) {
}

tempResourceComponent.setName(resource.getAlias());
tempResourceComponent.setFullName(resource.getFullName().replaceFirst("/", ""));
// tempResourceComponent.setFullName(resource.getFullName().replaceFirst("/",""));
tempResourceComponent.setFullName(resource.getFullName());
tempResourceComponent.setId(resource.getId());
tempResourceComponent.setPid(resource.getPid());
tempResourceComponent.setPid(resource.getPfullName());
tempResourceComponent.setIdValue(resource.getId(), resource.isDirectory());
tempResourceComponent.setDescription(resource.getDescription());
tempResourceComponent.setType(resource.getType());
Expand Down
2 changes: 1 addition & 1 deletion ...uler-api/src/main/java/org/apache/dolphinscheduler/api/dto/resources/visitor/Visitor.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,5 +28,5 @@ public interface Visitor {
* visit
* @return resource component
*/
ResourceComponent visit();
ResourceComponent visit(String rootPath);
}
1 change: 0 additions & 1 deletion .../main/java/org/apache/dolphinscheduler/api/permission/ResourcePermissionCheckService.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,6 @@ public interface ResourcePermissionCheckService<T> {
* @param authorizationType
* @param userId
* @param logger
* @param <T>
* @return
*/
Set<T> userOwnedResourceIdsAcquisition(Object authorizationType, Integer userId, Logger logger);
Expand Down
44 changes: 21 additions & 23 deletions ...scheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ResourcesService.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -57,9 +57,8 @@ Result<Object> createDirectory(User loginUser,
* @param loginUser login user
* @param name alias
* @param desc description
* @param file file
* @param type type
* @param pid parent id
* @param file file
* @param currentDir current directory
* @return create result code
*/
Expand All @@ -68,21 +67,20 @@ Result<Object> createResource(User loginUser,
String desc,
ResourceType type,
MultipartFile file,
int pid,
String currentDir);

/**
* update resource
* @param loginUser login user
* @param resourceId resource id
* @param name name
* @param desc description
* @param type resource type
* @param file resource file
* @return update result code
*/
Result<Object> updateResource(User loginUser,
int resourceId,
String fullName,
String tenantCode,
String name,
String desc,
ResourceType type,
Expand All @@ -98,8 +96,8 @@ Result<Object> updateResource(User loginUser,
* @param pageSize page size
* @return resource list page
*/
Result queryResourceListPaging(User loginUser, int directoryId, ResourceType type, String searchVal, Integer pageNo,
Integer pageSize);
Result queryResourceListPaging(User loginUser, String fullName, String resTenantCode,
ResourceType type, String searchVal, Integer pageNo, Integer pageSize);

/**
* query resource list
Expand All @@ -108,7 +106,7 @@ Result queryResourceListPaging(User loginUser, int directoryId, ResourceType typ
* @param type resource type
* @return resource list
*/
Map<String, Object> queryResourceList(User loginUser, ResourceType type);
Map<String, Object> queryResourceList(User loginUser, ResourceType type, String fullName);

/**
* query resource list by program type
Expand All @@ -123,11 +121,10 @@ Result queryResourceListPaging(User loginUser, int directoryId, ResourceType typ
* delete resource
*
* @param loginUser login user
* @param resourceId resource id
* @return delete result code
* @throws IOException exception
*/
Result<Object> delete(User loginUser, int resourceId) throws IOException;
Result<Object> delete(User loginUser, String fullName, String tenantCode) throws IOException;

/**
* verify resource by name and type
Expand All @@ -139,23 +136,22 @@ Result queryResourceListPaging(User loginUser, int directoryId, ResourceType typ
Result<Object> verifyResourceName(String fullName, ResourceType type, User loginUser);

/**
* verify resource by full name or pid and type
* @param fullName resource full name
* @param id resource id
* verify resource by file name
* @param fileName resource file name
* @param type resource type
* @return true if the resource full name or pid not exists, otherwise return false
* @return true if the resource file name, otherwise return false
*/
Result<Object> queryResource(User loginUser, String fullName, Integer id, ResourceType type);
Result<Object> queryResourceByFileName(User loginUser, String fileName, ResourceType type, String resTenantCode);

/**
* view resource file online
*
* @param resourceId resource id
* @param skipLineNum skip line number
* @param limit limit
* @param fullName fullName
* @return resource content
*/
Result<Object> readResource(User loginUser, int resourceId, int skipLineNum, int limit);
Result<Object> readResource(User loginUser, String fullName, String tenantCode, int skipLineNum, int limit);

/**
* create resource file online
Expand All @@ -169,7 +165,7 @@ Result queryResourceListPaging(User loginUser, int directoryId, ResourceType typ
* @return create result code
*/
Result<Object> onlineCreateResource(User loginUser, ResourceType type, String fileName, String fileSuffix,
String desc, String content, int pid, String currentDirectory);
String desc, String content, String currentDirectory);

/**
* create or update resource.
Expand Down Expand Up @@ -203,16 +199,16 @@ Result<Object> onlineCreateOrUpdateResourceWithDir(User loginUser, String fileFu
* @param content content
* @return update result cod
*/
Result<Object> updateResourceContent(User loginUser, int resourceId, String content);
Result<Object> updateResourceContent(User loginUser, String fullName, String tenantCode,
String content);

/**
* download file
*
* @param resourceId resource id
* @return resource content
* @throws IOException exception
*/
org.springframework.core.io.Resource downloadResource(User loginUser, int resourceId) throws IOException;
org.springframework.core.io.Resource downloadResource(User loginUser, String fullName) throws IOException;

/**
* list all file
Expand Down Expand Up @@ -270,9 +266,11 @@ Result<Object> onlineCreateOrUpdateResourceWithDir(User loginUser, String fileFu

/**
* get resource by id
* @param resourceId resource id
* @param fullName resource full name
* @param tenantCode owner's tenant code of resource
* @return resource
*/
Result<Object> queryResourceById(User loginUser, Integer resourceId);
Result<Object> queryResourceByFullName(User loginUser, String fullName, String tenantCode,
ResourceType type) throws IOException;

}
8 changes: 4 additions & 4 deletions ...inscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/UdfFuncService.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -35,18 +35,17 @@ public interface UdfFuncService {
* @param argTypes argument types
* @param database database
* @param desc description
* @param resourceId resource id
* @param className class name
* @return create result code
*/
Result<Object> createUdfFunction(User loginUser,
String funcName,
String className,
String fullName,
String argTypes,
String database,
String desc,
UdfType type,
int resourceId);
UdfType type);

/**
* query udf function
Expand All @@ -66,6 +65,7 @@ Result<Object> createUdfFunction(User loginUser,
* @param database data base
* @param desc description
* @param resourceId resource id
* @param fullName resource full name
* @param className class name
* @return update result code
*/
Expand All @@ -77,7 +77,7 @@ Result<Object> updateUdfFunc(User loginUser,
String database,
String desc,
UdfType type,
int resourceId);
String fullName);

/**
* query udf function list paging
Expand Down
Loading

0 comments on commit 489e7fe

Please sign in to comment.