[Python] Supports creating or editing resources.

dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/python/PythonGateway.java

@@ -547,6 +547,49 @@ public Map<String, Object> queryResourcesFileInfo(String userName, String fullNa
         return result;
     }
 
+    /**
+     * create or update resource.
+     * If the folder is not already created, it will be
+     *
+     * @param userName user who create or update resource
+     * @param resourceDir The folder where the resource resides.
+     * @param resourceName The name of resource.Do not include file suffixes.
+     * @param resourceSuffix suffix of resource
+     * @param description description of resource
+     * @param resourceContent content of resource
+     * @return id of resource
+     */
+    public Integer createOrUpdateResource(
+            String userName, String resourceDir, String resourceName, String resourceSuffix, String description, String resourceContent) {
+        User user = usersService.queryUser(userName);
+        String fullName = resourceDir + "/" + resourceName + "." + resourceSuffix;
+        Result<Object> existResult = resourceService.queryResource(user, fullName, null, ResourceType.FILE);
+        if (existResult.getCode() == Status.SUCCESS.getCode()) {
+            Resource resource = (Resource) existResult.getData();
+            return this.updateResoure(user, resource.getId(), fullName, resourceContent);
+        } else if (existResult.getCode() == Status.RESOURCE_NOT_EXIST.getCode()) {
+            Result<Object> onlineCreateResourceResult = resourceService.onlineCreateResourceWithDir(
+                    user, resourceName, resourceSuffix, description, resourceContent, resourceDir);
+            if (onlineCreateResourceResult.getCode() == Status.SUCCESS.getCode()) {
+                Map<String, Object> resultMap = (Map<String, Object>) onlineCreateResourceResult.getData();
+                return (int) resultMap.get("id");
+            }
+        }
+        String msg = String.format("Can not create or update resource: %s", fullName);
+        logger.error(msg);
+        throw new IllegalArgumentException(msg);
+    }
+
+    private int updateResoure(User user, int resourceId, String resourceFullName, String resourceContent) {
+        Result<Object> updateResult = resourceService.updateResourceContent(user, resourceId, resourceContent);
+        if (updateResult.getCode() != Status.SUCCESS.getCode()) {
+            String msg = String.format("Can not update resource %s", resourceFullName);
+            logger.error(msg);
+            throw new IllegalArgumentException(msg);
+        }
+        return resourceId;
+    }
+
     @PostConstruct
     public void init() {
         if (pythonGatewayConfiguration.getEnabled()) {

dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ResourcesService.java

@@ -168,6 +168,20 @@ Result<Object> updateResource(User loginUser,
      */
     Result<Object> onlineCreateResource(User loginUser, ResourceType type, String fileName, String fileSuffix, String desc, String content,int pid,String currentDirectory);
 
+    /**
+     * create or update resource.
+     * If the folder is not already created, it will be
+     *
+     * @param loginUser user who create or update resource
+     * @param currentDirectory The folder where the resource resides.
+     * @param fileName The name of resource.Do not include file suffixes.
+     * @param fileSuffix suffix of resource
+     * @param desc description of resource
+     * @param content content of resource
+     * @return create result code
+     */
+    Result<Object> onlineCreateResourceWithDir(User loginUser, String fileName, String fileSuffix, String desc, String content, String currentDirectory);
+
     /**
      * updateProcessInstance resource
      *

dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ResourcesServiceImpl.java

@@ -1117,6 +1117,54 @@ public Result<Object> onlineCreateResource(User loginUser, ResourceType type, St
         return result;
     }
 
+    /**
+     * create or update resource.
+     * If the folder is not already created, it will be
+     *
+     * @param loginUser user who create or update resource
+     * @param currentDirectory The folder where the resource resides.
+     * @param fileName The name of resource.Do not include file suffixes.
+     * @param fileSuffix suffix of resource
+     * @param desc description of resource
+     * @param content content of resource
+     * @return create result code
+     */
+    @Override
+    @Transactional
+    public Result<Object> onlineCreateResourceWithDir(User loginUser, String fileName, String fileSuffix, String desc, String content, String currentDirectory) {
+        String[] dirNames = currentDirectory.split("/");
+        int pid = -1;
+        StringBuilder currDirPath = new StringBuilder();
+        for (String dirName : dirNames) {
+            if (StringUtils.isNotEmpty(dirName)) {
+                pid = queryOrCreateDirId(loginUser, pid, currDirPath.toString(), dirName);
+                currDirPath.append("/").append(dirName);
+            }
+        }
+        return this.onlineCreateResource(
+                loginUser, ResourceType.FILE, fileName, fileSuffix, desc, content, pid, currDirPath.toString());
+    }
+
+    private int queryOrCreateDirId(User user, int pid, String currentDir, String dirName) {
+        String dirFullName = currentDir + "/" + dirName;
+        Result<Object> dirResult = this.queryResource(user, dirFullName, null, ResourceType.FILE);
+        if (dirResult.getCode() == Status.SUCCESS.getCode()) {
+            Resource dirResource = (Resource) dirResult.getData();
+            return dirResource.getId();
+        } else if (dirResult.getCode() == Status.RESOURCE_NOT_EXIST.getCode()) {
+            // create dir
+            Result<Object> createDirResult = this.createDirectory(
+                    user, dirName, "", ResourceType.FILE, pid, currentDir);
+            if (createDirResult.getCode() == Status.SUCCESS.getCode()) {
+                Map<String, Object> resultMap = (Map<String, Object>) createDirResult.getData();
+                return (int) resultMap.get("id");
+            }
+        }
+        String msg = String.format("Can not create dir %s", dirFullName);
+        logger.error(msg);
+        throw new IllegalArgumentException(msg);
+    }
+
     private void permissionPostHandle(ResourceType resourceType, User loginUser, Integer resourceId) {
         AuthorizationType authorizationType = resourceType.equals(ResourceType.FILE) ? AuthorizationType.RESOURCE_FILE_ID : AuthorizationType.UDF_FILE;
         permissionPostHandle(authorizationType, loginUser.getId(), Collections.singletonList(resourceId), logger);

dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/python/PythonGatewayTest.java

@@ -40,6 +40,7 @@
 import org.mockito.junit.MockitoJUnitRunner;
 
 import java.util.Date;
+import java.util.HashMap;
 import java.util.Map;
 
 /**
@@ -96,6 +97,64 @@ public void testGetDependentInfo() {
         Assert.assertEquals((long) result.get("taskDefinitionCode"), taskDefinition.getCode());
     }
 
+    @Test
+    public void testUpdateResource() {
+        User user = getTestUser();
+        Mockito.when(usersService.queryUser(user.getUserName())).thenReturn(user);
+
+        String resourceDir = "/dev";
+        String resourceName = "test";
+        String resourceSuffix = "py";
+        String resourceFullName = resourceDir + "/" + resourceName + "." + resourceSuffix;
+
+        Result<Object> queryMockResult = new Result<>();
+        queryMockResult.setCode(Status.SUCCESS.getCode());
+        Resource resource = getTestResource();
+        queryMockResult.setData(resource);
+        Mockito.when(resourcesService.queryResource(user, resourceFullName, null, ResourceType.FILE)).thenReturn(queryMockResult);
+
+        Result<Object> updateMockResult = new Result<>();
+        updateMockResult.setCode(Status.SUCCESS.getCode());
+
+        Mockito.when(resourcesService.updateResourceContent(user, resource.getId(), "")).thenReturn(updateMockResult);
+
+        int id = pythonGateway.createOrUpdateResource(
+                user.getUserName(), resourceDir, resourceName, resourceSuffix, "", "");
+        Assert.assertEquals(id, resource.getId());
+    }
+
+    @Test
+    public void testCreateResource() {
+        User user = getTestUser();
+        Mockito.when(usersService.queryUser(user.getUserName())).thenReturn(user);
+
+        String resourceDir = "/dir1/dir2";
+        String resourceName = "test";
+        String resourceSuffix = "py";
+        String desc = "desc";
+        String content = "content";
+        String resourceFullName = resourceDir + "/" + resourceName + "." + resourceSuffix;
+
+        Result<Object> queryMockResult = new Result<>();
+        queryMockResult.setCode(Status.RESOURCE_NOT_EXIST.getCode());
+        Mockito.when(resourcesService.queryResource(user, resourceFullName, null, ResourceType.FILE))
+                .thenReturn(queryMockResult);
+
+        int resourceId = 3;
+        Result<Object> createResourceResult = new Result<>();
+        createResourceResult.setCode(Status.SUCCESS.getCode());
+        Map<String, Object> resourceMap = new HashMap<>();
+        resourceMap.put("id", resourceId);
+        createResourceResult.setData(resourceMap);
+
+        Mockito.when(resourcesService.onlineCreateResourceWithDir(user, resourceName, resourceSuffix, desc, content, resourceDir))
+                .thenReturn(createResourceResult);
+
+        int id = pythonGateway.createOrUpdateResource(
+                user.getUserName(), resourceDir, resourceName, resourceSuffix, desc, content);
+        Assert.assertEquals(id, resourceId);
+    }
+
 
     @Test
     public void testQueryResourcesFileInfo() {

dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ResourcesServiceTest.java

@@ -605,6 +605,63 @@ public void testOnlineCreateResource() {
 
     }
 
+    @Test
+    public void testOnlineCreateResourceWithDir() {
+        User user = getUser();
+        user.setId(1);
+
+        String dir1Path = "/dir1";
+        String dir2Path = "/dir2";
+        String resourceDir = dir1Path + dir2Path;
+        String resourceName = "test";
+        String resourceSuffix = "py";
+        String desc = "desc";
+        String content = "content";
+
+        Resource dir1 = new Resource();
+        dir1.setFullName(dir1Path);
+        dir1.setId(1);
+        dir1.setUserId(user.getId());
+        Resource dir2 = new Resource();
+        dir2.setFullName(resourceDir);
+        dir2.setUserId(user.getId());
+        Mockito.when(resourcesMapper.queryResource(dir1.getFullName(), ResourceType.FILE.ordinal())).thenReturn(Collections.singletonList(dir1));
+        Mockito.when(resourcesMapper.queryResource(resourceDir, ResourceType.FILE.ordinal())).thenReturn(null);
+        PowerMockito.when(resourcePermissionCheckService.operationPermissionCheck(
+                AuthorizationType.RESOURCE_FILE_ID, 1, ApiFuncIdentificationConstant.FILE_VIEW, serviceLogger)).thenReturn(true);
+        PowerMockito.when(resourcePermissionCheckService.resourcePermissionCheck(
+                AuthorizationType.RESOURCE_FILE_ID, new Object[]{dir1.getId()}, 1, serviceLogger)).thenReturn(true);
+
+        Tenant tenant = getTenant();
+        PowerMockito.when(resourcePermissionCheckService.operationPermissionCheck(
+                AuthorizationType.RESOURCE_FILE_ID, 1, ApiFuncIdentificationConstant.FOLDER_ONLINE_CREATE, serviceLogger)).thenReturn(true);
+        PowerMockito.when(resourcePermissionCheckService.resourcePermissionCheck(
+                AuthorizationType.RESOURCE_FILE_ID, null, 1, serviceLogger)).thenReturn(true);
+        try {
+            PowerMockito.when(storageOperate.mkdir(tenant.getTenantCode(), null)).thenReturn(true);
+        } catch (IOException e) {
+            logger.error("storage error", e);
+        }
+
+        PowerMockito.when(resourcePermissionCheckService.operationPermissionCheck(
+                AuthorizationType.RESOURCE_FILE_ID, 1, ApiFuncIdentificationConstant.FILE_ONLINE_CREATE, serviceLogger)).thenReturn(true);
+        PowerMockito.when(resourcePermissionCheckService.resourcePermissionCheck(
+                AuthorizationType.RESOURCE_FILE_ID, null, 1, serviceLogger)).thenReturn(true);
+        PowerMockito.when(PropertyUtils.getResUploadStartupState()).thenReturn(true);
+        PowerMockito.when(resourcePermissionCheckService.operationPermissionCheck(
+                AuthorizationType.RESOURCE_FILE_ID, 1, ApiFuncIdentificationConstant.FILE_RENAME, serviceLogger)).thenReturn(true);
+        PowerMockito.when(resourcePermissionCheckService.resourcePermissionCheck(
+                AuthorizationType.RESOURCE_FILE_ID, null, 1, serviceLogger)).thenReturn(true);
+        Mockito.when(tenantMapper.queryById(1)).thenReturn(getTenant());
+        Mockito.when(resourcesMapper.selectById(dir1.getId())).thenReturn(dir1);
+        Mockito.when(resourcesMapper.selectById(dir2.getId())).thenReturn(dir2);
+        Mockito.when(FileUtils.getUploadFilename(Mockito.anyString(), Mockito.anyString())).thenReturn("test");
+        PowerMockito.when(FileUtils.writeContent2File(Mockito.anyString(), Mockito.anyString())).thenReturn(true);
+
+        Result<Object> result = resourcesService.onlineCreateResourceWithDir(user, resourceName, resourceSuffix, desc, content, resourceDir);
+        Assert.assertEquals(Status.SUCCESS.getMsg(), result.getMsg());
+    }
+
     @Test
     public void testUpdateResourceContent() {
         PowerMockito.when(PropertyUtils.getResUploadStartupState()).thenReturn(false);
@@ -896,7 +953,7 @@ private Resource getResource(int resourceId) {
         return resource;
     }
 
-    private Resource getResource(int resourceId,ResourceType type) {
+    private Resource getResource(int resourceId, ResourceType type) {
 
         Resource resource = new Resource();
         resource.setId(resourceId);

dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/core/resource_definition.py

@@ -0,0 +1,66 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+"""Module resource."""
+
+from typing import Optional
+
+from pydolphinscheduler.core.base import Base
+from pydolphinscheduler.java_gateway import launch_gateway
+
+
+class ResourceDefinition(Base):
+    """resource object, will define the resources that you want to create or update.
+
+    :param user: The user who create or update resource.
+    :param name: The name of resource.Do not include file suffixes.
+    :param suffix: The suffix of resource.
+    :param current_dir: The folder where the resource resides.
+    :param content: The description of resource.
+    :param description: The description of resource.
+    """
+
+    _DEFINE_ATTR = {"user", "name", "suffix", "current_dir", "content", "description"}
+
+    def __init__(
+        self,
+        user: str,
+        name: str,
+        suffix: str,
+        current_dir: str,
+        content: str,
+        description: Optional[str] = None,
+    ):
+        super().__init__(name, description)
+        self.user = user
+        self.suffix = suffix
+        self.current_dir = current_dir
+        self.content = content
+        self._resource_code = None
+
+    def submit(self) -> int:
+        """Submit resource to java gateway."""
+        gateway = launch_gateway()
+        self._resource_code = gateway.entry_point.createOrUpdateResource(
+            self.user,
+            self.current_dir,
+            self.name,
+            self.suffix,
+            self.description,
+            self.content,
+        )
+        return self._resource_code

dolphinscheduler-python/pydolphinscheduler/tests/core/test_resource_definition.py

@@ -0,0 +1,47 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+"""Test resource definition."""
+
+from pydolphinscheduler.core.resource_definition import ResourceDefinition
+
+
+def test_sql_get_define():
+    """Test resource set attributes which get with same type."""
+    user = "admin"
+    name = "test"
+    suffix = "py"
+    current_dir = "/dev"
+    content = """print("hello world")"""
+    description = "hello world"
+    expect = {
+        "user": user,
+        "name": name,
+        "suffix": suffix,
+        "currentDir": current_dir,
+        "content": content,
+        "description": description,
+    }
+    resourceDefinition = ResourceDefinition(
+        user=user,
+        name=name,
+        suffix=suffix,
+        current_dir=current_dir,
+        content=content,
+        description=description,
+    )
+    assert resourceDefinition.get_define() == expect