feat($OSS): new API to merge resource chunk

johnnymillergh · Aug 12, 2021 · 74e56d5 · 74e56d5
1 parent 314b713
commit 74e56d5
Show file tree

Hide file tree

Showing 6 changed files with 131 additions and 24 deletions.
diff --git a/.../src/main/java/com/jmsoftware/maf/osscenter/write/controller/WriteResourceController.java b/.../src/main/java/com/jmsoftware/maf/osscenter/write/controller/WriteResourceController.java
@@ -1,6 +1,8 @@
 package com.jmsoftware.maf.osscenter.write.controller;
 
 import com.jmsoftware.maf.common.bean.ResponseBodyBean;
+import com.jmsoftware.maf.osscenter.write.entity.MergeResourceChunkPayload;
+import com.jmsoftware.maf.osscenter.write.entity.ObjectResponse;
 import com.jmsoftware.maf.osscenter.write.service.WriteResourceService;
 import io.swagger.annotations.Api;
 import io.swagger.annotations.ApiOperation;
@@ -11,6 +13,8 @@
 import org.springframework.web.bind.annotation.*;
 import org.springframework.web.multipart.MultipartFile;
 
+import javax.validation.Valid;
+
 /**
  * <h1>WriteResourceController</h1>
  * <p>
@@ -28,22 +32,22 @@ public class WriteResourceController {
 
     @PostMapping("/upload/single")
     @ApiOperation(value = "Upload single resource", notes = "Upload single resource")
-    public ResponseBodyBean<String> uploadSingleResource(@RequestParam("file") MultipartFile multipartFile) {
+    public ResponseBodyBean<ObjectResponse> uploadSingleResource(@RequestParam("file") MultipartFile multipartFile) {
         return ResponseBodyBean.ofSuccess(this.writeResourceService.uploadSingleResource(multipartFile),
                                           this.messageSource.getMessage("uploaded", null,
                                                                         LocaleContextHolder.getLocale()));
     }
 
     @PostMapping("/upload/chunk/{chunkNumber}")
     @ApiOperation(value = "Upload chunk of resource", notes = "Upload chunk of resource")
-    public ResponseBodyBean<String> uploadResourceChunk(@RequestParam("file") MultipartFile multipartFile,
-                                                        @PathVariable Integer chunkNumber) {
+    public ResponseBodyBean<ObjectResponse> uploadResourceChunk(@RequestParam("file") MultipartFile multipartFile,
+                                                                @PathVariable Integer chunkNumber) {
         return ResponseBodyBean.ofSuccess(this.writeResourceService.uploadResourceChunk(multipartFile, chunkNumber));
     }
 
     @PutMapping("/merge/chunk")
     @ApiOperation(value = "Merge chunk of resource", notes = "Merge chunk of resource")
-    public ResponseBodyBean<String> mergeResourceChunk() {
-        return ResponseBodyBean.ofSuccess();
+    public ResponseBodyBean<ObjectResponse> mergeResourceChunk(@Valid @RequestBody MergeResourceChunkPayload payload) {
+        return ResponseBodyBean.ofSuccess(this.writeResourceService.mergeResourceChunk(payload));
     }
 }
diff --git a/...er/src/main/java/com/jmsoftware/maf/osscenter/write/entity/MergeResourceChunkPayload.java b/...er/src/main/java/com/jmsoftware/maf/osscenter/write/entity/MergeResourceChunkPayload.java
@@ -0,0 +1,21 @@
+package com.jmsoftware.maf.osscenter.write.entity;
+
+import lombok.Data;
+
+import javax.validation.Valid;
+import javax.validation.constraints.NotBlank;
+import javax.validation.constraints.NotEmpty;
+import java.util.List;
+
+/**
+ * Description: MergeResourceChunkPayload, change description here.
+ *
+ * @author Johnny Miller (锺俊), email: johnnysviva@outlook.com, date: 8/12/2021 8:48 PM
+ **/
+@Data
+public class MergeResourceChunkPayload {
+    @NotBlank
+    private String bucket;
+    @NotEmpty
+    private List<@Valid @NotBlank String> objectList;
+}
diff --git a/oss-center/src/main/java/com/jmsoftware/maf/osscenter/write/entity/ObjectResponse.java b/oss-center/src/main/java/com/jmsoftware/maf/osscenter/write/entity/ObjectResponse.java
@@ -0,0 +1,15 @@
+package com.jmsoftware.maf.osscenter.write.entity;
+
+import lombok.Data;
+
+/**
+ * Description: ObjectResponse, change description here.
+ *
+ * @author Johnny Miller (锺俊), email: johnnysviva@outlook.com, date: 8/12/2021 9:27 PM
+ **/
+@Data
+public class ObjectResponse {
+    private String bucket;
+    private String object;
+    private String etag;
+}
diff --git a/...center/src/main/java/com/jmsoftware/maf/osscenter/write/service/WriteResourceService.java b/...center/src/main/java/com/jmsoftware/maf/osscenter/write/service/WriteResourceService.java
@@ -1,9 +1,12 @@
 package com.jmsoftware.maf.osscenter.write.service;
 
+import com.jmsoftware.maf.osscenter.write.entity.MergeResourceChunkPayload;
+import com.jmsoftware.maf.osscenter.write.entity.ObjectResponse;
 import org.hibernate.validator.constraints.Range;
 import org.springframework.validation.annotation.Validated;
 import org.springframework.web.multipart.MultipartFile;
 
+import javax.validation.Valid;
 import javax.validation.constraints.NotNull;
 
 /**
@@ -23,7 +26,7 @@ public interface WriteResourceService {
      * @param multipartFile the multipart file
      * @return the string
      */
-    String uploadSingleResource(@NotNull MultipartFile multipartFile);
+    ObjectResponse uploadSingleResource(@NotNull MultipartFile multipartFile);
 
     /**
      * Upload resource chunk string.
@@ -32,6 +35,14 @@ public interface WriteResourceService {
      * @param chunkNumber   the chunk number
      * @return the string
      */
-    String uploadResourceChunk(@NotNull MultipartFile multipartFile,
-                               @NotNull @Range(max = MAX_CHUNK_NUMBER) Integer chunkNumber);
+    ObjectResponse uploadResourceChunk(@NotNull MultipartFile multipartFile,
+                                       @NotNull @Range(max = MAX_CHUNK_NUMBER) Integer chunkNumber);
+
+    /**
+     * Merge resource chunk string.
+     *
+     * @param payload the payload
+     * @return the string
+     */
+    ObjectResponse mergeResourceChunk(@Valid @NotNull MergeResourceChunkPayload payload);
 }
diff --git a/...c/main/java/com/jmsoftware/maf/osscenter/write/service/impl/WriteResourceServiceImpl.java b/...c/main/java/com/jmsoftware/maf/osscenter/write/service/impl/WriteResourceServiceImpl.java
@@ -1,12 +1,16 @@
 package com.jmsoftware.maf.osscenter.write.service.impl;
 
+import cn.hutool.core.collection.CollUtil;
 import cn.hutool.core.util.NumberUtil;
 import cn.hutool.core.util.ObjectUtil;
 import cn.hutool.core.util.StrUtil;
 import cn.hutool.crypto.digest.DigestUtil;
 import com.jmsoftware.maf.common.exception.BizException;
+import com.jmsoftware.maf.osscenter.write.entity.MergeResourceChunkPayload;
+import com.jmsoftware.maf.osscenter.write.entity.ObjectResponse;
 import com.jmsoftware.maf.osscenter.write.service.WriteResourceService;
 import com.jmsoftware.maf.springcloudstarter.minio.MinioHelper;
+import io.minio.ComposeSource;
 import io.minio.StatObjectResponse;
 import lombok.RequiredArgsConstructor;
 import lombok.SneakyThrows;
@@ -18,8 +22,12 @@
 import org.springframework.stereotype.Service;
 import org.springframework.web.multipart.MultipartFile;
 
+import javax.validation.Valid;
 import javax.validation.constraints.NotNull;
 import java.io.IOException;
+import java.util.HashMap;
+import java.util.List;
+import java.util.stream.Collectors;
 
 /**
  * <h1>WriteResourceServiceImpl</h1>
@@ -36,20 +44,23 @@ public class WriteResourceServiceImpl implements WriteResourceService {
 
     @Override
     @SneakyThrows
-    public String uploadSingleResource(@NotNull MultipartFile multipartFile) {
+    public ObjectResponse uploadSingleResource(@NotNull MultipartFile multipartFile) {
         val mediaType = this.parseMediaType(multipartFile);
-        val bucketMade = this.minioHelper.makeBucket(mediaType.getType());
+        this.minioHelper.makeBucket(mediaType.getType());
         val objectWriteResponse = this.minioHelper.put(mediaType.getType(), multipartFile.getOriginalFilename(),
                                                        multipartFile);
-        log.info("Uploaded single resource: {}/{}. bucketMade: {}", objectWriteResponse.bucket(),
-                 objectWriteResponse.object(), bucketMade);
-        return String.format("%s/%s", objectWriteResponse.bucket(), objectWriteResponse.object());
+        val objectResponse = new ObjectResponse();
+        objectResponse.setBucket(objectWriteResponse.bucket());
+        objectResponse.setObject(objectWriteResponse.object());
+        objectResponse.setEtag(objectWriteResponse.etag());
+        log.info("Uploaded single resource. {}", objectResponse);
+        return objectResponse;
     }
 
     @Override
     @SneakyThrows
-    public String uploadResourceChunk(@NotNull MultipartFile multipartFile,
-                                      @NotNull @Range(max = MAX_CHUNK_NUMBER) Integer chunkNumber) {
+    public ObjectResponse uploadResourceChunk(@NotNull MultipartFile multipartFile,
+                                              @NotNull @Range(max = MAX_CHUNK_NUMBER) Integer chunkNumber) {
         if (StrUtil.isBlank(multipartFile.getOriginalFilename())) {
             throw new IllegalArgumentException("File name required");
         }
@@ -62,20 +73,56 @@ public String uploadResourceChunk(@NotNull MultipartFile multipartFile,
         } catch (Exception e) {
             log.error("Exception occurred when looking for object. Exception message: {}", e.getMessage());
         }
+        val objectResponse = new ObjectResponse();
+        objectResponse.setBucket(mediaType.getType());
+        objectResponse.setObject(orderedFilename);
         if (ObjectUtil.isNotNull(statObjectResponse)) {
             val md5Hex = DigestUtil.md5Hex(multipartFile.getInputStream());
             if (StrUtil.equalsIgnoreCase(md5Hex, statObjectResponse.etag())) {
                 log.warn(
                         "Found previously uploaded file, skip uploading chunk. Filename: {}, statObjectResponse: {}, " +
                                 "MD5: {}", orderedFilename, statObjectResponse, md5Hex);
-                return statObjectResponse.etag();
+                objectResponse.setEtag(statObjectResponse.etag());
+                return objectResponse;
             }
         }
-        val bucketMade = this.minioHelper.makeBucket(mediaType.getType());
+        this.minioHelper.makeBucket(mediaType.getType());
         val objectWriteResponse = this.minioHelper.put(mediaType.getType(), orderedFilename, multipartFile);
-        log.info("Uploaded resource chunk: {}/{}. bucketMade: {}, etag (MD5): {}", objectWriteResponse.bucket(),
-                 objectWriteResponse.object(), bucketMade, objectWriteResponse.etag());
-        return objectWriteResponse.etag();
+        log.info("Uploaded resource chunk. {}", objectResponse);
+        objectResponse.setEtag(objectWriteResponse.etag());
+        return objectResponse;
+    }
+
+    @Override
+    public ObjectResponse mergeResourceChunk(@Valid @NotNull MergeResourceChunkPayload payload) {
+        val objectName = this.validateObject(payload.getObjectList());
+        val sources = payload.getObjectList()
+                .stream()
+                .map(object -> ComposeSource.builder().bucket(payload.getBucket()).object(object).build())
+                .collect(Collectors.toList());
+        val statObjectResponse = this.minioHelper.statObject(payload.getBucket(),
+                                                             CollUtil.getFirst(payload.getObjectList()));
+        val headers = new HashMap<String, String>(4);
+        headers.put("Content-Type", statObjectResponse.contentType());
+        val objectWriteResponse = this.minioHelper.composeObject(payload.getBucket(), objectName, sources, headers);
+        val objectResponse = new ObjectResponse();
+        objectResponse.setBucket(objectWriteResponse.bucket());
+        objectResponse.setObject(objectWriteResponse.object());
+        objectResponse.setEtag(objectWriteResponse.etag());
+        log.info("Merged resource chunks. {}", objectResponse);
+        return objectResponse;
+    }
+
+    private String validateObject(List<String> objectList) {
+        val objectNameSet = objectList.stream().map(object -> {
+            val lastIndexOfDot = StrUtil.lastIndexOfIgnoreCase(object, ".");
+            return StrUtil.subPre(object, lastIndexOfDot);
+        }).collect(Collectors.toSet());
+        if (CollUtil.size(objectNameSet) != 1) {
+            log.error("Object list is not valid! {}", objectNameSet);
+            throw new IllegalArgumentException("Object list is not valid");
+        }
+        return objectNameSet.iterator().next();
     }
 
     private MediaType parseMediaType(MultipartFile multipartFile) throws IOException, BizException {

diff --git a/...-cloud-starter/src/main/java/com/jmsoftware/maf/springcloudstarter/minio/MinioHelper.java b/...-cloud-starter/src/main/java/com/jmsoftware/maf/springcloudstarter/minio/MinioHelper.java
@@ -10,17 +10,16 @@
 import lombok.SneakyThrows;
 import lombok.extern.slf4j.Slf4j;
 import lombok.val;
+import org.springframework.lang.Nullable;
 import org.springframework.stereotype.Component;
 import org.springframework.validation.annotation.Validated;
 import org.springframework.web.multipart.MultipartFile;
 
-import javax.validation.constraints.Max;
-import javax.validation.constraints.Min;
-import javax.validation.constraints.NotBlank;
-import javax.validation.constraints.NotNull;
+import javax.validation.constraints.*;
 import java.io.InputStream;
 import java.util.LinkedList;
 import java.util.List;
+import java.util.Map;
 import java.util.stream.Collectors;
 
 /**
@@ -216,4 +215,14 @@ public String getPresignedObjectUrl(@NotBlank String bucket, @NotBlank String ob
         return this.minioClient.getPresignedObjectUrl(
                 GetPresignedObjectUrlArgs.builder().bucket(bucket).object(object).build());
     }
+
+    @SneakyThrows
+    public ObjectWriteResponse composeObject(@NotBlank String bucket, @NotBlank String object,
+                                             @NotEmpty List<ComposeSource> sources,@Nullable Map<String, String> headers) {
+        if (!this.bucketExists(bucket)) {
+            return null;
+        }
+        return this.minioClient.composeObject(
+                ComposeObjectArgs.builder().bucket(bucket).object(object).sources(sources).headers(headers).build());
+    }
 }