feat($OSS): new API to upload resource chunk

johnnymillergh · Aug 12, 2021 · 314b713 · 314b713
1 parent f8a8d56
commit 314b713
Show file tree

Hide file tree

Showing 3 changed files with 79 additions and 22 deletions.
diff --git a/.../src/main/java/com/jmsoftware/maf/osscenter/write/controller/WriteResourceController.java b/.../src/main/java/com/jmsoftware/maf/osscenter/write/controller/WriteResourceController.java
@@ -1,22 +1,16 @@
 package com.jmsoftware.maf.osscenter.write.controller;
 
 import com.jmsoftware.maf.common.bean.ResponseBodyBean;
-import com.jmsoftware.maf.common.exception.BizException;
 import com.jmsoftware.maf.osscenter.write.service.WriteResourceService;
 import io.swagger.annotations.Api;
 import io.swagger.annotations.ApiOperation;
 import lombok.RequiredArgsConstructor;
-import lombok.SneakyThrows;
 import org.springframework.context.MessageSource;
 import org.springframework.context.i18n.LocaleContextHolder;
 import org.springframework.validation.annotation.Validated;
-import org.springframework.web.bind.annotation.PostMapping;
-import org.springframework.web.bind.annotation.RequestParam;
-import org.springframework.web.bind.annotation.RestController;
+import org.springframework.web.bind.annotation.*;
 import org.springframework.web.multipart.MultipartFile;
 
-import java.io.IOException;
-
 /**
  * <h1>WriteResourceController</h1>
  * <p>
@@ -33,11 +27,23 @@ public class WriteResourceController {
     private final MessageSource messageSource;
 
     @PostMapping("/upload/single")
-    @SneakyThrows({IOException.class, BizException.class})
     @ApiOperation(value = "Upload single resource", notes = "Upload single resource")
     public ResponseBodyBean<String> uploadSingleResource(@RequestParam("file") MultipartFile multipartFile) {
         return ResponseBodyBean.ofSuccess(this.writeResourceService.uploadSingleResource(multipartFile),
                                           this.messageSource.getMessage("uploaded", null,
                                                                         LocaleContextHolder.getLocale()));
     }
+
+    @PostMapping("/upload/chunk/{chunkNumber}")
+    @ApiOperation(value = "Upload chunk of resource", notes = "Upload chunk of resource")
+    public ResponseBodyBean<String> uploadResourceChunk(@RequestParam("file") MultipartFile multipartFile,
+                                                        @PathVariable Integer chunkNumber) {
+        return ResponseBodyBean.ofSuccess(this.writeResourceService.uploadResourceChunk(multipartFile, chunkNumber));
+    }
+
+    @PutMapping("/merge/chunk")
+    @ApiOperation(value = "Merge chunk of resource", notes = "Merge chunk of resource")
+    public ResponseBodyBean<String> mergeResourceChunk() {
+        return ResponseBodyBean.ofSuccess();
+    }
 }
diff --git a/...center/src/main/java/com/jmsoftware/maf/osscenter/write/service/WriteResourceService.java b/...center/src/main/java/com/jmsoftware/maf/osscenter/write/service/WriteResourceService.java
@@ -1,11 +1,10 @@
 package com.jmsoftware.maf.osscenter.write.service;
 
-import com.jmsoftware.maf.common.exception.BizException;
+import org.hibernate.validator.constraints.Range;
 import org.springframework.validation.annotation.Validated;
 import org.springframework.web.multipart.MultipartFile;
 
 import javax.validation.constraints.NotNull;
-import java.io.IOException;
 
 /**
  * <h1>WriteResourceService</h1>
@@ -16,13 +15,23 @@
  **/
 @Validated
 public interface WriteResourceService {
+    long MAX_CHUNK_NUMBER = 999;
+
     /**
      * Upload single resource string.
      *
      * @param multipartFile the multipart file
      * @return the string
-     * @throws IOException  the io exception
-     * @throws BizException the business exception
      */
-    String uploadSingleResource(@NotNull MultipartFile multipartFile) throws IOException, BizException;
+    String uploadSingleResource(@NotNull MultipartFile multipartFile);
+
+    /**
+     * Upload resource chunk string.
+     *
+     * @param multipartFile the multipart file
+     * @param chunkNumber   the chunk number
+     * @return the string
+     */
+    String uploadResourceChunk(@NotNull MultipartFile multipartFile,
+                               @NotNull @Range(max = MAX_CHUNK_NUMBER) Integer chunkNumber);
 }
diff --git a/...c/main/java/com/jmsoftware/maf/osscenter/write/service/impl/WriteResourceServiceImpl.java b/...c/main/java/com/jmsoftware/maf/osscenter/write/service/impl/WriteResourceServiceImpl.java
@@ -1,14 +1,20 @@
 package com.jmsoftware.maf.osscenter.write.service.impl;
 
+import cn.hutool.core.util.NumberUtil;
+import cn.hutool.core.util.ObjectUtil;
 import cn.hutool.core.util.StrUtil;
+import cn.hutool.crypto.digest.DigestUtil;
 import com.jmsoftware.maf.common.exception.BizException;
 import com.jmsoftware.maf.osscenter.write.service.WriteResourceService;
 import com.jmsoftware.maf.springcloudstarter.minio.MinioHelper;
+import io.minio.StatObjectResponse;
 import lombok.RequiredArgsConstructor;
+import lombok.SneakyThrows;
 import lombok.extern.slf4j.Slf4j;
 import lombok.val;
 import org.apache.tika.Tika;
 import org.apache.tika.mime.MediaType;
+import org.hibernate.validator.constraints.Range;
 import org.springframework.stereotype.Service;
 import org.springframework.web.multipart.MultipartFile;
 
@@ -29,20 +35,56 @@ public class WriteResourceServiceImpl implements WriteResourceService {
     private final MinioHelper minioHelper;
 
     @Override
-    public String uploadSingleResource(@NotNull MultipartFile multipartFile) throws IOException, BizException {
+    @SneakyThrows
+    public String uploadSingleResource(@NotNull MultipartFile multipartFile) {
+        val mediaType = this.parseMediaType(multipartFile);
+        val bucketMade = this.minioHelper.makeBucket(mediaType.getType());
+        val objectWriteResponse = this.minioHelper.put(mediaType.getType(), multipartFile.getOriginalFilename(),
+                                                       multipartFile);
+        log.info("Uploaded single resource: {}/{}. bucketMade: {}", objectWriteResponse.bucket(),
+                 objectWriteResponse.object(), bucketMade);
+        return String.format("%s/%s", objectWriteResponse.bucket(), objectWriteResponse.object());
+    }
+
+    @Override
+    @SneakyThrows
+    public String uploadResourceChunk(@NotNull MultipartFile multipartFile,
+                                      @NotNull @Range(max = MAX_CHUNK_NUMBER) Integer chunkNumber) {
+        if (StrUtil.isBlank(multipartFile.getOriginalFilename())) {
+            throw new IllegalArgumentException("File name required");
+        }
+        val mediaType = this.parseMediaType(multipartFile);
+        val orderedFilename = String.format("%s.chunk%s", multipartFile.getOriginalFilename(),
+                                            NumberUtil.decimalFormat("000", chunkNumber));
+        StatObjectResponse statObjectResponse = null;
+        try {
+            statObjectResponse = this.minioHelper.statObject(mediaType.getType(), orderedFilename);
+        } catch (Exception e) {
+            log.error("Exception occurred when looking for object. Exception message: {}", e.getMessage());
+        }
+        if (ObjectUtil.isNotNull(statObjectResponse)) {
+            val md5Hex = DigestUtil.md5Hex(multipartFile.getInputStream());
+            if (StrUtil.equalsIgnoreCase(md5Hex, statObjectResponse.etag())) {
+                log.warn(
+                        "Found previously uploaded file, skip uploading chunk. Filename: {}, statObjectResponse: {}, " +
+                                "MD5: {}", orderedFilename, statObjectResponse, md5Hex);
+                return statObjectResponse.etag();
+            }
+        }
+        val bucketMade = this.minioHelper.makeBucket(mediaType.getType());
+        val objectWriteResponse = this.minioHelper.put(mediaType.getType(), orderedFilename, multipartFile);
+        log.info("Uploaded resource chunk: {}/{}. bucketMade: {}, etag (MD5): {}", objectWriteResponse.bucket(),
+                 objectWriteResponse.object(), bucketMade, objectWriteResponse.etag());
+        return objectWriteResponse.etag();
+    }
+
+    private MediaType parseMediaType(MultipartFile multipartFile) throws IOException, BizException {
         val tika = new Tika();
         val detectedMediaType = tika.detect(multipartFile.getInputStream());
         log.info("Detected media type: {}", detectedMediaType);
         if (StrUtil.isBlank(detectedMediaType)) {
             throw new BizException("Media extension detection failed!");
         }
-        val mediaType = MediaType.parse(detectedMediaType);
-        val mediaBaseType = mediaType.getType();
-        val bucketMade = this.minioHelper.makeBucket(mediaBaseType);
-        val objectWriteResponse = this.minioHelper.put(mediaBaseType, multipartFile.getOriginalFilename(),
-                                                       multipartFile);
-        log.info("Uploaded single resource: {}/{}. bucketMade: {}", objectWriteResponse.bucket(),
-                 objectWriteResponse.object(), bucketMade);
-        return String.format("%s/%s", objectWriteResponse.bucket(), objectWriteResponse.object());
+        return MediaType.parse(detectedMediaType);
     }
 }