perf($OSS): refine process of uploading chunk

oss-center/src/main/java/com/jmsoftware/maf/osscenter/write/controller/WriteResourceController.java

@@ -41,8 +41,10 @@ public ResponseBodyBean<ObjectResponse> uploadSingleResource(@RequestParam("file
     @PostMapping("/upload/chunk/{chunkNumber}")
     @ApiOperation(value = "Upload chunk of resource", notes = "Upload chunk of resource")
     public ResponseBodyBean<ObjectResponse> uploadResourceChunk(@RequestParam("file") MultipartFile multipartFile,
+                                                                @RequestParam(required = false) String bucket,
                                                                 @PathVariable Integer chunkNumber) {
-        return ResponseBodyBean.ofSuccess(this.writeResourceService.uploadResourceChunk(multipartFile, chunkNumber));
+        return ResponseBodyBean.ofSuccess(
+                this.writeResourceService.uploadResourceChunk(multipartFile, bucket, chunkNumber));
     }
 
     @PutMapping("/merge/chunk")

oss-center/src/main/java/com/jmsoftware/maf/osscenter/write/service/WriteResourceService.java

@@ -3,6 +3,7 @@
 import com.jmsoftware.maf.osscenter.write.entity.MergeResourceChunkPayload;
 import com.jmsoftware.maf.osscenter.write.entity.ObjectResponse;
 import org.hibernate.validator.constraints.Range;
+import org.springframework.lang.Nullable;
 import org.springframework.validation.annotation.Validated;
 import org.springframework.web.multipart.MultipartFile;
 
@@ -32,10 +33,12 @@ public interface WriteResourceService {
      * Upload resource chunk string.
      *
      * @param multipartFile the multipart file
+     * @param bucket        the bucket
      * @param chunkNumber   the chunk number
      * @return the string
      */
     ObjectResponse uploadResourceChunk(@NotNull MultipartFile multipartFile,
+                                       @Nullable String bucket,
                                        @NotNull @Range(max = MAX_CHUNK_NUMBER) Integer chunkNumber);
 
     /**

oss-center/src/main/java/com/jmsoftware/maf/osscenter/write/service/impl/WriteResourceServiceImpl.java

@@ -2,23 +2,20 @@
 
 import cn.hutool.core.collection.CollUtil;
 import cn.hutool.core.util.NumberUtil;
-import cn.hutool.core.util.ObjectUtil;
 import cn.hutool.core.util.StrUtil;
-import cn.hutool.crypto.digest.DigestUtil;
-import com.jmsoftware.maf.common.exception.BizException;
 import com.jmsoftware.maf.osscenter.write.entity.MergeResourceChunkPayload;
 import com.jmsoftware.maf.osscenter.write.entity.ObjectResponse;
 import com.jmsoftware.maf.osscenter.write.service.WriteResourceService;
 import com.jmsoftware.maf.springcloudstarter.minio.MinioHelper;
 import io.minio.ComposeSource;
-import io.minio.StatObjectResponse;
 import lombok.RequiredArgsConstructor;
 import lombok.SneakyThrows;
 import lombok.extern.slf4j.Slf4j;
 import lombok.val;
 import org.apache.tika.Tika;
 import org.apache.tika.mime.MediaType;
 import org.hibernate.validator.constraints.Range;
+import org.springframework.lang.Nullable;
 import org.springframework.stereotype.Service;
 import org.springframework.web.multipart.MultipartFile;
 
@@ -27,6 +24,7 @@
 import java.io.IOException;
 import java.util.HashMap;
 import java.util.List;
+import java.util.Objects;
 import java.util.stream.Collectors;
 
 /**
@@ -57,39 +55,56 @@ public ObjectResponse uploadSingleResource(@NotNull MultipartFile multipartFile)
         return objectResponse;
     }
 
+    /**
+     * {@inheritDoc}
+     * <p>
+     * <h2 >What is an S3 ETag?</h2>
+     * <p>According to
+     * <a href='https://docs.aws.amazon.com/AmazonS3/latest/API/RESTCommonResponseHeaders.html'>Amazon</a>:</p>
+     * <blockquote><p>The ETag may or may not be an MD5 digest of the object data</p>
+     * </blockquote>
+     * <p>Basically, if the object was uploaded with a single PUT operation and doesnt use Customer Managed or KMS
+     * keys for encryption then the resulting ETag is just the MD5 hexdigest of the object.</p>
+     * <p>However, more importantly:</p>
+     * <blockquote><p>If an object is created by either the Multipart Upload or Part Copy operation, the ETag is not
+     * an MD5 digest, regardless of the method of encryption.</p>
+     * </blockquote>
+     * <p>Well if it’s not an MD5 digest then what is it?!</p>
+     * <blockquote><p>For multipart uploads the ETag is the MD5 hexdigest of each part’s MD5 digest concatenated
+     * together, followed by the number of parts separated by a dash.</p>
+     * </blockquote>
+     * <p>E.g. for a two part object the ETag may look something like this:</p>
+     * <blockquote><p>d41d8cd98f00b204e9800998ecf8427e-2</p>
+     * </blockquote>
+     * <p>Which can be represented by:</p>
+     * <blockquote><p>hexmd5( md5( part1 ) + md5( part2 ) )-{ number of parts }</p>
+     * </blockquote>
+     *
+     * @see <a href='https://teppen.io/2018/06/23/aws_s3_etags/'>All about AWS S3 ETags</a>
+     */
     @Override
     @SneakyThrows
     public ObjectResponse uploadResourceChunk(@NotNull MultipartFile multipartFile,
+                                              @Nullable String bucket,
                                               @NotNull @Range(max = MAX_CHUNK_NUMBER) Integer chunkNumber) {
         if (StrUtil.isBlank(multipartFile.getOriginalFilename())) {
             throw new IllegalArgumentException("File name required");
         }
-        val mediaType = this.parseMediaType(multipartFile);
+        MediaType mediaType = null;
+        if (StrUtil.isBlank(bucket)) {
+            mediaType = this.parseMediaType(multipartFile);
+        }
+        // bucketName is either mediaType of given 'bucket'
+        val bucketName = StrUtil.isBlank(bucket) ? Objects.requireNonNull(mediaType).getType() : bucket;
         val orderedFilename = String.format("%s.chunk%s", multipartFile.getOriginalFilename(),
                                             NumberUtil.decimalFormat("000", chunkNumber));
-        StatObjectResponse statObjectResponse = null;
-        try {
-            statObjectResponse = this.minioHelper.statObject(mediaType.getType(), orderedFilename);
-        } catch (Exception e) {
-            log.error("Exception occurred when looking for object. Exception message: {}", e.getMessage());
-        }
         val objectResponse = new ObjectResponse();
-        objectResponse.setBucket(mediaType.getType());
+        objectResponse.setBucket(bucketName);
         objectResponse.setObject(orderedFilename);
-        if (ObjectUtil.isNotNull(statObjectResponse)) {
-            val md5Hex = DigestUtil.md5Hex(multipartFile.getInputStream());
-            if (StrUtil.equalsIgnoreCase(md5Hex, statObjectResponse.etag())) {
-                log.warn(
-                        "Found previously uploaded file, skip uploading chunk. Filename: {}, statObjectResponse: {}, " +
-                                "MD5: {}", orderedFilename, statObjectResponse, md5Hex);
-                objectResponse.setEtag(statObjectResponse.etag());
-                return objectResponse;
-            }
-        }
-        this.minioHelper.makeBucket(mediaType.getType());
-        val objectWriteResponse = this.minioHelper.put(mediaType.getType(), orderedFilename, multipartFile);
-        log.info("Uploaded resource chunk. {}", objectResponse);
+        this.minioHelper.makeBucket(bucketName);
+        val objectWriteResponse = this.minioHelper.put(bucketName, orderedFilename, multipartFile);
         objectResponse.setEtag(objectWriteResponse.etag());
+        log.info("Uploaded resource chunk. {}", objectResponse);
         return objectResponse;
     }
 
@@ -125,12 +140,12 @@ private String validateObject(List<String> objectList) {
         return objectNameSet.iterator().next();
     }
 
-    private MediaType parseMediaType(MultipartFile multipartFile) throws IOException, BizException {
+    private MediaType parseMediaType(MultipartFile multipartFile) throws IOException {
         val tika = new Tika();
         val detectedMediaType = tika.detect(multipartFile.getInputStream());
         log.info("Detected media type: {}", detectedMediaType);
         if (StrUtil.isBlank(detectedMediaType)) {
-            throw new BizException("Media extension detection failed!");
+            throw new IllegalStateException("Media extension detection failed!");
         }
         return MediaType.parse(detectedMediaType);
     }