From 314b713b1f5d8c34eb5378300a120239d3725e56 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johnny=20Miller=20=28=E9=94=BA=E4=BF=8A=29?= Date: Thu, 12 Aug 2021 20:46:36 +0800 Subject: [PATCH] feat($OSS): new API to upload resource chunk --- .../controller/WriteResourceController.java | 22 ++++--- .../write/service/WriteResourceService.java | 19 ++++-- .../impl/WriteResourceServiceImpl.java | 60 ++++++++++++++++--- 3 files changed, 79 insertions(+), 22 deletions(-) diff --git a/oss-center/src/main/java/com/jmsoftware/maf/osscenter/write/controller/WriteResourceController.java b/oss-center/src/main/java/com/jmsoftware/maf/osscenter/write/controller/WriteResourceController.java index d8999d23..99c9af55 100644 --- a/oss-center/src/main/java/com/jmsoftware/maf/osscenter/write/controller/WriteResourceController.java +++ b/oss-center/src/main/java/com/jmsoftware/maf/osscenter/write/controller/WriteResourceController.java @@ -1,22 +1,16 @@ package com.jmsoftware.maf.osscenter.write.controller; import com.jmsoftware.maf.common.bean.ResponseBodyBean; -import com.jmsoftware.maf.common.exception.BizException; import com.jmsoftware.maf.osscenter.write.service.WriteResourceService; import io.swagger.annotations.Api; import io.swagger.annotations.ApiOperation; import lombok.RequiredArgsConstructor; -import lombok.SneakyThrows; import org.springframework.context.MessageSource; import org.springframework.context.i18n.LocaleContextHolder; import org.springframework.validation.annotation.Validated; -import org.springframework.web.bind.annotation.PostMapping; -import org.springframework.web.bind.annotation.RequestParam; -import org.springframework.web.bind.annotation.RestController; +import org.springframework.web.bind.annotation.*; import org.springframework.web.multipart.MultipartFile; -import java.io.IOException; - /** *

WriteResourceController

*

@@ -33,11 +27,23 @@ public class WriteResourceController { private final MessageSource messageSource; @PostMapping("/upload/single") - @SneakyThrows({IOException.class, BizException.class}) @ApiOperation(value = "Upload single resource", notes = "Upload single resource") public ResponseBodyBean uploadSingleResource(@RequestParam("file") MultipartFile multipartFile) { return ResponseBodyBean.ofSuccess(this.writeResourceService.uploadSingleResource(multipartFile), this.messageSource.getMessage("uploaded", null, LocaleContextHolder.getLocale())); } + + @PostMapping("/upload/chunk/{chunkNumber}") + @ApiOperation(value = "Upload chunk of resource", notes = "Upload chunk of resource") + public ResponseBodyBean uploadResourceChunk(@RequestParam("file") MultipartFile multipartFile, + @PathVariable Integer chunkNumber) { + return ResponseBodyBean.ofSuccess(this.writeResourceService.uploadResourceChunk(multipartFile, chunkNumber)); + } + + @PutMapping("/merge/chunk") + @ApiOperation(value = "Merge chunk of resource", notes = "Merge chunk of resource") + public ResponseBodyBean mergeResourceChunk() { + return ResponseBodyBean.ofSuccess(); + } } diff --git a/oss-center/src/main/java/com/jmsoftware/maf/osscenter/write/service/WriteResourceService.java b/oss-center/src/main/java/com/jmsoftware/maf/osscenter/write/service/WriteResourceService.java index 9a8d63b4..8495d46e 100644 --- a/oss-center/src/main/java/com/jmsoftware/maf/osscenter/write/service/WriteResourceService.java +++ b/oss-center/src/main/java/com/jmsoftware/maf/osscenter/write/service/WriteResourceService.java @@ -1,11 +1,10 @@ package com.jmsoftware.maf.osscenter.write.service; -import com.jmsoftware.maf.common.exception.BizException; +import org.hibernate.validator.constraints.Range; import org.springframework.validation.annotation.Validated; import org.springframework.web.multipart.MultipartFile; import javax.validation.constraints.NotNull; -import java.io.IOException; /** *

WriteResourceService

@@ -16,13 +15,23 @@ **/ @Validated public interface WriteResourceService { + long MAX_CHUNK_NUMBER = 999; + /** * Upload single resource string. * * @param multipartFile the multipart file * @return the string - * @throws IOException the io exception - * @throws BizException the business exception */ - String uploadSingleResource(@NotNull MultipartFile multipartFile) throws IOException, BizException; + String uploadSingleResource(@NotNull MultipartFile multipartFile); + + /** + * Upload resource chunk string. + * + * @param multipartFile the multipart file + * @param chunkNumber the chunk number + * @return the string + */ + String uploadResourceChunk(@NotNull MultipartFile multipartFile, + @NotNull @Range(max = MAX_CHUNK_NUMBER) Integer chunkNumber); } diff --git a/oss-center/src/main/java/com/jmsoftware/maf/osscenter/write/service/impl/WriteResourceServiceImpl.java b/oss-center/src/main/java/com/jmsoftware/maf/osscenter/write/service/impl/WriteResourceServiceImpl.java index 3c27ff1d..696f09b2 100644 --- a/oss-center/src/main/java/com/jmsoftware/maf/osscenter/write/service/impl/WriteResourceServiceImpl.java +++ b/oss-center/src/main/java/com/jmsoftware/maf/osscenter/write/service/impl/WriteResourceServiceImpl.java @@ -1,14 +1,20 @@ package com.jmsoftware.maf.osscenter.write.service.impl; +import cn.hutool.core.util.NumberUtil; +import cn.hutool.core.util.ObjectUtil; import cn.hutool.core.util.StrUtil; +import cn.hutool.crypto.digest.DigestUtil; import com.jmsoftware.maf.common.exception.BizException; import com.jmsoftware.maf.osscenter.write.service.WriteResourceService; import com.jmsoftware.maf.springcloudstarter.minio.MinioHelper; +import io.minio.StatObjectResponse; import lombok.RequiredArgsConstructor; +import lombok.SneakyThrows; import lombok.extern.slf4j.Slf4j; import lombok.val; import org.apache.tika.Tika; import org.apache.tika.mime.MediaType; +import org.hibernate.validator.constraints.Range; import org.springframework.stereotype.Service; import org.springframework.web.multipart.MultipartFile; @@ -29,20 +35,56 @@ public class WriteResourceServiceImpl implements WriteResourceService { private final MinioHelper minioHelper; @Override - public String uploadSingleResource(@NotNull MultipartFile multipartFile) throws IOException, BizException { + @SneakyThrows + public String uploadSingleResource(@NotNull MultipartFile multipartFile) { + val mediaType = this.parseMediaType(multipartFile); + val bucketMade = this.minioHelper.makeBucket(mediaType.getType()); + val objectWriteResponse = this.minioHelper.put(mediaType.getType(), multipartFile.getOriginalFilename(), + multipartFile); + log.info("Uploaded single resource: {}/{}. bucketMade: {}", objectWriteResponse.bucket(), + objectWriteResponse.object(), bucketMade); + return String.format("%s/%s", objectWriteResponse.bucket(), objectWriteResponse.object()); + } + + @Override + @SneakyThrows + public String uploadResourceChunk(@NotNull MultipartFile multipartFile, + @NotNull @Range(max = MAX_CHUNK_NUMBER) Integer chunkNumber) { + if (StrUtil.isBlank(multipartFile.getOriginalFilename())) { + throw new IllegalArgumentException("File name required"); + } + val mediaType = this.parseMediaType(multipartFile); + val orderedFilename = String.format("%s.chunk%s", multipartFile.getOriginalFilename(), + NumberUtil.decimalFormat("000", chunkNumber)); + StatObjectResponse statObjectResponse = null; + try { + statObjectResponse = this.minioHelper.statObject(mediaType.getType(), orderedFilename); + } catch (Exception e) { + log.error("Exception occurred when looking for object. Exception message: {}", e.getMessage()); + } + if (ObjectUtil.isNotNull(statObjectResponse)) { + val md5Hex = DigestUtil.md5Hex(multipartFile.getInputStream()); + if (StrUtil.equalsIgnoreCase(md5Hex, statObjectResponse.etag())) { + log.warn( + "Found previously uploaded file, skip uploading chunk. Filename: {}, statObjectResponse: {}, " + + "MD5: {}", orderedFilename, statObjectResponse, md5Hex); + return statObjectResponse.etag(); + } + } + val bucketMade = this.minioHelper.makeBucket(mediaType.getType()); + val objectWriteResponse = this.minioHelper.put(mediaType.getType(), orderedFilename, multipartFile); + log.info("Uploaded resource chunk: {}/{}. bucketMade: {}, etag (MD5): {}", objectWriteResponse.bucket(), + objectWriteResponse.object(), bucketMade, objectWriteResponse.etag()); + return objectWriteResponse.etag(); + } + + private MediaType parseMediaType(MultipartFile multipartFile) throws IOException, BizException { val tika = new Tika(); val detectedMediaType = tika.detect(multipartFile.getInputStream()); log.info("Detected media type: {}", detectedMediaType); if (StrUtil.isBlank(detectedMediaType)) { throw new BizException("Media extension detection failed!"); } - val mediaType = MediaType.parse(detectedMediaType); - val mediaBaseType = mediaType.getType(); - val bucketMade = this.minioHelper.makeBucket(mediaBaseType); - val objectWriteResponse = this.minioHelper.put(mediaBaseType, multipartFile.getOriginalFilename(), - multipartFile); - log.info("Uploaded single resource: {}/{}. bucketMade: {}", objectWriteResponse.bucket(), - objectWriteResponse.object(), bucketMade); - return String.format("%s/%s", objectWriteResponse.bucket(), objectWriteResponse.object()); + return MediaType.parse(detectedMediaType); } }