From 74e56d55cad9fea83484551a031b5a84db204ffe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johnny=20Miller=20=28=E9=94=BA=E4=BF=8A=29?= Date: Thu, 12 Aug 2021 22:44:04 +0800 Subject: [PATCH] feat($OSS): new API to merge resource chunk --- .../controller/WriteResourceController.java | 14 ++-- .../entity/MergeResourceChunkPayload.java | 21 ++++++ .../write/entity/ObjectResponse.java | 15 ++++ .../write/service/WriteResourceService.java | 17 ++++- .../impl/WriteResourceServiceImpl.java | 71 +++++++++++++++---- .../springcloudstarter/minio/MinioHelper.java | 17 +++-- 6 files changed, 131 insertions(+), 24 deletions(-) create mode 100644 oss-center/src/main/java/com/jmsoftware/maf/osscenter/write/entity/MergeResourceChunkPayload.java create mode 100644 oss-center/src/main/java/com/jmsoftware/maf/osscenter/write/entity/ObjectResponse.java diff --git a/oss-center/src/main/java/com/jmsoftware/maf/osscenter/write/controller/WriteResourceController.java b/oss-center/src/main/java/com/jmsoftware/maf/osscenter/write/controller/WriteResourceController.java index 99c9af55..5c50331e 100644 --- a/oss-center/src/main/java/com/jmsoftware/maf/osscenter/write/controller/WriteResourceController.java +++ b/oss-center/src/main/java/com/jmsoftware/maf/osscenter/write/controller/WriteResourceController.java @@ -1,6 +1,8 @@ package com.jmsoftware.maf.osscenter.write.controller; import com.jmsoftware.maf.common.bean.ResponseBodyBean; +import com.jmsoftware.maf.osscenter.write.entity.MergeResourceChunkPayload; +import com.jmsoftware.maf.osscenter.write.entity.ObjectResponse; import com.jmsoftware.maf.osscenter.write.service.WriteResourceService; import io.swagger.annotations.Api; import io.swagger.annotations.ApiOperation; @@ -11,6 +13,8 @@ import org.springframework.web.bind.annotation.*; import org.springframework.web.multipart.MultipartFile; +import javax.validation.Valid; + /** *

WriteResourceController

*

@@ -28,7 +32,7 @@ public class WriteResourceController { @PostMapping("/upload/single") @ApiOperation(value = "Upload single resource", notes = "Upload single resource") - public ResponseBodyBean uploadSingleResource(@RequestParam("file") MultipartFile multipartFile) { + public ResponseBodyBean uploadSingleResource(@RequestParam("file") MultipartFile multipartFile) { return ResponseBodyBean.ofSuccess(this.writeResourceService.uploadSingleResource(multipartFile), this.messageSource.getMessage("uploaded", null, LocaleContextHolder.getLocale())); @@ -36,14 +40,14 @@ public ResponseBodyBean uploadSingleResource(@RequestParam("file") Multi @PostMapping("/upload/chunk/{chunkNumber}") @ApiOperation(value = "Upload chunk of resource", notes = "Upload chunk of resource") - public ResponseBodyBean uploadResourceChunk(@RequestParam("file") MultipartFile multipartFile, - @PathVariable Integer chunkNumber) { + public ResponseBodyBean uploadResourceChunk(@RequestParam("file") MultipartFile multipartFile, + @PathVariable Integer chunkNumber) { return ResponseBodyBean.ofSuccess(this.writeResourceService.uploadResourceChunk(multipartFile, chunkNumber)); } @PutMapping("/merge/chunk") @ApiOperation(value = "Merge chunk of resource", notes = "Merge chunk of resource") - public ResponseBodyBean mergeResourceChunk() { - return ResponseBodyBean.ofSuccess(); + public ResponseBodyBean mergeResourceChunk(@Valid @RequestBody MergeResourceChunkPayload payload) { + return ResponseBodyBean.ofSuccess(this.writeResourceService.mergeResourceChunk(payload)); } } diff --git a/oss-center/src/main/java/com/jmsoftware/maf/osscenter/write/entity/MergeResourceChunkPayload.java b/oss-center/src/main/java/com/jmsoftware/maf/osscenter/write/entity/MergeResourceChunkPayload.java new file mode 100644 index 00000000..6417a0ce --- /dev/null +++ b/oss-center/src/main/java/com/jmsoftware/maf/osscenter/write/entity/MergeResourceChunkPayload.java @@ -0,0 +1,21 @@ +package com.jmsoftware.maf.osscenter.write.entity; + +import lombok.Data; + +import javax.validation.Valid; +import javax.validation.constraints.NotBlank; +import javax.validation.constraints.NotEmpty; +import java.util.List; + +/** + * Description: MergeResourceChunkPayload, change description here. + * + * @author Johnny Miller (锺俊), email: johnnysviva@outlook.com, date: 8/12/2021 8:48 PM + **/ +@Data +public class MergeResourceChunkPayload { + @NotBlank + private String bucket; + @NotEmpty + private List<@Valid @NotBlank String> objectList; +} diff --git a/oss-center/src/main/java/com/jmsoftware/maf/osscenter/write/entity/ObjectResponse.java b/oss-center/src/main/java/com/jmsoftware/maf/osscenter/write/entity/ObjectResponse.java new file mode 100644 index 00000000..d15abbf2 --- /dev/null +++ b/oss-center/src/main/java/com/jmsoftware/maf/osscenter/write/entity/ObjectResponse.java @@ -0,0 +1,15 @@ +package com.jmsoftware.maf.osscenter.write.entity; + +import lombok.Data; + +/** + * Description: ObjectResponse, change description here. + * + * @author Johnny Miller (锺俊), email: johnnysviva@outlook.com, date: 8/12/2021 9:27 PM + **/ +@Data +public class ObjectResponse { + private String bucket; + private String object; + private String etag; +} diff --git a/oss-center/src/main/java/com/jmsoftware/maf/osscenter/write/service/WriteResourceService.java b/oss-center/src/main/java/com/jmsoftware/maf/osscenter/write/service/WriteResourceService.java index 8495d46e..d1ed14a4 100644 --- a/oss-center/src/main/java/com/jmsoftware/maf/osscenter/write/service/WriteResourceService.java +++ b/oss-center/src/main/java/com/jmsoftware/maf/osscenter/write/service/WriteResourceService.java @@ -1,9 +1,12 @@ package com.jmsoftware.maf.osscenter.write.service; +import com.jmsoftware.maf.osscenter.write.entity.MergeResourceChunkPayload; +import com.jmsoftware.maf.osscenter.write.entity.ObjectResponse; import org.hibernate.validator.constraints.Range; import org.springframework.validation.annotation.Validated; import org.springframework.web.multipart.MultipartFile; +import javax.validation.Valid; import javax.validation.constraints.NotNull; /** @@ -23,7 +26,7 @@ public interface WriteResourceService { * @param multipartFile the multipart file * @return the string */ - String uploadSingleResource(@NotNull MultipartFile multipartFile); + ObjectResponse uploadSingleResource(@NotNull MultipartFile multipartFile); /** * Upload resource chunk string. @@ -32,6 +35,14 @@ public interface WriteResourceService { * @param chunkNumber the chunk number * @return the string */ - String uploadResourceChunk(@NotNull MultipartFile multipartFile, - @NotNull @Range(max = MAX_CHUNK_NUMBER) Integer chunkNumber); + ObjectResponse uploadResourceChunk(@NotNull MultipartFile multipartFile, + @NotNull @Range(max = MAX_CHUNK_NUMBER) Integer chunkNumber); + + /** + * Merge resource chunk string. + * + * @param payload the payload + * @return the string + */ + ObjectResponse mergeResourceChunk(@Valid @NotNull MergeResourceChunkPayload payload); } diff --git a/oss-center/src/main/java/com/jmsoftware/maf/osscenter/write/service/impl/WriteResourceServiceImpl.java b/oss-center/src/main/java/com/jmsoftware/maf/osscenter/write/service/impl/WriteResourceServiceImpl.java index 696f09b2..a09663b8 100644 --- a/oss-center/src/main/java/com/jmsoftware/maf/osscenter/write/service/impl/WriteResourceServiceImpl.java +++ b/oss-center/src/main/java/com/jmsoftware/maf/osscenter/write/service/impl/WriteResourceServiceImpl.java @@ -1,12 +1,16 @@ package com.jmsoftware.maf.osscenter.write.service.impl; +import cn.hutool.core.collection.CollUtil; import cn.hutool.core.util.NumberUtil; import cn.hutool.core.util.ObjectUtil; import cn.hutool.core.util.StrUtil; import cn.hutool.crypto.digest.DigestUtil; import com.jmsoftware.maf.common.exception.BizException; +import com.jmsoftware.maf.osscenter.write.entity.MergeResourceChunkPayload; +import com.jmsoftware.maf.osscenter.write.entity.ObjectResponse; import com.jmsoftware.maf.osscenter.write.service.WriteResourceService; import com.jmsoftware.maf.springcloudstarter.minio.MinioHelper; +import io.minio.ComposeSource; import io.minio.StatObjectResponse; import lombok.RequiredArgsConstructor; import lombok.SneakyThrows; @@ -18,8 +22,12 @@ import org.springframework.stereotype.Service; import org.springframework.web.multipart.MultipartFile; +import javax.validation.Valid; import javax.validation.constraints.NotNull; import java.io.IOException; +import java.util.HashMap; +import java.util.List; +import java.util.stream.Collectors; /** *

WriteResourceServiceImpl

@@ -36,20 +44,23 @@ public class WriteResourceServiceImpl implements WriteResourceService { @Override @SneakyThrows - public String uploadSingleResource(@NotNull MultipartFile multipartFile) { + public ObjectResponse uploadSingleResource(@NotNull MultipartFile multipartFile) { val mediaType = this.parseMediaType(multipartFile); - val bucketMade = this.minioHelper.makeBucket(mediaType.getType()); + this.minioHelper.makeBucket(mediaType.getType()); val objectWriteResponse = this.minioHelper.put(mediaType.getType(), multipartFile.getOriginalFilename(), multipartFile); - log.info("Uploaded single resource: {}/{}. bucketMade: {}", objectWriteResponse.bucket(), - objectWriteResponse.object(), bucketMade); - return String.format("%s/%s", objectWriteResponse.bucket(), objectWriteResponse.object()); + val objectResponse = new ObjectResponse(); + objectResponse.setBucket(objectWriteResponse.bucket()); + objectResponse.setObject(objectWriteResponse.object()); + objectResponse.setEtag(objectWriteResponse.etag()); + log.info("Uploaded single resource. {}", objectResponse); + return objectResponse; } @Override @SneakyThrows - public String uploadResourceChunk(@NotNull MultipartFile multipartFile, - @NotNull @Range(max = MAX_CHUNK_NUMBER) Integer chunkNumber) { + public ObjectResponse uploadResourceChunk(@NotNull MultipartFile multipartFile, + @NotNull @Range(max = MAX_CHUNK_NUMBER) Integer chunkNumber) { if (StrUtil.isBlank(multipartFile.getOriginalFilename())) { throw new IllegalArgumentException("File name required"); } @@ -62,20 +73,56 @@ public String uploadResourceChunk(@NotNull MultipartFile multipartFile, } catch (Exception e) { log.error("Exception occurred when looking for object. Exception message: {}", e.getMessage()); } + val objectResponse = new ObjectResponse(); + objectResponse.setBucket(mediaType.getType()); + objectResponse.setObject(orderedFilename); if (ObjectUtil.isNotNull(statObjectResponse)) { val md5Hex = DigestUtil.md5Hex(multipartFile.getInputStream()); if (StrUtil.equalsIgnoreCase(md5Hex, statObjectResponse.etag())) { log.warn( "Found previously uploaded file, skip uploading chunk. Filename: {}, statObjectResponse: {}, " + "MD5: {}", orderedFilename, statObjectResponse, md5Hex); - return statObjectResponse.etag(); + objectResponse.setEtag(statObjectResponse.etag()); + return objectResponse; } } - val bucketMade = this.minioHelper.makeBucket(mediaType.getType()); + this.minioHelper.makeBucket(mediaType.getType()); val objectWriteResponse = this.minioHelper.put(mediaType.getType(), orderedFilename, multipartFile); - log.info("Uploaded resource chunk: {}/{}. bucketMade: {}, etag (MD5): {}", objectWriteResponse.bucket(), - objectWriteResponse.object(), bucketMade, objectWriteResponse.etag()); - return objectWriteResponse.etag(); + log.info("Uploaded resource chunk. {}", objectResponse); + objectResponse.setEtag(objectWriteResponse.etag()); + return objectResponse; + } + + @Override + public ObjectResponse mergeResourceChunk(@Valid @NotNull MergeResourceChunkPayload payload) { + val objectName = this.validateObject(payload.getObjectList()); + val sources = payload.getObjectList() + .stream() + .map(object -> ComposeSource.builder().bucket(payload.getBucket()).object(object).build()) + .collect(Collectors.toList()); + val statObjectResponse = this.minioHelper.statObject(payload.getBucket(), + CollUtil.getFirst(payload.getObjectList())); + val headers = new HashMap(4); + headers.put("Content-Type", statObjectResponse.contentType()); + val objectWriteResponse = this.minioHelper.composeObject(payload.getBucket(), objectName, sources, headers); + val objectResponse = new ObjectResponse(); + objectResponse.setBucket(objectWriteResponse.bucket()); + objectResponse.setObject(objectWriteResponse.object()); + objectResponse.setEtag(objectWriteResponse.etag()); + log.info("Merged resource chunks. {}", objectResponse); + return objectResponse; + } + + private String validateObject(List objectList) { + val objectNameSet = objectList.stream().map(object -> { + val lastIndexOfDot = StrUtil.lastIndexOfIgnoreCase(object, "."); + return StrUtil.subPre(object, lastIndexOfDot); + }).collect(Collectors.toSet()); + if (CollUtil.size(objectNameSet) != 1) { + log.error("Object list is not valid! {}", objectNameSet); + throw new IllegalArgumentException("Object list is not valid"); + } + return objectNameSet.iterator().next(); } private MediaType parseMediaType(MultipartFile multipartFile) throws IOException, BizException { diff --git a/spring-cloud-starter/src/main/java/com/jmsoftware/maf/springcloudstarter/minio/MinioHelper.java b/spring-cloud-starter/src/main/java/com/jmsoftware/maf/springcloudstarter/minio/MinioHelper.java index 438e11e0..cc6e0e26 100644 --- a/spring-cloud-starter/src/main/java/com/jmsoftware/maf/springcloudstarter/minio/MinioHelper.java +++ b/spring-cloud-starter/src/main/java/com/jmsoftware/maf/springcloudstarter/minio/MinioHelper.java @@ -10,17 +10,16 @@ import lombok.SneakyThrows; import lombok.extern.slf4j.Slf4j; import lombok.val; +import org.springframework.lang.Nullable; import org.springframework.stereotype.Component; import org.springframework.validation.annotation.Validated; import org.springframework.web.multipart.MultipartFile; -import javax.validation.constraints.Max; -import javax.validation.constraints.Min; -import javax.validation.constraints.NotBlank; -import javax.validation.constraints.NotNull; +import javax.validation.constraints.*; import java.io.InputStream; import java.util.LinkedList; import java.util.List; +import java.util.Map; import java.util.stream.Collectors; /** @@ -216,4 +215,14 @@ public String getPresignedObjectUrl(@NotBlank String bucket, @NotBlank String ob return this.minioClient.getPresignedObjectUrl( GetPresignedObjectUrlArgs.builder().bucket(bucket).object(object).build()); } + + @SneakyThrows + public ObjectWriteResponse composeObject(@NotBlank String bucket, @NotBlank String object, + @NotEmpty List sources,@Nullable Map headers) { + if (!this.bucketExists(bucket)) { + return null; + } + return this.minioClient.composeObject( + ComposeObjectArgs.builder().bucket(bucket).object(object).sources(sources).headers(headers).build()); + } }