diff --git a/oss-center/src/main/java/com/jmsoftware/maf/osscenter/write/controller/WriteResourceController.java b/oss-center/src/main/java/com/jmsoftware/maf/osscenter/write/controller/WriteResourceController.java index 5c50331e..fa17128e 100644 --- a/oss-center/src/main/java/com/jmsoftware/maf/osscenter/write/controller/WriteResourceController.java +++ b/oss-center/src/main/java/com/jmsoftware/maf/osscenter/write/controller/WriteResourceController.java @@ -41,8 +41,10 @@ public ResponseBodyBean uploadSingleResource(@RequestParam("file @PostMapping("/upload/chunk/{chunkNumber}") @ApiOperation(value = "Upload chunk of resource", notes = "Upload chunk of resource") public ResponseBodyBean uploadResourceChunk(@RequestParam("file") MultipartFile multipartFile, + @RequestParam(required = false) String bucket, @PathVariable Integer chunkNumber) { - return ResponseBodyBean.ofSuccess(this.writeResourceService.uploadResourceChunk(multipartFile, chunkNumber)); + return ResponseBodyBean.ofSuccess( + this.writeResourceService.uploadResourceChunk(multipartFile, bucket, chunkNumber)); } @PutMapping("/merge/chunk") diff --git a/oss-center/src/main/java/com/jmsoftware/maf/osscenter/write/service/WriteResourceService.java b/oss-center/src/main/java/com/jmsoftware/maf/osscenter/write/service/WriteResourceService.java index d1ed14a4..a970872a 100644 --- a/oss-center/src/main/java/com/jmsoftware/maf/osscenter/write/service/WriteResourceService.java +++ b/oss-center/src/main/java/com/jmsoftware/maf/osscenter/write/service/WriteResourceService.java @@ -3,6 +3,7 @@ import com.jmsoftware.maf.osscenter.write.entity.MergeResourceChunkPayload; import com.jmsoftware.maf.osscenter.write.entity.ObjectResponse; import org.hibernate.validator.constraints.Range; +import org.springframework.lang.Nullable; import org.springframework.validation.annotation.Validated; import org.springframework.web.multipart.MultipartFile; @@ -32,10 +33,12 @@ public interface WriteResourceService { * Upload resource chunk string. * * @param multipartFile the multipart file + * @param bucket the bucket * @param chunkNumber the chunk number * @return the string */ ObjectResponse uploadResourceChunk(@NotNull MultipartFile multipartFile, + @Nullable String bucket, @NotNull @Range(max = MAX_CHUNK_NUMBER) Integer chunkNumber); /** diff --git a/oss-center/src/main/java/com/jmsoftware/maf/osscenter/write/service/impl/WriteResourceServiceImpl.java b/oss-center/src/main/java/com/jmsoftware/maf/osscenter/write/service/impl/WriteResourceServiceImpl.java index a09663b8..5dd8b6de 100644 --- a/oss-center/src/main/java/com/jmsoftware/maf/osscenter/write/service/impl/WriteResourceServiceImpl.java +++ b/oss-center/src/main/java/com/jmsoftware/maf/osscenter/write/service/impl/WriteResourceServiceImpl.java @@ -2,16 +2,12 @@ import cn.hutool.core.collection.CollUtil; import cn.hutool.core.util.NumberUtil; -import cn.hutool.core.util.ObjectUtil; import cn.hutool.core.util.StrUtil; -import cn.hutool.crypto.digest.DigestUtil; -import com.jmsoftware.maf.common.exception.BizException; import com.jmsoftware.maf.osscenter.write.entity.MergeResourceChunkPayload; import com.jmsoftware.maf.osscenter.write.entity.ObjectResponse; import com.jmsoftware.maf.osscenter.write.service.WriteResourceService; import com.jmsoftware.maf.springcloudstarter.minio.MinioHelper; import io.minio.ComposeSource; -import io.minio.StatObjectResponse; import lombok.RequiredArgsConstructor; import lombok.SneakyThrows; import lombok.extern.slf4j.Slf4j; @@ -19,6 +15,7 @@ import org.apache.tika.Tika; import org.apache.tika.mime.MediaType; import org.hibernate.validator.constraints.Range; +import org.springframework.lang.Nullable; import org.springframework.stereotype.Service; import org.springframework.web.multipart.MultipartFile; @@ -27,6 +24,7 @@ import java.io.IOException; import java.util.HashMap; import java.util.List; +import java.util.Objects; import java.util.stream.Collectors; /** @@ -57,39 +55,56 @@ public ObjectResponse uploadSingleResource(@NotNull MultipartFile multipartFile) return objectResponse; } + /** + * {@inheritDoc} + *

+ *

What is an S3 ETag?

+ *

According to + * Amazon:

+ *

The ETag may or may not be an MD5 digest of the object data

+ *
+ *

Basically, if the object was uploaded with a single PUT operation and doesnt use Customer Managed or KMS + * keys for encryption then the resulting ETag is just the MD5 hexdigest of the object.

+ *

However, more importantly:

+ *

If an object is created by either the Multipart Upload or Part Copy operation, the ETag is not + * an MD5 digest, regardless of the method of encryption.

+ *
+ *

Well if it’s not an MD5 digest then what is it?!

+ *

For multipart uploads the ETag is the MD5 hexdigest of each part’s MD5 digest concatenated + * together, followed by the number of parts separated by a dash.

+ *
+ *

E.g. for a two part object the ETag may look something like this:

+ *

d41d8cd98f00b204e9800998ecf8427e-2

+ *
+ *

Which can be represented by:

+ *

hexmd5( md5( part1 ) + md5( part2 ) )-{ number of parts }

+ *
+ * + * @see All about AWS S3 ETags + */ @Override @SneakyThrows public ObjectResponse uploadResourceChunk(@NotNull MultipartFile multipartFile, + @Nullable String bucket, @NotNull @Range(max = MAX_CHUNK_NUMBER) Integer chunkNumber) { if (StrUtil.isBlank(multipartFile.getOriginalFilename())) { throw new IllegalArgumentException("File name required"); } - val mediaType = this.parseMediaType(multipartFile); + MediaType mediaType = null; + if (StrUtil.isBlank(bucket)) { + mediaType = this.parseMediaType(multipartFile); + } + // bucketName is either mediaType of given 'bucket' + val bucketName = StrUtil.isBlank(bucket) ? Objects.requireNonNull(mediaType).getType() : bucket; val orderedFilename = String.format("%s.chunk%s", multipartFile.getOriginalFilename(), NumberUtil.decimalFormat("000", chunkNumber)); - StatObjectResponse statObjectResponse = null; - try { - statObjectResponse = this.minioHelper.statObject(mediaType.getType(), orderedFilename); - } catch (Exception e) { - log.error("Exception occurred when looking for object. Exception message: {}", e.getMessage()); - } val objectResponse = new ObjectResponse(); - objectResponse.setBucket(mediaType.getType()); + objectResponse.setBucket(bucketName); objectResponse.setObject(orderedFilename); - if (ObjectUtil.isNotNull(statObjectResponse)) { - val md5Hex = DigestUtil.md5Hex(multipartFile.getInputStream()); - if (StrUtil.equalsIgnoreCase(md5Hex, statObjectResponse.etag())) { - log.warn( - "Found previously uploaded file, skip uploading chunk. Filename: {}, statObjectResponse: {}, " + - "MD5: {}", orderedFilename, statObjectResponse, md5Hex); - objectResponse.setEtag(statObjectResponse.etag()); - return objectResponse; - } - } - this.minioHelper.makeBucket(mediaType.getType()); - val objectWriteResponse = this.minioHelper.put(mediaType.getType(), orderedFilename, multipartFile); - log.info("Uploaded resource chunk. {}", objectResponse); + this.minioHelper.makeBucket(bucketName); + val objectWriteResponse = this.minioHelper.put(bucketName, orderedFilename, multipartFile); objectResponse.setEtag(objectWriteResponse.etag()); + log.info("Uploaded resource chunk. {}", objectResponse); return objectResponse; } @@ -125,12 +140,12 @@ private String validateObject(List objectList) { return objectNameSet.iterator().next(); } - private MediaType parseMediaType(MultipartFile multipartFile) throws IOException, BizException { + private MediaType parseMediaType(MultipartFile multipartFile) throws IOException { val tika = new Tika(); val detectedMediaType = tika.detect(multipartFile.getInputStream()); log.info("Detected media type: {}", detectedMediaType); if (StrUtil.isBlank(detectedMediaType)) { - throw new BizException("Media extension detection failed!"); + throw new IllegalStateException("Media extension detection failed!"); } return MediaType.parse(detectedMediaType); }