[DO NOT MERGE] Backport ETag improvements and bug fixes from master to ozone-1.4

hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java

@@ -598,4 +598,9 @@ private OzoneConsts() {
    */
   public static final String COMPACTION_LOG_TABLE =
       "compactionLogTable";
+
+  /**
+   * S3G multipart upload request's ETag header key.
+   */
+  public static final String ETAG = "ETag";
 }

hadoop-hdds/docs/content/feature/S3-Tenant-Commands.md

@@ -432,7 +432,7 @@ bash-4.2$ aws s3api --endpoint-url http://s3g:9878 list-objects --bucket bucket-
         {
             "Key": "file1",
             "LastModified": "2022-02-16T00:10:00.000Z",
-            "ETag": "2022-02-16T00:10:00.000Z",
+            "ETag": "e99f93dedfe22e9a133dc3c634f14634",
             "Size": 3811,
             "StorageClass": "STANDARD"
         }

hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneMultipartUploadPartListParts.java

@@ -98,18 +98,21 @@ public ReplicationConfig getReplicationConfig() {
   /**
    * Class that represents each Part information of a multipart upload part.
    */
-  public static class PartInfo {
+  public static final class PartInfo {
 
-    private int partNumber;
-    private String partName;
-    private long modificationTime;
-    private long size;
+    private final int partNumber;
+    private final String partName;
+    private final long modificationTime;
+    private final long size;
+    private final String eTag;
 
-    public PartInfo(int number, String name, long time, long size) {
+    public PartInfo(int number, String name, long time, long size,
+                    String eTag) {
       this.partNumber = number;
       this.partName = name;
       this.modificationTime = time;
       this.size = size;
+      this.eTag = eTag;
     }
 
     public int getPartNumber() {
@@ -127,5 +130,9 @@ public long getModificationTime() {
     public long getSize() {
       return size;
     }
+
+    public String getETag() {
+      return eTag;
+    }
   }
 }

hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java

@@ -1948,7 +1948,8 @@ public OzoneMultipartUploadPartListParts listParts(String volumeName,
       ozoneMultipartUploadPartListParts.addPart(
           new OzoneMultipartUploadPartListParts.PartInfo(
               omPartInfo.getPartNumber(), omPartInfo.getPartName(),
-              omPartInfo.getModificationTime(), omPartInfo.getSize()));
+              omPartInfo.getModificationTime(), omPartInfo.getSize(),
+              omPartInfo.getETag()));
     }
     return ozoneMultipartUploadPartListParts;
 

hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmMultipartCommitUploadPartInfo.java

@@ -24,8 +24,15 @@ public class OmMultipartCommitUploadPartInfo {
 
   private final String partName;
 
-  public OmMultipartCommitUploadPartInfo(String name) {
-    this.partName = name;
+  private final String eTag;
+
+  public OmMultipartCommitUploadPartInfo(String partName, String eTag) {
+    this.partName = partName;
+    this.eTag = eTag;
+  }
+
+  public String getETag() {
+    return eTag;
   }
 
   public String getPartName() {

hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmMultipartUploadCompleteList.java

@@ -56,8 +56,9 @@ public Map<Integer, String> getMultipartMap() {
    */
   public List<Part> getPartsList() {
     List<Part> partList = new ArrayList<>();
-    multipartMap.forEach((partNumber, partName) -> partList.add(Part
-        .newBuilder().setPartName(partName).setPartNumber(partNumber).build()));
+    multipartMap.forEach((partNumber, eTag) -> partList.add(Part
+        // set partName equal to eTag for back compatibility (partName is a required property)
+        .newBuilder().setPartName(eTag).setETag(eTag).setPartNumber(partNumber).build()));
     return partList;
   }
 }

hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmMultipartUploadListParts.java

@@ -79,6 +79,7 @@ public void addPartList(List<OmPartInfo> partInfos) {
   public void addProtoPartList(List<PartInfo> partInfos) {
     partInfos.forEach(partInfo -> partInfoList.add(new OmPartInfo(
         partInfo.getPartNumber(), partInfo.getPartName(),
-        partInfo.getModificationTime(), partInfo.getSize())));
+        partInfo.getModificationTime(), partInfo.getSize(),
+        partInfo.getETag())));
   }
 }

hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmPartInfo.java

@@ -23,17 +23,20 @@
 /**
  * Class that defines information about each part of a multipart upload key.
  */
-public class OmPartInfo {
-  private int partNumber;
-  private String partName;
-  private long modificationTime;
-  private long size;
+public final class OmPartInfo {
+  private final int partNumber;
+  private final String partName;
+  private final long modificationTime;
+  private final long size;
+  private final String eTag;
 
-  public OmPartInfo(int number, String name, long time, long size) {
+  public OmPartInfo(int number, String name, long time, long size,
+                    String eTag) {
     this.partNumber = number;
     this.partName = name;
     this.modificationTime = time;
     this.size = size;
+    this.eTag = eTag;
   }
 
   public int getPartNumber() {
@@ -52,9 +55,19 @@ public long getSize() {
     return size;
   }
 
+  public String getETag() {
+    return eTag;
+  }
+
   public PartInfo getProto() {
-    return PartInfo.newBuilder().setPartNumber(partNumber).setPartName(partName)
-       .setModificationTime(modificationTime)
-       .setSize(size).build();
+    PartInfo.Builder builder = PartInfo.newBuilder()
+        .setPartNumber(partNumber)
+        .setPartName(partName)
+        .setModificationTime(modificationTime)
+        .setSize(size);
+    if (eTag != null) {
+      builder.setETag(eTag);
+    }
+    return builder.build();
   }
 }

hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OzoneManagerProtocolClientSideTranslatorPB.java

@@ -1628,7 +1628,8 @@ public OmMultipartCommitUploadPartInfo commitMultipartUploadPart(
         .getCommitMultiPartUploadResponse();
 
     OmMultipartCommitUploadPartInfo info = new
-        OmMultipartCommitUploadPartInfo(response.getPartName());
+        OmMultipartCommitUploadPartInfo(response.getPartName(),
+          response.getETag());
     return info;
   }
 

hadoop-ozone/dist/src/main/smoketest/s3/MultipartUpload.robot

@@ -94,21 +94,28 @@ Test Multipart Upload Complete
                         Should contain          ${result}    UploadId
 
 #upload parts
-    Run Keyword         Create Random file      5
-    ${result} =         Execute AWSS3APICli     upload-part --bucket ${BUCKET} --key ${PREFIX}/multipartKey1 --part-number 1 --body /tmp/part1 --upload-id ${uploadID}
-    ${eTag1} =          Execute and checkrc     echo '${result}' | jq -r '.ETag'   0
-                        Should contain          ${result}    ETag
-
-                        Execute                 echo "Part2" > /tmp/part2
-    ${result} =         Execute AWSS3APICli     upload-part --bucket ${BUCKET} --key ${PREFIX}/multipartKey1 --part-number 2 --body /tmp/part2 --upload-id ${uploadID}
-    ${eTag2} =          Execute and checkrc     echo '${result}' | jq -r '.ETag'   0
-                        Should contain          ${result}    ETag
+    Run Keyword         Create Random file            5
+    ${result} =         Execute AWSS3APICli           upload-part --bucket ${BUCKET} --key ${PREFIX}/multipartKey1 --part-number 1 --body /tmp/part1 --upload-id ${uploadID}
+    ${eTag1} =          Execute and checkrc           echo '${result}' | jq -r '.ETag'   0
+                        Should contain                ${result}    ETag
+    ${part1Md5Sum} =    Execute                       md5sum /tmp/part1 | awk '{print $1}'
+                        Should Be Equal As Strings    ${eTag1}  ${part1Md5Sum}
+
+                        Execute                       echo "Part2" > /tmp/part2
+    ${result} =         Execute AWSS3APICli           upload-part --bucket ${BUCKET} --key ${PREFIX}/multipartKey1 --part-number 2 --body /tmp/part2 --upload-id ${uploadID}
+    ${eTag2} =          Execute and checkrc           echo '${result}' | jq -r '.ETag'   0
+                        Should contain                ${result}    ETag
+    ${part2Md5Sum} =    Execute                       md5sum /tmp/part2 | awk '{print $1}'
+                        Should Be Equal As Strings    ${eTag2}  ${part2Md5Sum}
 
 #complete multipart upload
-    ${result} =         Execute AWSS3APICli     complete-multipart-upload --upload-id ${uploadID} --bucket ${BUCKET} --key ${PREFIX}/multipartKey1 --multipart-upload 'Parts=[{ETag=${eTag1},PartNumber=1},{ETag=${eTag2},PartNumber=2}]'
-                        Should contain          ${result}    ${BUCKET}
-                        Should contain          ${result}    ${PREFIX}/multipartKey1
-                        Should contain          ${result}    ETag
+    ${result} =                 Execute AWSS3APICli           complete-multipart-upload --upload-id ${uploadID} --bucket ${BUCKET} --key ${PREFIX}/multipartKey1 --multipart-upload 'Parts=[{ETag=${eTag1},PartNumber=1},{ETag=${eTag2},PartNumber=2}]'
+                                Should contain                ${result}    ${BUCKET}
+                                Should contain                ${result}    ${PREFIX}/multipartKey1
+    ${resultETag} =             Execute and checkrc           echo '${result}' | jq -r '.ETag'   0
+    ${expectedResultETag} =     Execute                       echo -n ${eTag1}${eTag2} | md5sum | awk '{print $1}'
+                                Should contain                ${result}    ETag
+                                Should Be Equal As Strings    ${resultETag}     "${expectedResultETag}-2"
 
 #read file and check the key
     ${result} =                 Execute AWSS3ApiCli        get-object --bucket ${BUCKET} --key ${PREFIX}/multipartKey1 /tmp/${PREFIX}-multipartKey1.result

hadoop-ozone/dist/src/main/smoketest/s3/objectcopy.robot

@@ -37,15 +37,26 @@ Create Dest Bucket
 Copy Object Happy Scenario
     Run Keyword if    '${DESTBUCKET}' == 'generated1'    Create Dest Bucket
                         Execute                    date > /tmp/copyfile
+    ${file_checksum} =  Execute                    md5sum /tmp/copyfile | awk '{print $1}'
+
     ${result} =         Execute AWSS3ApiCli        put-object --bucket ${BUCKET} --key ${PREFIX}/copyobject/key=value/f1 --body /tmp/copyfile
+    ${eTag} =           Execute and checkrc        echo '${result}' | jq -r '.ETag'  0
+                        Should Be Equal            ${eTag}           \"${file_checksum}\"
+
     ${result} =         Execute AWSS3ApiCli        list-objects --bucket ${BUCKET} --prefix ${PREFIX}/copyobject/key=value/
                         Should contain             ${result}         f1
 
     ${result} =         Execute AWSS3ApiCli        copy-object --bucket ${DESTBUCKET} --key ${PREFIX}/copyobject/key=value/f1 --copy-source ${BUCKET}/${PREFIX}/copyobject/key=value/f1
+    ${eTag} =           Execute and checkrc        echo '${result}' | jq -r '.CopyObjectResult.ETag'  0
+                        Should Be Equal            ${eTag}           \"${file_checksum}\"
+
     ${result} =         Execute AWSS3ApiCli        list-objects --bucket ${DESTBUCKET} --prefix ${PREFIX}/copyobject/key=value/
                         Should contain             ${result}         f1
     #copying again will not throw error
     ${result} =         Execute AWSS3ApiCli        copy-object --bucket ${DESTBUCKET} --key ${PREFIX}/copyobject/key=value/f1 --copy-source ${BUCKET}/${PREFIX}/copyobject/key=value/f1
+    ${eTag} =           Execute and checkrc        echo '${result}' | jq -r '.CopyObjectResult.ETag'  0
+                        Should Be Equal            ${eTag}           \"${file_checksum}\"
+
     ${result} =         Execute AWSS3ApiCli        list-objects --bucket ${DESTBUCKET} --prefix ${PREFIX}/copyobject/key=value/
                         Should contain             ${result}         f1
 
@@ -56,8 +67,11 @@ Copy Object Where Bucket is not available
                         Should contain             ${result}        NoSuchBucket
 
 Copy Object Where both source and dest are same with change to storageclass
+     ${file_checksum} =  Execute                    md5sum /tmp/copyfile | awk '{print $1}'
      ${result} =         Execute AWSS3APICli        copy-object --storage-class REDUCED_REDUNDANCY --bucket ${DESTBUCKET} --key ${PREFIX}/copyobject/key=value/f1 --copy-source ${DESTBUCKET}/${PREFIX}/copyobject/key=value/f1
                          Should contain             ${result}        ETag
+     ${eTag} =           Execute and checkrc        echo '${result}' | jq -r '.CopyObjectResult.ETag'  0
+                         Should Be Equal            ${eTag}           \"${file_checksum}\"
 
 Copy Object Where Key not available
     ${result} =         Execute AWSS3APICli and checkrc        copy-object --bucket ${DESTBUCKET} --key ${PREFIX}/copyobject/key=value/f1 --copy-source ${BUCKET}/nonnonexistentkey       255

hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFSWithObjectStoreCreate.java

@@ -18,6 +18,7 @@
 
 package org.apache.hadoop.fs.ozone;
 
+import javax.xml.bind.DatatypeConverter;
 import org.apache.commons.lang3.RandomStringUtils;
 import org.apache.hadoop.fs.FSDataInputStream;
 import org.apache.hadoop.fs.FileAlreadyExistsException;
@@ -49,6 +50,7 @@
 import java.io.FileNotFoundException;
 import java.io.IOException;
 import java.net.URI;
+import java.security.MessageDigest;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.HashMap;
@@ -58,6 +60,8 @@
 import java.util.Map;
 
 import static java.nio.charset.StandardCharsets.UTF_8;
+import static org.apache.hadoop.ozone.OzoneConsts.ETAG;
+import static org.apache.hadoop.ozone.OzoneConsts.MD5_HASH;
 import static org.apache.hadoop.ozone.OzoneConsts.OZONE_URI_SCHEME;
 import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.NOT_A_FILE;
 import static org.junit.jupiter.api.Assertions.assertEquals;
@@ -302,10 +306,13 @@ public void testMPUFailDuetoDirectoryCreationBeforeComplete()
 
     // This should succeed, as we check during creation of part or during
     // complete MPU.
+    ozoneOutputStream.getMetadata().put(ETAG,
+        DatatypeConverter.printHexBinary(MessageDigest.getInstance(MD5_HASH)
+            .digest(b)).toLowerCase());
     ozoneOutputStream.close();
 
     Map<Integer, String> partsMap = new HashMap<>();
-    partsMap.put(1, ozoneOutputStream.getCommitUploadPartInfo().getPartName());
+    partsMap.put(1, ozoneOutputStream.getCommitUploadPartInfo().getETag());
 
     // Should fail, as we have directory with same name.
     try {

hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestMultipartObjectGet.java

@@ -146,11 +146,11 @@ private CompleteMultipartUploadRequest.Part uploadPart(String uploadID,
     Response response = REST.put(BUCKET, KEY, content.length(),
         partNumber, uploadID, body);
     assertEquals(200, response.getStatus());
-    assertNotNull(response.getHeaderString("ETag"));
+    assertNotNull(response.getHeaderString(OzoneConsts.ETAG));
 
     CompleteMultipartUploadRequest.Part
         part = new CompleteMultipartUploadRequest.Part();
-    part.seteTag(response.getHeaderString("ETag"));
+    part.setETag(response.getHeaderString(OzoneConsts.ETAG));
     part.setPartNumber(partNumber);
     return part;
   }

hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneAtRestEncryption.java

@@ -22,6 +22,7 @@
 import java.net.URI;
 import java.nio.ByteBuffer;
 import java.nio.charset.StandardCharsets;
+import java.security.MessageDigest;
 import java.security.NoSuchAlgorithmException;
 import java.time.Instant;
 import java.util.ArrayList;
@@ -35,6 +36,7 @@
 
 import com.google.common.cache.Cache;
 import org.apache.hadoop.conf.StorageUnit;
+import javax.xml.bind.DatatypeConverter;
 import org.apache.hadoop.crypto.key.KeyProvider;
 import org.apache.hadoop.crypto.key.kms.KMSClientProvider;
 import org.apache.hadoop.crypto.key.kms.server.MiniKMS;
@@ -117,6 +119,7 @@ class TestOzoneAtRestEncryption {
   private static final int DEFAULT_CRYPTO_BUFFER_SIZE = 8 * 1024; // 8KB
   // (this is the default Crypto Buffer size as determined by the config
   // hadoop.security.crypto.buffer.size)
+  private static MessageDigest eTagProvider;
 
   @BeforeAll
   static void init() throws Exception {
@@ -166,6 +169,7 @@ static void init() throws Exception {
 
     // create test key
     createKey(TEST_KEY, cluster.getOzoneManager().getKmsProvider(), conf);
+    eTagProvider = MessageDigest.getInstance(OzoneConsts.MD5_HASH);
   }
 
   @AfterAll
@@ -647,29 +651,35 @@ private String uploadStreamPart(OzoneBucket bucket, String keyName,
 
     ByteBuffer dataBuffer = ByteBuffer.wrap(data);
     multipartStreamKey.write(dataBuffer, 0, length);
+    multipartStreamKey.getMetadata().put(OzoneConsts.ETAG,
+        DatatypeConverter.printHexBinary(eTagProvider.digest(data))
+            .toLowerCase());
     multipartStreamKey.close();
 
     OmMultipartCommitUploadPartInfo omMultipartCommitUploadPartInfo =
         multipartStreamKey.getCommitUploadPartInfo();
 
     assertNotNull(omMultipartCommitUploadPartInfo);
-    assertNotNull(omMultipartCommitUploadPartInfo.getPartName());
-    return omMultipartCommitUploadPartInfo.getPartName();
+    assertNotNull(omMultipartCommitUploadPartInfo.getETag());
+    return omMultipartCommitUploadPartInfo.getETag();
   }
 
   private String uploadPart(OzoneBucket bucket, String keyName,
       String uploadID, int partNumber, byte[] data) throws Exception {
     OzoneOutputStream ozoneOutputStream = bucket.createMultipartKey(keyName,
         data.length, partNumber, uploadID);
     ozoneOutputStream.write(data, 0, data.length);
+    ozoneOutputStream.getMetadata().put(OzoneConsts.ETAG,
+        DatatypeConverter.printHexBinary(eTagProvider.digest(data))
+            .toLowerCase());
     ozoneOutputStream.close();
 
     OmMultipartCommitUploadPartInfo omMultipartCommitUploadPartInfo =
         ozoneOutputStream.getCommitUploadPartInfo();
 
     assertNotNull(omMultipartCommitUploadPartInfo);
-    assertNotNull(omMultipartCommitUploadPartInfo.getPartName());
-    return omMultipartCommitUploadPartInfo.getPartName();
+    assertNotNull(omMultipartCommitUploadPartInfo.getETag());
+    return omMultipartCommitUploadPartInfo.getETag();
   }
 
   private void completeMultipartUpload(OzoneBucket bucket, String keyName,