From 322035f991aa35859b200b31b2fda380fc3f36fc Mon Sep 17 00:00:00 2001 From: saketa Date: Thu, 4 Apr 2024 22:47:39 -0700 Subject: [PATCH 1/5] HDDS-10630. Adding missing parent and prefix directories that were deleted between commit and complete mpu uploads. --- .../S3MultipartUploadCompleteRequest.java | 76 +++++++++++++++++- ...MultipartUploadCompleteRequestWithFSO.java | 79 +++++++++++++++++++ 2 files changed, 151 insertions(+), 4 deletions(-) diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3MultipartUploadCompleteRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3MultipartUploadCompleteRequest.java index 83b46de7fd1..7fe6b42aa3b 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3MultipartUploadCompleteRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3MultipartUploadCompleteRequest.java @@ -23,6 +23,7 @@ import java.io.IOException; import java.nio.file.InvalidPathException; +import java.nio.file.Paths; import java.util.ArrayList; import java.util.Collections; import java.util.List; @@ -31,6 +32,10 @@ import java.util.function.BiFunction; import org.apache.hadoop.hdds.client.ReplicationConfig; +import org.apache.hadoop.ozone.om.OzoneConfigUtil; +import org.apache.hadoop.ozone.om.request.file.OMDirectoryCreateRequestWithFSO; +import org.apache.hadoop.ozone.om.request.file.OMFileRequest; +import org.apache.hadoop.ozone.protocolPB.OMPBHelper; import org.apache.ratis.server.protocol.TermIndex; import org.apache.hadoop.hdds.utils.db.cache.CacheKey; import org.apache.hadoop.hdds.utils.db.cache.CacheValue; @@ -41,6 +46,7 @@ import org.apache.hadoop.ozone.om.exceptions.OMException; import org.apache.hadoop.ozone.om.helpers.KeyValueUtil; import org.apache.hadoop.ozone.om.helpers.OmBucketInfo; +import org.apache.hadoop.ozone.om.helpers.OmDirectoryInfo; import org.apache.hadoop.ozone.om.helpers.OmKeyInfo; import org.apache.hadoop.ozone.om.helpers.BucketLayout; import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfo; @@ -176,11 +182,17 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, TermIn OmBucketInfo omBucketInfo = getBucketInfo(omMetadataManager, volumeName, bucketName); - String ozoneKey = omMetadataManager.getOzoneKey( - volumeName, bucketName, keyName); + List missingParentInfos; + OMFileRequest.OMPathInfoWithFSO pathInfoFSO = OMFileRequest + .verifyDirectoryKeysInPath(omMetadataManager, volumeName, bucketName, + keyName, Paths.get(keyName)); + missingParentInfos = OMDirectoryCreateRequestWithFSO + .getAllMissingParentDirInfo(ozoneManager, keyArgs, omBucketInfo, + pathInfoFSO, trxnLogIndex); - String dbOzoneKey = - getDBOzoneKey(omMetadataManager, volumeName, bucketName, keyName); + // add all missing parents to dir table + addMissingParentsToTable(omBucketInfo, missingParentInfos, + omMetadataManager, volumeName, bucketName, trxnLogIndex); String dbMultipartOpenKey = getDBMultipartOpenKey(volumeName, bucketName, keyName, uploadID, @@ -189,6 +201,45 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, TermIn OmMultipartKeyInfo multipartKeyInfo = omMetadataManager .getMultipartInfoTable().get(multipartKey); + if (null != missingParentInfos) { + + // Add prefix directories to cache and open table + final ReplicationConfig replicationConfig = OzoneConfigUtil + .resolveReplicationConfigPreference(keyArgs.getType(), + keyArgs.getFactor(), keyArgs.getEcReplicationConfig(), + omBucketInfo != null ? + omBucketInfo.getDefaultReplicationConfig() : + null, ozoneManager); + + OmKeyInfo keyInfoFromArgs = new OmKeyInfo.Builder() + .setVolumeName(volumeName) + .setBucketName(bucketName) + .setKeyName(keyName) + .setCreationTime(keyArgs.getModificationTime()) + .setModificationTime(keyArgs.getModificationTime()) + .setReplicationConfig(replicationConfig) + .setOmKeyLocationInfos(Collections.singletonList( + new OmKeyLocationInfoGroup(0, new ArrayList<>(), true))) + .setAcls(getAclsForKey(keyArgs, omBucketInfo, pathInfoFSO, + ozoneManager.getPrefixManager())) + .setObjectID(pathInfoFSO.getLeafNodeObjectId()) + .setUpdateID(trxnLogIndex) + .setFileEncryptionInfo(keyArgs.hasFileEncryptionInfo() ? + OMPBHelper.convert(keyArgs.getFileEncryptionInfo()) : null) + .setParentObjectID(pathInfoFSO.getLastKnownParentId()) + .build(); + + addPrefixDirstoOpenTable(omMetadataManager, dbMultipartOpenKey, + pathInfoFSO, keyInfoFromArgs, multipartKeyInfo, trxnLogIndex, + volumeName, bucketName); + } + + String ozoneKey = omMetadataManager.getOzoneKey( + volumeName, bucketName, keyName); + + String dbOzoneKey = + getDBOzoneKey(omMetadataManager, volumeName, bucketName, keyName); + // Check for directory exists with same name for the LEGACY_FS, // if it exists throw error. checkDirectoryAlreadyExists(ozoneManager, omBucketInfo, keyName, @@ -464,6 +515,23 @@ protected String getDBOzoneKey(OMMetadataManager omMetadataManager, return omMetadataManager.getOzoneKey(volumeName, bucketName, keyName); } + protected void addMissingParentsToTable(OmBucketInfo omBucketInfo, + List missingParentInfos, + OMMetadataManager omMetadataManager, + String volumeName, String bucketName, long transactionLogIndex + ) throws IOException { + return; + } + + protected void addPrefixDirstoOpenTable( + OMMetadataManager omMetadataManager, String multipartOpenKey, + OMFileRequest.OMPathInfoWithFSO pathInfoFSO, OmKeyInfo omKeyInfo, + OmMultipartKeyInfo multipartKeyInfo, long transactionLogIndex, + String volumeName, String bucketName + ) throws IOException { + return; + } + protected OmKeyInfo getOmKeyInfoFromKeyTable(String dbOzoneKey, String keyName, OMMetadataManager omMetadataManager) throws IOException { return omMetadataManager.getKeyTable(getBucketLayout()).get(dbOzoneKey); diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3MultipartUploadCompleteRequestWithFSO.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3MultipartUploadCompleteRequestWithFSO.java index c224786b108..e2007f71546 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3MultipartUploadCompleteRequestWithFSO.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3MultipartUploadCompleteRequestWithFSO.java @@ -18,12 +18,15 @@ package org.apache.hadoop.ozone.om.request.s3.multipart; +import org.apache.hadoop.hdds.utils.db.BatchOperation; import org.apache.hadoop.ozone.om.OMMetadataManager; import org.apache.hadoop.ozone.om.OzoneManager; import org.apache.hadoop.ozone.om.exceptions.OMException; import org.apache.hadoop.ozone.om.helpers.BucketLayout; import org.apache.hadoop.ozone.om.helpers.OmBucketInfo; +import org.apache.hadoop.ozone.om.helpers.OmDirectoryInfo; import org.apache.hadoop.ozone.om.helpers.OmKeyInfo; +import org.apache.hadoop.ozone.om.helpers.OmMultipartKeyInfo; import org.apache.hadoop.ozone.om.request.file.OMFileRequest; import org.apache.hadoop.ozone.om.response.OMClientResponse; import org.apache.hadoop.ozone.om.response.s3.multipart.S3MultipartUploadCompleteResponse; @@ -74,6 +77,82 @@ protected void checkDirectoryAlreadyExists(OzoneManager ozoneManager, } } + @Override + protected void addMissingParentsToTable(OmBucketInfo omBucketInfo, + List missingParentInfos, + OMMetadataManager omMetadataManager, String volumeName, String bucketName, + long transactionLogIndex + ) throws IOException { + + if (null == missingParentInfos) { + return; + } + final long volumeId = omMetadataManager.getVolumeId(volumeName); + final long bucketId = omMetadataManager.getBucketId(volumeName, + bucketName); + + // validate and update namespace for missing parent directory. + checkBucketQuotaInNamespace(omBucketInfo, missingParentInfos.size()); + omBucketInfo.incrUsedNamespace(missingParentInfos.size()); + + // Add cache entries for the missing parent directories. + OMFileRequest.addDirectoryTableCacheEntries(omMetadataManager, + volumeId, bucketId, transactionLogIndex, + missingParentInfos, null); + + // Create missing parent directory entries. + try(BatchOperation batchOperation = omMetadataManager.getStore() + .initBatchOperation()){ + for (OmDirectoryInfo parentDirInfo : missingParentInfos) { + final String parentKey = omMetadataManager.getOzonePathKey( + volumeId, bucketId, parentDirInfo.getParentObjectID(), + parentDirInfo.getName()); + omMetadataManager.getDirectoryTable().putWithBatch(batchOperation, + parentKey, parentDirInfo); + } + + // namespace quota changes for parent directory + String bucketKey = omMetadataManager.getBucketKey( + omBucketInfo.getVolumeName(), + omBucketInfo.getBucketName()); + omMetadataManager.getBucketTable().putWithBatch(batchOperation, + bucketKey, omBucketInfo); + + omMetadataManager.getStore().commitBatchOperation(batchOperation); + } + } + + @Override + protected void addPrefixDirstoOpenTable( + OMMetadataManager omMetadataManager, String multipartOpenKey, + OMFileRequest.OMPathInfoWithFSO pathInfoFSO, OmKeyInfo omKeyInfo, + OmMultipartKeyInfo multipartKeyInfo, long transactionLogIndex, + String volumeName, String bucketName + ) throws IOException { + + final long volumeId = omMetadataManager.getVolumeId(volumeName); + final long bucketId = omMetadataManager.getBucketId(volumeName, + bucketName); + + // Add to prefix directories to cache + OMFileRequest.addOpenFileTableCacheEntry(omMetadataManager, + multipartOpenKey, omKeyInfo, pathInfoFSO.getLeafNodeName(), + transactionLogIndex); + + // Create missing parent directory entries. + try(BatchOperation batchOperation = omMetadataManager.getStore() + .initBatchOperation()){ + + OMFileRequest.addToOpenFileTableForMultipart(omMetadataManager, + batchOperation, + omKeyInfo, multipartKeyInfo.getUploadID(), volumeId, + bucketId); + + omMetadataManager.getStore().commitBatchOperation(batchOperation); + } + } + + @Override protected OmKeyInfo getOmKeyInfoFromKeyTable(String dbOzoneFileKey, String keyName, OMMetadataManager omMetadataManager) throws IOException { From 7c102bb32fa9ea44552468570f3e5064cf842d45 Mon Sep 17 00:00:00 2001 From: saketa Date: Fri, 5 Apr 2024 10:10:46 -0700 Subject: [PATCH 2/5] HDDS-10630. Fixed checkstyle errors. --- .../multipart/S3MultipartUploadCompleteRequest.java | 5 +++-- .../S3MultipartUploadCompleteRequestWithFSO.java | 11 +++++------ 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3MultipartUploadCompleteRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3MultipartUploadCompleteRequest.java index 7fe6b42aa3b..2e504f98483 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3MultipartUploadCompleteRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3MultipartUploadCompleteRequest.java @@ -520,16 +520,17 @@ protected void addMissingParentsToTable(OmBucketInfo omBucketInfo, OMMetadataManager omMetadataManager, String volumeName, String bucketName, long transactionLogIndex ) throws IOException { - return; + // FSO is disabled. Do nothing. } + @SuppressWarnings("checkstyle:ParameterNumber") protected void addPrefixDirstoOpenTable( OMMetadataManager omMetadataManager, String multipartOpenKey, OMFileRequest.OMPathInfoWithFSO pathInfoFSO, OmKeyInfo omKeyInfo, OmMultipartKeyInfo multipartKeyInfo, long transactionLogIndex, String volumeName, String bucketName ) throws IOException { - return; + // FSO is disabled. Do nothing. } protected OmKeyInfo getOmKeyInfoFromKeyTable(String dbOzoneKey, diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3MultipartUploadCompleteRequestWithFSO.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3MultipartUploadCompleteRequestWithFSO.java index e2007f71546..62746de8d98 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3MultipartUploadCompleteRequestWithFSO.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3MultipartUploadCompleteRequestWithFSO.java @@ -81,8 +81,7 @@ protected void checkDirectoryAlreadyExists(OzoneManager ozoneManager, protected void addMissingParentsToTable(OmBucketInfo omBucketInfo, List missingParentInfos, OMMetadataManager omMetadataManager, String volumeName, String bucketName, - long transactionLogIndex - ) throws IOException { + long transactionLogIndex) throws IOException { if (null == missingParentInfos) { return; @@ -101,8 +100,8 @@ protected void addMissingParentsToTable(OmBucketInfo omBucketInfo, missingParentInfos, null); // Create missing parent directory entries. - try(BatchOperation batchOperation = omMetadataManager.getStore() - .initBatchOperation()){ + try (BatchOperation batchOperation = omMetadataManager.getStore() + .initBatchOperation()) { for (OmDirectoryInfo parentDirInfo : missingParentInfos) { final String parentKey = omMetadataManager.getOzonePathKey( volumeId, bucketId, parentDirInfo.getParentObjectID(), @@ -140,8 +139,8 @@ protected void addPrefixDirstoOpenTable( transactionLogIndex); // Create missing parent directory entries. - try(BatchOperation batchOperation = omMetadataManager.getStore() - .initBatchOperation()){ + try (BatchOperation batchOperation = omMetadataManager.getStore() + .initBatchOperation()) { OMFileRequest.addToOpenFileTableForMultipart(omMetadataManager, batchOperation, From bff45538706b56e4649795b9642748bc6ffcdbd1 Mon Sep 17 00:00:00 2001 From: saketa Date: Mon, 8 Apr 2024 19:05:21 -0700 Subject: [PATCH 3/5] HDDS-10630. Add multi part to open key table only when it and parents are missing. --- .../S3MultipartUploadCompleteRequest.java | 103 +++++++++++------- ...MultipartUploadCompleteRequestWithFSO.java | 23 +--- 2 files changed, 69 insertions(+), 57 deletions(-) diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3MultipartUploadCompleteRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3MultipartUploadCompleteRequest.java index 2e504f98483..37e1fe1d8de 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3MultipartUploadCompleteRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3MultipartUploadCompleteRequest.java @@ -190,9 +190,65 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, TermIn .getAllMissingParentDirInfo(ozoneManager, keyArgs, omBucketInfo, pathInfoFSO, trxnLogIndex); - // add all missing parents to dir table - addMissingParentsToTable(omBucketInfo, missingParentInfos, - omMetadataManager, volumeName, bucketName, trxnLogIndex); + if (missingParentInfos != null) { + final long volumeId = omMetadataManager.getVolumeId(volumeName); + final long bucketId = omMetadataManager.getBucketId(volumeName, + bucketName); + + // add all missing parents to directory table + addMissingParentsToTable(omBucketInfo, missingParentInfos, + omMetadataManager, volumeId, bucketId, trxnLogIndex); + + String multipartOpenKey = omMetadataManager + .getMultipartKey(volumeId, bucketId, + pathInfoFSO.getLastKnownParentId(), + pathInfoFSO.getLeafNodeName(), + keyArgs.getMultipartUploadID()); + + if (getOmKeyInfoFromOpenKeyTable(multipartOpenKey, + keyName, omMetadataManager) == null) { + + final ReplicationConfig replicationConfig = OzoneConfigUtil + .resolveReplicationConfigPreference(keyArgs.getType(), + keyArgs.getFactor(), keyArgs.getEcReplicationConfig(), + omBucketInfo != null ? + omBucketInfo.getDefaultReplicationConfig() : + null, ozoneManager); + + OmMultipartKeyInfo multipartKeyInfoFromArgs = + new OmMultipartKeyInfo.Builder() + .setUploadID(keyArgs.getMultipartUploadID()) + .setCreationTime(keyArgs.getModificationTime()) + .setReplicationConfig(replicationConfig) + .setObjectID(pathInfoFSO.getLeafNodeObjectId()) + .setUpdateID(trxnLogIndex) + .setParentID(pathInfoFSO.getLastKnownParentId()) + .build(); + + OmKeyInfo keyInfoFromArgs = new OmKeyInfo.Builder() + .setVolumeName(volumeName) + .setBucketName(bucketName) + .setKeyName(keyName) + .setCreationTime(keyArgs.getModificationTime()) + .setModificationTime(keyArgs.getModificationTime()) + .setReplicationConfig(replicationConfig) + .setOmKeyLocationInfos(Collections.singletonList( + new OmKeyLocationInfoGroup(0, new ArrayList<>(), true))) + .setAcls(getAclsForKey(keyArgs, omBucketInfo, pathInfoFSO, + ozoneManager.getPrefixManager())) + .setObjectID(pathInfoFSO.getLeafNodeObjectId()) + .setUpdateID(trxnLogIndex) + .setFileEncryptionInfo(keyArgs.hasFileEncryptionInfo() ? + OMPBHelper.convert(keyArgs.getFileEncryptionInfo()) : null) + .setParentObjectID(pathInfoFSO.getLastKnownParentId()) + .build(); + + // Add missing multi part info to open key table + addMultiParttoOpenTable(omMetadataManager, multipartOpenKey, + multipartKeyInfoFromArgs, pathInfoFSO, keyInfoFromArgs, + volumeId, bucketId, trxnLogIndex); + } + } String dbMultipartOpenKey = getDBMultipartOpenKey(volumeName, bucketName, keyName, uploadID, @@ -201,39 +257,6 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, TermIn OmMultipartKeyInfo multipartKeyInfo = omMetadataManager .getMultipartInfoTable().get(multipartKey); - if (null != missingParentInfos) { - - // Add prefix directories to cache and open table - final ReplicationConfig replicationConfig = OzoneConfigUtil - .resolveReplicationConfigPreference(keyArgs.getType(), - keyArgs.getFactor(), keyArgs.getEcReplicationConfig(), - omBucketInfo != null ? - omBucketInfo.getDefaultReplicationConfig() : - null, ozoneManager); - - OmKeyInfo keyInfoFromArgs = new OmKeyInfo.Builder() - .setVolumeName(volumeName) - .setBucketName(bucketName) - .setKeyName(keyName) - .setCreationTime(keyArgs.getModificationTime()) - .setModificationTime(keyArgs.getModificationTime()) - .setReplicationConfig(replicationConfig) - .setOmKeyLocationInfos(Collections.singletonList( - new OmKeyLocationInfoGroup(0, new ArrayList<>(), true))) - .setAcls(getAclsForKey(keyArgs, omBucketInfo, pathInfoFSO, - ozoneManager.getPrefixManager())) - .setObjectID(pathInfoFSO.getLeafNodeObjectId()) - .setUpdateID(trxnLogIndex) - .setFileEncryptionInfo(keyArgs.hasFileEncryptionInfo() ? - OMPBHelper.convert(keyArgs.getFileEncryptionInfo()) : null) - .setParentObjectID(pathInfoFSO.getLastKnownParentId()) - .build(); - - addPrefixDirstoOpenTable(omMetadataManager, dbMultipartOpenKey, - pathInfoFSO, keyInfoFromArgs, multipartKeyInfo, trxnLogIndex, - volumeName, bucketName); - } - String ozoneKey = omMetadataManager.getOzoneKey( volumeName, bucketName, keyName); @@ -518,17 +541,17 @@ protected String getDBOzoneKey(OMMetadataManager omMetadataManager, protected void addMissingParentsToTable(OmBucketInfo omBucketInfo, List missingParentInfos, OMMetadataManager omMetadataManager, - String volumeName, String bucketName, long transactionLogIndex + long volumeId, long bucketId, long transactionLogIndex ) throws IOException { // FSO is disabled. Do nothing. } @SuppressWarnings("checkstyle:ParameterNumber") - protected void addPrefixDirstoOpenTable( + protected void addMultiParttoOpenTable( OMMetadataManager omMetadataManager, String multipartOpenKey, + OmMultipartKeyInfo multipartKeyInfo, OMFileRequest.OMPathInfoWithFSO pathInfoFSO, OmKeyInfo omKeyInfo, - OmMultipartKeyInfo multipartKeyInfo, long transactionLogIndex, - String volumeName, String bucketName + long volumeId, long bucketId, long transactionLogIndex ) throws IOException { // FSO is disabled. Do nothing. } diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3MultipartUploadCompleteRequestWithFSO.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3MultipartUploadCompleteRequestWithFSO.java index 62746de8d98..6da824b32fb 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3MultipartUploadCompleteRequestWithFSO.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3MultipartUploadCompleteRequestWithFSO.java @@ -80,16 +80,9 @@ protected void checkDirectoryAlreadyExists(OzoneManager ozoneManager, @Override protected void addMissingParentsToTable(OmBucketInfo omBucketInfo, List missingParentInfos, - OMMetadataManager omMetadataManager, String volumeName, String bucketName, + OMMetadataManager omMetadataManager, long volumeId, long bucketId, long transactionLogIndex) throws IOException { - if (null == missingParentInfos) { - return; - } - final long volumeId = omMetadataManager.getVolumeId(volumeName); - final long bucketId = omMetadataManager.getBucketId(volumeName, - bucketName); - // validate and update namespace for missing parent directory. checkBucketQuotaInNamespace(omBucketInfo, missingParentInfos.size()); omBucketInfo.incrUsedNamespace(missingParentInfos.size()); @@ -122,23 +115,19 @@ protected void addMissingParentsToTable(OmBucketInfo omBucketInfo, } @Override - protected void addPrefixDirstoOpenTable( + protected void addMultiParttoOpenTable( OMMetadataManager omMetadataManager, String multipartOpenKey, + OmMultipartKeyInfo multipartKeyInfo, OMFileRequest.OMPathInfoWithFSO pathInfoFSO, OmKeyInfo omKeyInfo, - OmMultipartKeyInfo multipartKeyInfo, long transactionLogIndex, - String volumeName, String bucketName + long volumeId, long bucketId, long transactionLogIndex ) throws IOException { - final long volumeId = omMetadataManager.getVolumeId(volumeName); - final long bucketId = omMetadataManager.getBucketId(volumeName, - bucketName); - - // Add to prefix directories to cache + // Add multi part to cache OMFileRequest.addOpenFileTableCacheEntry(omMetadataManager, multipartOpenKey, omKeyInfo, pathInfoFSO.getLeafNodeName(), transactionLogIndex); - // Create missing parent directory entries. + // Add multi part to open key table. try (BatchOperation batchOperation = omMetadataManager.getStore() .initBatchOperation()) { From c200cf12487645d2287b7f0e435a9a6e34fa71df Mon Sep 17 00:00:00 2001 From: saketa Date: Wed, 10 Apr 2024 19:49:26 -0700 Subject: [PATCH 4/5] HDDS-10630. Moved DB batch operation to S3MultipartUploadCompleteResponseWithFSO.java --- .../S3MultipartUploadCompleteRequest.java | 20 ++++---- ...MultipartUploadCompleteRequestWithFSO.java | 45 +++--------------- .../S3MultipartUploadCompleteResponse.java | 8 ++++ ...ultipartUploadCompleteResponseWithFSO.java | 47 +++++++++++++++++-- .../s3/multipart/TestS3MultipartResponse.java | 5 +- 5 files changed, 71 insertions(+), 54 deletions(-) diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3MultipartUploadCompleteRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3MultipartUploadCompleteRequest.java index 37e1fe1d8de..5a2359560aa 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3MultipartUploadCompleteRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3MultipartUploadCompleteRequest.java @@ -196,7 +196,7 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, TermIn bucketName); // add all missing parents to directory table - addMissingParentsToTable(omBucketInfo, missingParentInfos, + addMissingParentsToCache(omBucketInfo, missingParentInfos, omMetadataManager, volumeId, bucketId, trxnLogIndex); String multipartOpenKey = omMetadataManager @@ -244,9 +244,8 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, TermIn .build(); // Add missing multi part info to open key table - addMultiParttoOpenTable(omMetadataManager, multipartOpenKey, - multipartKeyInfoFromArgs, pathInfoFSO, keyInfoFromArgs, - volumeId, bucketId, trxnLogIndex); + addMultiPartToCache(omMetadataManager, multipartOpenKey, + pathInfoFSO, keyInfoFromArgs, trxnLogIndex); } } @@ -358,7 +357,7 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, TermIn omClientResponse = getOmClientResponse(multipartKey, omResponse, dbMultipartOpenKey, omKeyInfo, allKeyInfoToRemove, omBucketInfo, - volumeId, bucketId); + volumeId, bucketId, missingParentInfos, multipartKeyInfo); result = Result.SUCCESS; } else { @@ -399,7 +398,8 @@ protected OMClientResponse getOmClientResponse(String multipartKey, OMResponse.Builder omResponse, String dbMultipartOpenKey, OmKeyInfo omKeyInfo, List allKeyInfoToRemove, OmBucketInfo omBucketInfo, - long volumeId, long bucketId) { + long volumeId, long bucketId, List missingParentInfos, + OmMultipartKeyInfo multipartKeyInfo) { return new S3MultipartUploadCompleteResponse(omResponse.build(), multipartKey, dbMultipartOpenKey, omKeyInfo, allKeyInfoToRemove, @@ -538,7 +538,7 @@ protected String getDBOzoneKey(OMMetadataManager omMetadataManager, return omMetadataManager.getOzoneKey(volumeName, bucketName, keyName); } - protected void addMissingParentsToTable(OmBucketInfo omBucketInfo, + protected void addMissingParentsToCache(OmBucketInfo omBucketInfo, List missingParentInfos, OMMetadataManager omMetadataManager, long volumeId, long bucketId, long transactionLogIndex @@ -546,12 +546,10 @@ protected void addMissingParentsToTable(OmBucketInfo omBucketInfo, // FSO is disabled. Do nothing. } - @SuppressWarnings("checkstyle:ParameterNumber") - protected void addMultiParttoOpenTable( + protected void addMultiPartToCache( OMMetadataManager omMetadataManager, String multipartOpenKey, - OmMultipartKeyInfo multipartKeyInfo, OMFileRequest.OMPathInfoWithFSO pathInfoFSO, OmKeyInfo omKeyInfo, - long volumeId, long bucketId, long transactionLogIndex + long transactionLogIndex ) throws IOException { // FSO is disabled. Do nothing. } diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3MultipartUploadCompleteRequestWithFSO.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3MultipartUploadCompleteRequestWithFSO.java index 6da824b32fb..2ee594e70eb 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3MultipartUploadCompleteRequestWithFSO.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3MultipartUploadCompleteRequestWithFSO.java @@ -78,7 +78,7 @@ protected void checkDirectoryAlreadyExists(OzoneManager ozoneManager, } @Override - protected void addMissingParentsToTable(OmBucketInfo omBucketInfo, + protected void addMissingParentsToCache(OmBucketInfo omBucketInfo, List missingParentInfos, OMMetadataManager omMetadataManager, long volumeId, long bucketId, long transactionLogIndex) throws IOException { @@ -91,35 +91,13 @@ protected void addMissingParentsToTable(OmBucketInfo omBucketInfo, OMFileRequest.addDirectoryTableCacheEntries(omMetadataManager, volumeId, bucketId, transactionLogIndex, missingParentInfos, null); - - // Create missing parent directory entries. - try (BatchOperation batchOperation = omMetadataManager.getStore() - .initBatchOperation()) { - for (OmDirectoryInfo parentDirInfo : missingParentInfos) { - final String parentKey = omMetadataManager.getOzonePathKey( - volumeId, bucketId, parentDirInfo.getParentObjectID(), - parentDirInfo.getName()); - omMetadataManager.getDirectoryTable().putWithBatch(batchOperation, - parentKey, parentDirInfo); - } - - // namespace quota changes for parent directory - String bucketKey = omMetadataManager.getBucketKey( - omBucketInfo.getVolumeName(), - omBucketInfo.getBucketName()); - omMetadataManager.getBucketTable().putWithBatch(batchOperation, - bucketKey, omBucketInfo); - - omMetadataManager.getStore().commitBatchOperation(batchOperation); - } } @Override - protected void addMultiParttoOpenTable( + protected void addMultiPartToCache( OMMetadataManager omMetadataManager, String multipartOpenKey, - OmMultipartKeyInfo multipartKeyInfo, OMFileRequest.OMPathInfoWithFSO pathInfoFSO, OmKeyInfo omKeyInfo, - long volumeId, long bucketId, long transactionLogIndex + long transactionLogIndex ) throws IOException { // Add multi part to cache @@ -127,17 +105,6 @@ protected void addMultiParttoOpenTable( multipartOpenKey, omKeyInfo, pathInfoFSO.getLeafNodeName(), transactionLogIndex); - // Add multi part to open key table. - try (BatchOperation batchOperation = omMetadataManager.getStore() - .initBatchOperation()) { - - OMFileRequest.addToOpenFileTableForMultipart(omMetadataManager, - batchOperation, - omKeyInfo, multipartKeyInfo.getUploadID(), volumeId, - bucketId); - - omMetadataManager.getStore().commitBatchOperation(batchOperation); - } } @@ -214,11 +181,13 @@ protected OMClientResponse getOmClientResponse(String multipartKey, OzoneManagerProtocolProtos.OMResponse.Builder omResponse, String dbMultipartOpenKey, OmKeyInfo omKeyInfo, List allKeyInfoToRemove, OmBucketInfo omBucketInfo, - long volumeId, long bucketId) { + long volumeId, long bucketId, List missingParentInfos, + OmMultipartKeyInfo multipartKeyInfo) { return new S3MultipartUploadCompleteResponseWithFSO(omResponse.build(), multipartKey, dbMultipartOpenKey, omKeyInfo, allKeyInfoToRemove, - getBucketLayout(), omBucketInfo, volumeId, bucketId); + getBucketLayout(), omBucketInfo, volumeId, bucketId, + missingParentInfos, multipartKeyInfo); } @Override diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/s3/multipart/S3MultipartUploadCompleteResponse.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/s3/multipart/S3MultipartUploadCompleteResponse.java index 9fb843dcbe1..a1f7b796cd8 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/s3/multipart/S3MultipartUploadCompleteResponse.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/s3/multipart/S3MultipartUploadCompleteResponse.java @@ -129,4 +129,12 @@ protected String addToKeyTable(OMMetadataManager omMetadataManager, protected OmKeyInfo getOmKeyInfo() { return omKeyInfo; } + + protected OmBucketInfo getOmBucketInfo() { + return omBucketInfo; + } + + protected String getMultiPartKey() { + return multipartKey; + } } diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/s3/multipart/S3MultipartUploadCompleteResponseWithFSO.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/s3/multipart/S3MultipartUploadCompleteResponseWithFSO.java index 8774627ee66..9ab84c63501 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/s3/multipart/S3MultipartUploadCompleteResponseWithFSO.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/s3/multipart/S3MultipartUploadCompleteResponseWithFSO.java @@ -20,9 +20,7 @@ import org.apache.hadoop.hdds.utils.db.BatchOperation; import org.apache.hadoop.ozone.om.OMMetadataManager; -import org.apache.hadoop.ozone.om.helpers.BucketLayout; -import org.apache.hadoop.ozone.om.helpers.OmBucketInfo; -import org.apache.hadoop.ozone.om.helpers.OmKeyInfo; +import org.apache.hadoop.ozone.om.helpers.*; import org.apache.hadoop.ozone.om.request.file.OMFileRequest; import org.apache.hadoop.ozone.om.response.CleanupTableInfo; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMResponse; @@ -52,6 +50,10 @@ public class S3MultipartUploadCompleteResponseWithFSO private long volumeId; private long bucketId; + private List missingParentInfos; + + private OmMultipartKeyInfo multipartKeyInfo; + @SuppressWarnings("checkstyle:ParameterNumber") public S3MultipartUploadCompleteResponseWithFSO( @Nonnull OMResponse omResponse, @@ -61,11 +63,15 @@ public S3MultipartUploadCompleteResponseWithFSO( @Nonnull List allKeyInfoToRemove, @Nonnull BucketLayout bucketLayout, OmBucketInfo omBucketInfo, - @Nonnull long volumeId, @Nonnull long bucketId) { + @Nonnull long volumeId, @Nonnull long bucketId, + List missingParentInfos, + OmMultipartKeyInfo multipartKeyInfo) { super(omResponse, multipartKey, multipartOpenKey, omKeyInfo, allKeyInfoToRemove, bucketLayout, omBucketInfo); this.volumeId = volumeId; this.bucketId = bucketId; + this.missingParentInfos = missingParentInfos; + this.multipartKeyInfo = multipartKeyInfo; } /** @@ -78,6 +84,39 @@ public S3MultipartUploadCompleteResponseWithFSO( checkStatusNotOK(); } + @Override + public void addToDBBatch(OMMetadataManager omMetadataManager, + BatchOperation batchOperation) throws IOException { + if (missingParentInfos != null) { + // Create missing parent directory entries. + for (OmDirectoryInfo parentDirInfo : missingParentInfos) { + final String parentKey = omMetadataManager.getOzonePathKey( + volumeId, bucketId, parentDirInfo.getParentObjectID(), + parentDirInfo.getName()); + omMetadataManager.getDirectoryTable().putWithBatch(batchOperation, + parentKey, parentDirInfo); + } + + // namespace quota changes for parent directory + String bucketKey = omMetadataManager.getBucketKey( + getOmBucketInfo().getVolumeName(), + getOmBucketInfo().getBucketName()); + omMetadataManager.getBucketTable().putWithBatch(batchOperation, + bucketKey, getOmBucketInfo()); + + if (OMFileRequest.getOmKeyInfoFromFileTable(true, + omMetadataManager, getMultiPartKey(), getOmKeyInfo().getKeyName()) + != null) { + // Add multi part to open key table. + OMFileRequest.addToOpenFileTableForMultipart(omMetadataManager, + batchOperation, + getOmKeyInfo(), multipartKeyInfo.getUploadID(), volumeId, + bucketId); + } + } + super.addToDBBatch(omMetadataManager, batchOperation); + } + @Override protected String addToKeyTable(OMMetadataManager omMetadataManager, BatchOperation batchOperation) throws IOException { diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/s3/multipart/TestS3MultipartResponse.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/s3/multipart/TestS3MultipartResponse.java index 51963a00a1c..a2192ddb880 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/s3/multipart/TestS3MultipartResponse.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/s3/multipart/TestS3MultipartResponse.java @@ -306,6 +306,8 @@ public S3MultipartUploadCompleteResponse createS3CompleteMPUResponseFSO( String multipartKey = omMetadataManager .getMultipartKey(volumeName, bucketName, keyName, multipartUploadID); + OmMultipartKeyInfo multipartKeyInfo = omMetadataManager + .getMultipartInfoTable().get(multipartKey); final long volumeId = omMetadataManager.getVolumeId(volumeName); final long bucketId = omMetadataManager.getBucketId(volumeName, @@ -324,7 +326,8 @@ public S3MultipartUploadCompleteResponse createS3CompleteMPUResponseFSO( return new S3MultipartUploadCompleteResponseWithFSO(omResponse, multipartKey, multipartOpenKey, omKeyInfo, allKeyInfoToRemove, - getBucketLayout(), omBucketInfo, volumeId, bucketId); + getBucketLayout(), omBucketInfo, volumeId, bucketId, null, + multipartKeyInfo); } protected S3InitiateMultipartUploadResponse getS3InitiateMultipartUploadResp( From a1c5c53bd54bd298bfed04e4a07eca9a4b694b15 Mon Sep 17 00:00:00 2001 From: saketa Date: Wed, 10 Apr 2024 20:09:39 -0700 Subject: [PATCH 5/5] HDDS-10630. Fixed checkstyle errors. --- .../multipart/S3MultipartUploadCompleteRequestWithFSO.java | 1 - .../multipart/S3MultipartUploadCompleteResponseWithFSO.java | 6 +++++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3MultipartUploadCompleteRequestWithFSO.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3MultipartUploadCompleteRequestWithFSO.java index 2ee594e70eb..c13d2746ae4 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3MultipartUploadCompleteRequestWithFSO.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3MultipartUploadCompleteRequestWithFSO.java @@ -18,7 +18,6 @@ package org.apache.hadoop.ozone.om.request.s3.multipart; -import org.apache.hadoop.hdds.utils.db.BatchOperation; import org.apache.hadoop.ozone.om.OMMetadataManager; import org.apache.hadoop.ozone.om.OzoneManager; import org.apache.hadoop.ozone.om.exceptions.OMException; diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/s3/multipart/S3MultipartUploadCompleteResponseWithFSO.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/s3/multipart/S3MultipartUploadCompleteResponseWithFSO.java index 9ab84c63501..4d1a6ce09bc 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/s3/multipart/S3MultipartUploadCompleteResponseWithFSO.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/s3/multipart/S3MultipartUploadCompleteResponseWithFSO.java @@ -20,7 +20,11 @@ import org.apache.hadoop.hdds.utils.db.BatchOperation; import org.apache.hadoop.ozone.om.OMMetadataManager; -import org.apache.hadoop.ozone.om.helpers.*; +import org.apache.hadoop.ozone.om.helpers.BucketLayout; +import org.apache.hadoop.ozone.om.helpers.OmBucketInfo; +import org.apache.hadoop.ozone.om.helpers.OmDirectoryInfo; +import org.apache.hadoop.ozone.om.helpers.OmKeyInfo; +import org.apache.hadoop.ozone.om.helpers.OmMultipartKeyInfo; import org.apache.hadoop.ozone.om.request.file.OMFileRequest; import org.apache.hadoop.ozone.om.response.CleanupTableInfo; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMResponse;