diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneBucket.java b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneBucket.java index 6972831477e..575701be800 100644 --- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneBucket.java +++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneBucket.java @@ -39,6 +39,7 @@ import org.apache.hadoop.ozone.OzoneAcl; import org.apache.hadoop.ozone.om.exceptions.OMException; import org.apache.hadoop.ozone.om.helpers.BasicOmKeyInfo; +import org.apache.hadoop.ozone.om.helpers.ErrorInfo; import org.apache.hadoop.ozone.om.helpers.OmMultipartInfo; import org.apache.hadoop.ozone.om.helpers.OmMultipartUploadCompleteInfo; import org.apache.hadoop.ozone.om.helpers.OmKeyInfo; @@ -675,6 +676,16 @@ public void deleteKeys(List keyList) throws IOException { proxy.deleteKeys(volumeName, name, keyList); } + /** + * Deletes the given list of keys from the bucket. + * @param keyList List of the key name to be deleted. + * @param quiet flag to not throw exception if delete fails + * @throws IOException + */ + public Map deleteKeys(List keyList, boolean quiet) throws IOException { + return proxy.deleteKeys(volumeName, name, keyList, quiet); + } + /** * Rename the keyname from fromKeyName to toKeyName. * @param fromKeyName The original key name. diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/protocol/ClientProtocol.java b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/protocol/ClientProtocol.java index 81a2e7c25c6..74344344f89 100644 --- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/protocol/ClientProtocol.java +++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/protocol/ClientProtocol.java @@ -49,6 +49,7 @@ import org.apache.hadoop.ozone.om.OMConfigKeys; import org.apache.hadoop.ozone.om.exceptions.OMException; import org.apache.hadoop.ozone.om.helpers.DeleteTenantState; +import org.apache.hadoop.ozone.om.helpers.ErrorInfo; import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfo; import org.apache.hadoop.ozone.om.helpers.OmMultipartInfo; import org.apache.hadoop.ozone.om.helpers.OmMultipartUploadCompleteInfo; @@ -437,6 +438,18 @@ void deleteKeys(String volumeName, String bucketName, List keyNameList) throws IOException; + /** + * Deletes keys through the list. + * @param volumeName Name of the Volume + * @param bucketName Name of the Bucket + * @param keyNameList List of the Key + * @param quiet flag to not throw exception if delete fails + * @throws IOException + */ + Map deleteKeys(String volumeName, String bucketName, + List keyNameList, boolean quiet) + throws IOException; + /** * Renames an existing key within a bucket. * @param volumeName Name of the Volume diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java index a1b35d65a59..248f21cd703 100644 --- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java +++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java @@ -98,6 +98,7 @@ import org.apache.hadoop.ozone.om.helpers.BucketEncryptionKeyInfo; import org.apache.hadoop.ozone.om.helpers.BucketLayout; import org.apache.hadoop.ozone.om.helpers.DeleteTenantState; +import org.apache.hadoop.ozone.om.helpers.ErrorInfo; import org.apache.hadoop.ozone.om.helpers.KeyInfoWithVolumeContext; import org.apache.hadoop.ozone.om.helpers.OmBucketArgs; import org.apache.hadoop.ozone.om.helpers.OmBucketInfo; @@ -1641,6 +1642,18 @@ public void deleteKeys( ozoneManagerClient.deleteKeys(omDeleteKeys); } + @Override + public Map deleteKeys( + String volumeName, String bucketName, List keyNameList, boolean quiet) + throws IOException { + verifyVolumeName(volumeName); + verifyBucketName(bucketName); + Preconditions.checkNotNull(keyNameList); + OmDeleteKeys omDeleteKeys = new OmDeleteKeys(volumeName, bucketName, + keyNameList); + return ozoneManagerClient.deleteKeys(omDeleteKeys, quiet); + } + @Override public void renameKey(String volumeName, String bucketName, String fromKeyName, String toKeyName) throws IOException { diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/ErrorInfo.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/ErrorInfo.java new file mode 100644 index 00000000000..7889a568be8 --- /dev/null +++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/ErrorInfo.java @@ -0,0 +1,49 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.ozone.om.helpers; + +/** + * Represent class which has info of error thrown for any operation. + */ +public class ErrorInfo { + private String code; + private String message; + + public ErrorInfo(String errorCode, String errorMessage) { + this.code = errorCode; + this.message = errorMessage; + } + + public String getCode() { + return code; + } + + public void setCode(String errorCode) { + this.code = errorCode; + } + + public String getMessage() { + return message; + } + + public void setMessage(String errorMessage) { + this.message = errorMessage; + } + +} diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocol/OzoneManagerProtocol.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocol/OzoneManagerProtocol.java index 970eb9d509d..5dde55c841d 100644 --- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocol/OzoneManagerProtocol.java +++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocol/OzoneManagerProtocol.java @@ -21,6 +21,7 @@ import java.io.Closeable; import java.io.IOException; import java.util.List; +import java.util.Map; import java.util.UUID; import jakarta.annotation.Nonnull; @@ -32,6 +33,7 @@ import org.apache.hadoop.ozone.om.exceptions.OMException; import org.apache.hadoop.ozone.om.helpers.DBUpdates; import org.apache.hadoop.ozone.om.helpers.DeleteTenantState; +import org.apache.hadoop.ozone.om.helpers.ErrorInfo; import org.apache.hadoop.ozone.om.helpers.KeyInfoWithVolumeContext; import org.apache.hadoop.ozone.om.helpers.OmBucketArgs; import org.apache.hadoop.ozone.om.helpers.OmBucketInfo; @@ -360,6 +362,21 @@ default void deleteKeys(OmDeleteKeys deleteKeys) throws IOException { "this to be implemented, as write requests use a new approach."); } + /** + * Deletes existing key/keys. This interface supports delete + * multiple keys and a single key. Used by deleting files + * through OzoneFileSystem. + * + * @param deleteKeys + * @param quiet - flag to not throw exception if delete fails + * @throws IOException + */ + default Map deleteKeys(OmDeleteKeys deleteKeys, boolean quiet) + throws IOException { + throw new UnsupportedOperationException("OzoneManager does not require " + + "this to be implemented, as write requests use a new approach."); + } + /** * Deletes an existing empty bucket from volume. diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OzoneManagerProtocolClientSideTranslatorPB.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OzoneManagerProtocolClientSideTranslatorPB.java index 1208df0f28e..3e21494ac61 100644 --- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OzoneManagerProtocolClientSideTranslatorPB.java +++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OzoneManagerProtocolClientSideTranslatorPB.java @@ -20,6 +20,7 @@ import java.io.IOException; import java.time.Instant; import java.util.ArrayList; +import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.UUID; @@ -42,6 +43,7 @@ import org.apache.hadoop.ozone.OzoneAcl; import org.apache.hadoop.ozone.om.exceptions.OMException; import org.apache.hadoop.ozone.om.helpers.BasicOmKeyInfo; +import org.apache.hadoop.ozone.om.helpers.ErrorInfo; import org.apache.hadoop.ozone.om.helpers.ListKeysLightResult; import org.apache.hadoop.ozone.om.helpers.ListKeysResult; import org.apache.hadoop.ozone.om.helpers.DBUpdates; @@ -954,6 +956,12 @@ public void deleteKey(OmKeyArgs args) throws IOException { */ @Override public void deleteKeys(OmDeleteKeys deleteKeys) throws IOException { + deleteKeys(deleteKeys, false); + } + + @Override + public Map deleteKeys(OmDeleteKeys deleteKeys, boolean quiet) + throws IOException { DeleteKeysRequest.Builder req = DeleteKeysRequest.newBuilder(); DeleteKeyArgs deletedKeys = DeleteKeyArgs.newBuilder() .setBucketName(deleteKeys.getBucket()) @@ -963,9 +971,20 @@ public void deleteKeys(OmDeleteKeys deleteKeys) throws IOException { OMRequest omRequest = createOMRequest(Type.DeleteKeys) .setDeleteKeysRequest(req) .build(); + OMResponse omResponse = submitRequest(omRequest); - handleError(submitRequest(omRequest)); - + Map keyToErrors = new HashMap<>(); + if (quiet) { + List errors = + omResponse.getDeleteKeysResponse().getErrorsList(); + for (OzoneManagerProtocolProtos.DeleteKeyError deleteKeyError : errors) { + keyToErrors.put(deleteKeyError.getKey(), + new ErrorInfo(deleteKeyError.getErrorCode(), deleteKeyError.getErrorMsg())); + } + } else { + handleError(omResponse); + } + return keyToErrors; } /** diff --git a/hadoop-ozone/dist/src/main/compose/common/s3a-test.sh b/hadoop-ozone/dist/src/main/compose/common/s3a-test.sh index 1a784b11610..554b22b5a39 100644 --- a/hadoop-ozone/dist/src/main/compose/common/s3a-test.sh +++ b/hadoop-ozone/dist/src/main/compose/common/s3a-test.sh @@ -95,10 +95,9 @@ EOF # Some tests are skipped due to known issues. # - ITestS3AContractDistCp: HDDS-10616 # - ITestS3AContractGetFileStatusV1List: HDDS-10617 - # - ITestS3AContractMkdir: HDDS-10572 # - ITestS3AContractRename: HDDS-10665 mvn -B -V --fail-never --no-transfer-progress \ - -Dtest='ITestS3AContract*, ITestS3ACommitterMRJob, !ITestS3AContractDistCp, !ITestS3AContractGetFileStatusV1List, !ITestS3AContractMkdir, !ITestS3AContractRename' \ + -Dtest='ITestS3AContract*, ITestS3ACommitterMRJob, !ITestS3AContractDistCp, !ITestS3AContractGetFileStatusV1List, !ITestS3AContractRename' \ clean test local target="${RESULT_DIR}/junit/${bucket}/target" diff --git a/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto b/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto index 48b64b71ddb..5f5a50d1c86 100644 --- a/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto +++ b/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto @@ -1301,9 +1301,16 @@ message DeleteKeyArgs { repeated string keys = 3; } +message DeleteKeyError { + optional string key = 1; + optional string errorCode = 2; + optional string errorMsg = 3; +} + message DeleteKeysResponse { optional DeleteKeyArgs unDeletedKeys = 1; optional bool status = 2; + repeated DeleteKeyError errors = 3; } message DeleteKeyResponse { diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeysDeleteRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeysDeleteRequest.java index be89da369cd..19e9ed716e4 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeysDeleteRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeysDeleteRequest.java @@ -31,6 +31,7 @@ import org.apache.hadoop.ozone.om.ResolvedBucket; import org.apache.hadoop.ozone.om.exceptions.OMException; import org.apache.hadoop.ozone.om.helpers.BucketLayout; +import org.apache.hadoop.ozone.om.helpers.ErrorInfo; import org.apache.hadoop.ozone.om.helpers.OmBucketInfo; import org.apache.hadoop.ozone.om.helpers.OmKeyInfo; import org.apache.hadoop.ozone.om.helpers.OzoneFileStatus; @@ -57,6 +58,7 @@ import java.io.IOException; import java.nio.file.InvalidPathException; import java.util.ArrayList; +import java.util.HashMap; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; @@ -95,6 +97,7 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, TermIn Exception exception = null; OMClientResponse omClientResponse = null; Result result = null; + Map keyToError = new HashMap<>(); OMMetrics omMetrics = ozoneManager.getMetrics(); omMetrics.incNumKeyDeletes(); @@ -150,6 +153,7 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, TermIn objectKey); deleteKeys.remove(keyName); unDeletedKeys.addKeys(keyName); + keyToError.put(keyName, new ErrorInfo(OMException.ResultCodes.KEY_NOT_FOUND.name(), "Key does not exist")); continue; } @@ -167,6 +171,7 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, TermIn LOG.error("Acl check failed for Key: {}", objectKey, ex); deleteKeys.remove(keyName); unDeletedKeys.addKeys(keyName); + keyToError.put(keyName, new ErrorInfo(OMException.ResultCodes.ACCESS_DENIED.name(), "ACL check failed")); } } @@ -185,7 +190,7 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, TermIn final long volumeId = omMetadataManager.getVolumeId(volumeName); omClientResponse = getOmClientResponse(ozoneManager, omKeyInfoList, dirList, omResponse, - unDeletedKeys, deleteStatus, omBucketInfo, volumeId, dbOpenKeys); + unDeletedKeys, keyToError, deleteStatus, omBucketInfo, volumeId, dbOpenKeys); result = Result.SUCCESS; @@ -199,6 +204,8 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, TermIn // Add all keys which are failed due to any other exception . for (int i = indexFailed; i < length; i++) { unDeletedKeys.addKeys(deleteKeyArgs.getKeys(i)); + keyToError.put(deleteKeyArgs.getKeys(i), new ErrorInfo(OMException.ResultCodes.INTERNAL_ERROR.name(), + ex.getMessage())); } omResponse.setDeleteKeysResponse( @@ -260,12 +267,18 @@ protected OMClientResponse getOmClientResponse(OzoneManager ozoneManager, List omKeyInfoList, List dirList, OMResponse.Builder omResponse, OzoneManagerProtocolProtos.DeleteKeyArgs.Builder unDeletedKeys, + Map keyToErrors, boolean deleteStatus, OmBucketInfo omBucketInfo, long volumeId, List dbOpenKeys) { OMClientResponse omClientResponse; + List deleteKeyErrors = new ArrayList<>(); + for (Map.Entry key : keyToErrors.entrySet()) { + deleteKeyErrors.add(OzoneManagerProtocolProtos.DeleteKeyError.newBuilder().setKey(key.getKey()) + .setErrorCode(key.getValue().getCode()).setErrorMsg(key.getValue().getMessage()).build()); + } omClientResponse = new OMKeysDeleteResponse(omResponse .setDeleteKeysResponse( DeleteKeysResponse.newBuilder().setStatus(deleteStatus) - .setUnDeletedKeys(unDeletedKeys)) + .setUnDeletedKeys(unDeletedKeys).addAllErrors(deleteKeyErrors)) .setStatus(deleteStatus ? OK : PARTIAL_DELETE).setSuccess(deleteStatus) .build(), omKeyInfoList, ozoneManager.isRatisEnabled(), omBucketInfo.copyObject(), dbOpenKeys); diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OmKeysDeleteRequestWithFSO.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OmKeysDeleteRequestWithFSO.java index 255a3db9fef..421d7d1a9bf 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OmKeysDeleteRequestWithFSO.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OmKeysDeleteRequestWithFSO.java @@ -24,6 +24,7 @@ import org.apache.hadoop.ozone.om.OMMetadataManager; import org.apache.hadoop.ozone.om.OzoneManager; import org.apache.hadoop.ozone.om.helpers.BucketLayout; +import org.apache.hadoop.ozone.om.helpers.ErrorInfo; import org.apache.hadoop.ozone.om.helpers.OmBucketInfo; import org.apache.hadoop.ozone.om.helpers.OmKeyInfo; import org.apache.hadoop.ozone.om.helpers.OzoneFileStatus; @@ -36,7 +37,9 @@ import org.slf4j.LoggerFactory; import java.io.IOException; +import java.util.ArrayList; import java.util.List; +import java.util.Map; import static org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.Status.OK; import static org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.Status.PARTIAL_DELETE; @@ -146,12 +149,19 @@ protected OMClientResponse getOmClientResponse(OzoneManager ozoneManager, List omKeyInfoList, List dirList, OzoneManagerProtocolProtos.OMResponse.Builder omResponse, OzoneManagerProtocolProtos.DeleteKeyArgs.Builder unDeletedKeys, + Map keyToErrors, boolean deleteStatus, OmBucketInfo omBucketInfo, long volumeId, List dbOpenKeys) { OMClientResponse omClientResponse; + List deleteKeyErrors = new ArrayList<>(); + for (Map.Entry key : keyToErrors.entrySet()) { + deleteKeyErrors.add(OzoneManagerProtocolProtos.DeleteKeyError.newBuilder() + .setKey(key.getKey()).setErrorCode(key.getValue().getCode()) + .setErrorMsg(key.getValue().getMessage()).build()); + } omClientResponse = new OMKeysDeleteResponseWithFSO(omResponse .setDeleteKeysResponse( OzoneManagerProtocolProtos.DeleteKeysResponse.newBuilder() - .setStatus(deleteStatus).setUnDeletedKeys(unDeletedKeys)) + .setStatus(deleteStatus).setUnDeletedKeys(unDeletedKeys).addAllErrors(deleteKeyErrors)) .setStatus(deleteStatus ? OK : PARTIAL_DELETE).setSuccess(deleteStatus) .build(), omKeyInfoList, dirList, ozoneManager.isRatisEnabled(), omBucketInfo.copyObject(), volumeId, dbOpenKeys); diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMKeysDeleteRequest.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMKeysDeleteRequest.java index d0cfd48e35d..3f90f153176 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMKeysDeleteRequest.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMKeysDeleteRequest.java @@ -23,6 +23,7 @@ import org.apache.hadoop.ozone.om.response.OMClientResponse; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.DeleteKeyArgs; +import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.DeleteKeyError; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.DeleteKeysRequest; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMRequest; import org.junit.jupiter.api.Test; @@ -73,6 +74,9 @@ protected void checkDeleteKeysResponse( omClientResponse.getOMResponse().getDeleteKeysResponse() .getUnDeletedKeys(); assertEquals(0, unDeletedKeys.getKeysCount()); + List keyErrors = omClientResponse.getOMResponse().getDeleteKeysResponse() + .getErrorsList(); + assertEquals(0, keyErrors.size()); // Check all keys are deleted. for (String deleteKey : deleteKeyList) { @@ -123,6 +127,9 @@ protected void checkDeleteKeysResponseForFailure( .getDeleteKeysResponse().getUnDeletedKeys(); assertEquals(1, unDeletedKeys.getKeysCount()); + List keyErrors = omClientResponse.getOMResponse().getDeleteKeysResponse() + .getErrorsList(); + assertEquals(1, keyErrors.size()); assertEquals("dummy", unDeletedKeys.getKeys(0)); } diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/BucketEndpoint.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/BucketEndpoint.java index ec434e4bb56..1b845c79aeb 100644 --- a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/BucketEndpoint.java +++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/BucketEndpoint.java @@ -28,6 +28,7 @@ import org.apache.hadoop.ozone.client.OzoneVolume; import org.apache.hadoop.ozone.om.exceptions.OMException; import org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes; +import org.apache.hadoop.ozone.om.helpers.ErrorInfo; import org.apache.hadoop.ozone.om.helpers.OzoneAclUtil; import org.apache.hadoop.ozone.s3.commontypes.EncodingTypeObject; import org.apache.hadoop.ozone.s3.commontypes.KeyMetadata; @@ -67,6 +68,7 @@ import java.util.HashSet; import java.util.Iterator; import java.util.List; +import java.util.Map; import java.util.Set; import static org.apache.hadoop.ozone.OzoneConsts.ETAG; @@ -445,47 +447,48 @@ public MultiDeleteResponse multiDelete(@PathParam("bucket") String bucketName, OzoneBucket bucket = getBucket(bucketName); MultiDeleteResponse result = new MultiDeleteResponse(); + List deleteKeys = new ArrayList<>(); + if (request.getObjects() != null) { + Map undeletedKeyResultMap; for (DeleteObject keyToDelete : request.getObjects()) { - long startNanos = Time.monotonicNowNanos(); - try { - bucket.deleteKey(keyToDelete.getKey()); - getMetrics().updateDeleteKeySuccessStats(startNanos); - - if (!request.isQuiet()) { - result.addDeleted(new DeletedObject(keyToDelete.getKey())); - } - } catch (OMException ex) { - if (isAccessDenied(ex)) { - getMetrics().updateDeleteKeyFailureStats(startNanos); - result.addError( - new Error(keyToDelete.getKey(), "PermissionDenied", - ex.getMessage())); - } else if (ex.getResult() != ResultCodes.KEY_NOT_FOUND) { - getMetrics().updateDeleteKeyFailureStats(startNanos); - result.addError( - new Error(keyToDelete.getKey(), "InternalError", - ex.getMessage())); - } else { + deleteKeys.add(keyToDelete.getKey()); + } + long startNanos = Time.monotonicNowNanos(); + try { + undeletedKeyResultMap = bucket.deleteKeys(deleteKeys, true); + for (DeleteObject d : request.getObjects()) { + ErrorInfo error = undeletedKeyResultMap.get(d.getKey()); + boolean deleted = error == null || + // if the key is not found, it is assumed to be successfully deleted + ResultCodes.KEY_NOT_FOUND.name().equals(error.getCode()); + if (deleted) { + deleteKeys.remove(d.getKey()); if (!request.isQuiet()) { - result.addDeleted(new DeletedObject(keyToDelete.getKey())); + result.addDeleted(new DeletedObject(d.getKey())); } - getMetrics().updateDeleteKeySuccessStats(startNanos); + } else { + result.addError(new Error(d.getKey(), error.getCode(), error.getMessage())); } - } catch (Exception ex) { - getMetrics().updateDeleteKeyFailureStats(startNanos); - result.addError( - new Error(keyToDelete.getKey(), "InternalError", - ex.getMessage())); } + getMetrics().updateDeleteKeySuccessStats(startNanos); + } catch (IOException ex) { + LOG.error("Delete key failed: {}", ex.getMessage()); + getMetrics().updateDeleteKeyFailureStats(startNanos); + result.addError( + new Error("ALL", "InternalError", + ex.getMessage())); } } + + Map auditMap = getAuditParameters(); + auditMap.put("failedDeletes", deleteKeys.toString()); if (result.getErrors().size() != 0) { AUDIT.logWriteFailure(buildAuditMessageForFailure(s3GAction, - getAuditParameters(), new Exception("MultiDelete Exception"))); + auditMap, new Exception("MultiDelete Exception"))); } else { AUDIT.logWriteSuccess( - buildAuditMessageForSuccess(s3GAction, getAuditParameters())); + buildAuditMessageForSuccess(s3GAction, auditMap)); } return result; } diff --git a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/client/ClientProtocolStub.java b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/client/ClientProtocolStub.java index e70e2215597..b75167c89ca 100644 --- a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/client/ClientProtocolStub.java +++ b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/client/ClientProtocolStub.java @@ -33,6 +33,7 @@ import org.apache.hadoop.ozone.client.io.OzoneOutputStream; import org.apache.hadoop.ozone.client.protocol.ClientProtocol; import org.apache.hadoop.ozone.om.helpers.DeleteTenantState; +import org.apache.hadoop.ozone.om.helpers.ErrorInfo; import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfo; import org.apache.hadoop.ozone.om.helpers.OmMultipartInfo; import org.apache.hadoop.ozone.om.helpers.OmMultipartUploadCompleteInfo; @@ -58,6 +59,7 @@ import java.io.IOException; import java.net.URI; import java.util.Collections; +import java.util.HashMap; import java.util.List; import java.util.Map; @@ -261,6 +263,13 @@ public void deleteKeys(String volumeName, String bucketName, } + @Override + public Map deleteKeys(String volumeName, String bucketName, + List keyNameList, boolean quiet) + throws IOException { + return new HashMap<>(); + } + @Override public void renameKey(String volumeName, String bucketName, String fromKeyName, String toKeyName) diff --git a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/client/OzoneBucketStub.java b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/client/OzoneBucketStub.java index 22b002945eb..3320801874d 100644 --- a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/client/OzoneBucketStub.java +++ b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/client/OzoneBucketStub.java @@ -52,6 +52,7 @@ import org.apache.hadoop.ozone.client.OzoneMultipartUploadPartListParts.PartInfo; import org.apache.hadoop.ozone.om.exceptions.OMException; import org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes; +import org.apache.hadoop.ozone.om.helpers.ErrorInfo; import org.apache.hadoop.ozone.om.helpers.OmMultipartInfo; import org.apache.hadoop.ozone.om.helpers.OmMultipartUploadCompleteInfo; import org.apache.hadoop.security.UserGroupInformation; @@ -358,6 +359,17 @@ public void deleteKey(String key) throws IOException { keyDetails.remove(key); } + @Override + public Map deleteKeys(List keyList, boolean quiet) throws IOException { + Map keyErrorMap = new HashMap<>(); + for (String key : keyList) { + if (keyDetails.remove(key) == null) { + keyErrorMap.put(key, new ErrorInfo("KEY_NOT_FOUND", "Key does not exist")); + } + } + return keyErrorMap; + } + @Override public void renameKey(String fromKeyName, String toKeyName) throws IOException { diff --git a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestPermissionCheck.java b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestPermissionCheck.java index 04551ac7cc4..b74808de953 100644 --- a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestPermissionCheck.java +++ b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestPermissionCheck.java @@ -27,6 +27,7 @@ import org.apache.hadoop.ozone.client.OzoneVolume; import org.apache.hadoop.ozone.client.protocol.ClientProtocol; import org.apache.hadoop.ozone.om.exceptions.OMException; +import org.apache.hadoop.ozone.om.helpers.ErrorInfo; import org.apache.hadoop.ozone.s3.exception.OS3Exception; import org.apache.hadoop.ozone.s3.metrics.S3GatewayMetrics; import org.junit.jupiter.api.BeforeEach; @@ -38,6 +39,7 @@ import java.io.ByteArrayInputStream; import java.io.IOException; import java.util.ArrayList; +import java.util.HashMap; import java.util.List; import java.util.Map; @@ -167,7 +169,10 @@ public void testListKey() throws IOException { public void testDeleteKeys() throws IOException, OS3Exception { when(objectStore.getVolume(anyString())).thenReturn(volume); when(objectStore.getS3Bucket(anyString())).thenReturn(bucket); - doThrow(exception).when(bucket).deleteKey(any()); + Map deleteErrors = new HashMap<>(); + deleteErrors.put("deleteKeyName", new ErrorInfo("ACCESS_DENIED", "ACL check failed")); + when(bucket.deleteKeys(any(), anyBoolean())).thenReturn(deleteErrors); + BucketEndpoint bucketEndpoint = new BucketEndpoint(); bucketEndpoint.setClient(client); MultiDeleteRequest request = new MultiDeleteRequest(); @@ -179,7 +184,7 @@ public void testDeleteKeys() throws IOException, OS3Exception { MultiDeleteResponse response = bucketEndpoint.multiDelete("BucketName", "keyName", request); assertEquals(1, response.getErrors().size()); - assertEquals("PermissionDenied", response.getErrors().get(0).getCode()); + assertEquals("ACCESS_DENIED", response.getErrors().get(0).getCode()); } @Test