Skip to content

Commit

Permalink
HDDS-10570. S3A: fs -touch creates directory instead of empty file …
Browse files Browse the repository at this point in the history 
…in FSO bucket (apache#6452)
  • Loading branch information
@adoroszlai
adoroszlai authored Mar 29, 2024
1 parent 9f6ac2f commit 85c9c97
Show file tree
Hide file tree
Showing 6 changed files with 36 additions and 49 deletions.
8 changes: 5 additions & 3 deletions hadoop-ozone/dist/src/main/compose/ozone-ha/test.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -38,9 +38,11 @@ execute_robot_test ${SCM} -v SCHEME:ofs -v BUCKET_TYPE:link -N ozonefs-ofs-link
## Exclude virtual-host tests. This is tested separately as it requires additional config.
exclude="--exclude virtual-host"
for bucket in generated; do
execute_robot_test ${SCM} -v BUCKET:${bucket} -N s3-${bucket} ${exclude} s3
# some tests are independent of the bucket type, only need to be run once
exclude="--exclude virtual-host --exclude no-bucket-type"
for layout in OBJECT_STORE LEGACY FILE_SYSTEM_OPTIMIZED; do
execute_robot_test ${SCM} -v BUCKET:${bucket} -v BUCKET_LAYOUT:${layout} -N s3-${layout}-${bucket} ${exclude} s3
# some tests are independent of the bucket type, only need to be run once
exclude="--exclude virtual-host --exclude no-bucket-type"
done
done

execute_robot_test ${SCM} freon
Expand Down
24 changes: 11 additions & 13 deletions hadoop-ozone/dist/src/main/smoketest/s3/commonawslib.robot
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@ ${ENDPOINT_URL} http://s3g:9878
${OZONE_S3_HEADER_VERSION} v4
${OZONE_S3_SET_CREDENTIALS} true
${BUCKET} generated
${BUCKET_LAYOUT} OBJECT_STORE
${KEY_NAME} key1
${OZONE_S3_TESTS_SET_UP} ${FALSE}
${OZONE_AWS_ACCESS_KEY_ID} ${EMPTY}
Expand Down Expand Up @@ -127,24 +128,20 @@ Create bucket with name
${result} = Execute AWSS3APICli create-bucket --bucket ${bucket}
Should contain ${result} Location
Should contain ${result} ${bucket}
Create legacy bucket
${postfix} = Generate Ozone String
${legacy_bucket} = Set Variable legacy-bucket-${postfix}
${result} = Execute and checkrc ozone sh bucket create -l LEGACY s3v/${legacy_bucket} 0
[Return] ${legacy_bucket}

Create obs bucket
Create bucket with layout
[Arguments] ${layout}
${postfix} = Generate Ozone String
${bucket} = Set Variable obs-bucket-${postfix}
${result} = Execute and checkrc ozone sh bucket create -l OBJECT_STORE s3v/${bucket} 0
${bucket} = Set Variable bucket-${postfix}
${result} = Execute ozone sh bucket create --layout ${layout} s3v/${bucket}
[Return] ${bucket}

Setup s3 tests
Return From Keyword if ${OZONE_S3_TESTS_SET_UP}
Run Keyword Generate random prefix
Run Keyword Install aws cli
Run Keyword if '${OZONE_S3_SET_CREDENTIALS}' == 'true' Setup v4 headers
Run Keyword if '${BUCKET}' == 'generated' Create generated bucket
Run Keyword if '${BUCKET}' == 'generated' Create generated bucket ${BUCKET_LAYOUT}
Run Keyword if '${BUCKET}' == 'link' Setup links for S3 tests
Run Keyword if '${BUCKET}' == 'encrypted' Create encrypted bucket
Run Keyword if '${BUCKET}' == 'erasure' Create EC bucket
Expand All @@ -154,18 +151,19 @@ Setup links for S3 tests
${exists} = Bucket Exists o3://${OM_SERVICE_ID}/s3v/link
Return From Keyword If ${exists}
Execute ozone sh volume create o3://${OM_SERVICE_ID}/legacy
Execute ozone sh bucket create o3://${OM_SERVICE_ID}/legacy/source-bucket
Execute ozone sh bucket create --layout ${BUCKET_LAYOUT} o3://${OM_SERVICE_ID}/legacy/source-bucket
Create link link

Create generated bucket
${BUCKET} = Create bucket
[Arguments] ${layout}=OBJECT_STORE
${BUCKET} = Create bucket with layout ${layout}
Set Global Variable ${BUCKET}

Create encrypted bucket
Return From Keyword if '${SECURITY_ENABLED}' == 'false'
${exists} = Bucket Exists o3://${OM_SERVICE_ID}/s3v/encrypted
Return From Keyword If ${exists}
Execute ozone sh bucket create -k ${KEY_NAME} o3://${OM_SERVICE_ID}/s3v/encrypted
Execute ozone sh bucket create -k ${KEY_NAME} --layout ${BUCKET_LAYOUT} o3://${OM_SERVICE_ID}/s3v/encrypted

Create link
[arguments] ${bucket}
Expand All @@ -175,7 +173,7 @@ Create link
Create EC bucket
${exists} = Bucket Exists o3://${OM_SERVICE_ID}/s3v/erasure
Return From Keyword If ${exists}
Execute ozone sh bucket create --replication rs-3-2-1024k --type EC o3://${OM_SERVICE_ID}/s3v/erasure
Execute ozone sh bucket create --replication rs-3-2-1024k --type EC --layout ${BUCKET_LAYOUT} o3://${OM_SERVICE_ID}/s3v/erasure

Generate random prefix
${random} = Generate Ozone String
Expand Down
17 changes: 9 additions & 8 deletions hadoop-ozone/dist/src/main/smoketest/s3/objecthead.robot
View file
Original file line number Diff line number Diff line change
Expand Up @@ -40,22 +40,23 @@ Head object in non existing bucket
${result} = Execute AWSS3APICli and checkrc head-object --bucket ${BUCKET}-non-existent --key ${PREFIX}/headobject/key=value/f1 255
Should contain ${result} 404
Should contain ${result} Not Found

Head object where path is a directory
${legacy-bucket} = Create legacy bucket
${result} = Execute AWSS3APICli and checkrc put-object --bucket ${legacy-bucket} --key ${PREFIX}/headobject/keyvalue/f1 --body /tmp/testfile 0
${result} = Execute AWSS3APICli and checkrc head-object --bucket ${legacy-bucket} --key ${PREFIX}/headobject/keyvalue/ 255
Pass Execution If '${BUCKET_LAYOUT}' == 'FILE_SYSTEM_OPTIMIZED' does not apply to FSO buckets
${result} = Execute AWSS3APICli and checkrc put-object --bucket ${BUCKET} --key ${PREFIX}/headobject/keyvalue/f1 --body /tmp/testfile 0
${result} = Execute AWSS3APICli and checkrc head-object --bucket ${BUCKET} --key ${PREFIX}/headobject/keyvalue/ 255
Should contain ${result} 404
Should contain ${result} Not Found

Head directory objects
${obs-bucket} = Create obs bucket
${result} = Execute AWSS3APICli and checkrc put-object --bucket ${obs-bucket} --key ${PREFIX}/mydir/ --body /tmp/testfile 0
${result} = Execute AWSS3APICli and checkrc head-object --bucket ${obs-bucket} --key ${PREFIX}/mydir 255
Pass Execution If '${BUCKET_LAYOUT}' == 'FILE_SYSTEM_OPTIMIZED' does not apply to FSO buckets
${result} = Execute AWSS3APICli and checkrc put-object --bucket ${BUCKET} --key ${PREFIX}/mydir/ --body /tmp/testfile 0
${result} = Execute AWSS3APICli and checkrc head-object --bucket ${BUCKET} --key ${PREFIX}/mydir 255
Should contain ${result} 404
Should contain ${result} Not Found
${result} = Execute AWSS3APICli and checkrc head-object --bucket ${obs-bucket} --key ${PREFIX}/mydir/ 0
${result} = Execute AWSS3APICli and checkrc head-object --bucket ${BUCKET} --key ${PREFIX}/mydir/ 0

Head non existing key
${result} = Execute AWSS3APICli and checkrc head-object --bucket ${BUCKET} --key ${PREFIX}/non-existent 255
Should contain ${result} 404
Should contain ${result} Not Found
Should contain ${result} Not Found
24 changes: 3 additions & 21 deletions hadoop-ozone/dist/src/main/smoketest/s3/objectputget.robot
View file
Original file line number Diff line number Diff line change
Expand Up @@ -44,6 +44,8 @@ Put object to s3
Get object from s3
${result} = Execute AWSS3ApiCli get-object --bucket ${BUCKET} --key ${PREFIX}/putobject/key=value/f1 /tmp/testfile.result
Compare files /tmp/testfile /tmp/testfile.result
${result} = Execute AWSS3ApiCli get-object --bucket ${BUCKET} --key ${PREFIX}/putobject/key=value/zerobyte /tmp/zerobyte.result
Compare files /tmp/zerobyte /tmp/zerobyte.result

#This test depends on the previous test case. Can't be executed alone
Get object with wrong signature
Expand Down Expand Up @@ -151,34 +153,14 @@ Incorrect values for end and start offset
Should Be Equal ${expectedData} ${actualData}

Zero byte file
${result} = Execute ozone sh bucket info /s3v/${BUCKET}
${linked} = Execute echo '${result}' | jq -j '.sourceVolume,"/",.sourceBucket'
${eval} = Evaluate "source" in """${linked}"""
IF ${eval} == ${True}
${result} = Execute ozone sh bucket info ${linked}
END
${fsolayout} = Evaluate "OPTIMIZED" in """${result}"""

${result} = Execute AWSS3APICli and checkrc get-object --bucket ${BUCKET} --key ${PREFIX}/putobject/key=value/zerobyte --range bytes=0-0 /tmp/testfile2.result 255
IF ${fsolayout} == ${True}
Should contain ${result} NoSuchKey
ELSE
Should contain ${result} InvalidRange
END

${result} = Execute AWSS3APICli and checkrc get-object --bucket ${BUCKET} --key ${PREFIX}/putobject/key=value/zerobyte --range bytes=0-1 /tmp/testfile2.result 255
IF ${fsolayout} == ${True}
Should contain ${result} NoSuchKey
ELSE
Should contain ${result} InvalidRange
END

${result} = Execute AWSS3APICli and checkrc get-object --bucket ${BUCKET} --key ${PREFIX}/putobject/key=value/zerobyte --range bytes=0-10000 /tmp/testfile2.result 255
IF ${fsolayout} == ${True}
Should contain ${result} NoSuchKey
ELSE
Should contain ${result} InvalidRange
END

Create file with user defined metadata
Execute echo "Randomtext" > /tmp/testfile2
Expand Down Expand Up @@ -257,4 +239,4 @@ Create key twice with different content and expect different ETags
# clean up
Execute AWSS3Cli rm s3://${BUCKET}/test_key_to_check_etag_differences
Execute rm -rf /tmp/file1
Execute rm -rf /tmp/file2
Execute rm -rf /tmp/file2
4 changes: 3 additions & 1 deletion hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpoint.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -274,7 +274,9 @@ public Response put(
boolean hasAmzDecodedLengthZero = amzDecodedLength != null &&
Long.parseLong(amzDecodedLength) == 0;
if (canCreateDirectory &&
(length == 0 || hasAmzDecodedLengthZero)) {
(length == 0 || hasAmzDecodedLengthZero) &&
StringUtils.endsWith(keyPath, "/")
) {
s3GAction = S3GAction.CREATE_DIRECTORY;
getClientProtocol()
.createDirectory(volume.getName(), bucketName, keyPath);
Expand Down
8 changes: 5 additions & 3 deletions hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestObjectPut.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -95,7 +95,9 @@ static Stream<Arguments> argumentsForPutObject() {
ReplicationConfig ratis3 = RatisReplicationConfig.getInstance(HddsProtos.ReplicationFactor.THREE);
ECReplicationConfig ec = new ECReplicationConfig("rs-3-2-1024K");
return Stream.of(
Arguments.of(0, ratis3),
Arguments.of(10, ratis3),
Arguments.of(0, ec),
Arguments.of(10, ec)
);
}
Expand Down Expand Up @@ -410,7 +412,7 @@ void testEmptyStorageType() throws IOException, OS3Exception {
void testDirectoryCreation() throws IOException,
OS3Exception {
// GIVEN
final String path = "dir";
final String path = "dir/";

// WHEN
try (Response response = objectEndpoint.put(fsoBucket.getName(), path,
Expand All @@ -426,15 +428,15 @@ void testDirectoryCreation() throws IOException,
@Test
void testDirectoryCreationOverFile() throws IOException, OS3Exception {
// GIVEN
final String path = "dir";
final String path = "key";
final ByteArrayInputStream body =
new ByteArrayInputStream(CONTENT.getBytes(UTF_8));
objectEndpoint.put(FSO_BUCKET_NAME, path, CONTENT.length(), 0, "", body);

// WHEN
final OS3Exception exception = assertThrows(OS3Exception.class,
() -> objectEndpoint
.put(FSO_BUCKET_NAME, path, 0, 0, "", null)
.put(FSO_BUCKET_NAME, path + "/", 0, 0, "", null)
.close());

// THEN
Expand Down

0 comments on commit 85c9c97

Please sign in to comment.