Skip to content

Commit

Permalink
Merge pull request #22538 from vbotbuildovich/backport-pr-22331-v24.2…
Browse files Browse the repository at this point in the history 
….x-824

[v24.2.x] cloud_storage_clients: fix `abs_parse_impl` error
  • Loading branch information
@WillemKauf
WillemKauf authored Jul 27, 2024
2 parents ff04b65 + 7f312e7 commit 0182e7b
Show file tree
Hide file tree
Showing 2 changed files with 114 additions and 67 deletions.
177 changes: 111 additions & 66 deletions src/v/cloud_storage_clients/tests/xml_sax_parser_test.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -44,47 +44,47 @@ static constexpr std::string_view payload = R"XML(
)XML";

static constexpr std::string_view abs_payload = R"XML(
<EnumerationResults ServiceEndpoint="http://myaccount.blob.core.windows.net/" ContainerName="mycontainer">
<Prefix>prefix</Prefix>
<Marker>string-value</Marker>
<MaxResults>int-value</MaxResults>
<Delimiter>string-value</Delimiter>
<Blobs>
<Blob>
<Name>blob-name</Name>
<Snapshot>date-time-value</Snapshot>
<EnumerationResults ServiceEndpoint="http://myaccount.blob.core.windows.net/" ContainerName="mycontainer">
<Prefix>prefix</Prefix>
<Marker>string-value</Marker>
<MaxResults>int-value</MaxResults>
<Delimiter>string-value</Delimiter>
<Blobs>
<Blob>
<Name>blob-name</Name>
<Snapshot>date-time-value</Snapshot>
<VersionId>date-time-vlue</VersionId>
<IsCurrentVersion>true</IsCurrentVersion>
<Deleted>true</Deleted>
<Properties>
<Properties>
<Creation-Time>date-time-value</Creation-Time>
<Last-Modified>2021-01-10T02:00:00.000Z</Last-Modified>
<Last-Modified>2021-01-10T02:00:00.000Z</Last-Modified>
<Etag>etag</Etag>
<Owner>owner user id</Owner>
<Group>owning group id</Group>
<Permissions>permission string</Permissions>
<Acl>access control list</Acl>
<ResourceType>file | directory</ResourceType>
<Placeholder>true</Placeholder>
<Content-Length>1112</Content-Length>
<Content-Type>blob-content-type</Content-Type>
<Content-Encoding />
<Content-Language />
<Content-MD5 />
<Cache-Control />
<x-ms-blob-sequence-number>sequence-number</x-ms-blob-sequence-number>
<BlobType>BlockBlob|PageBlob|AppendBlob</BlobType>
<AccessTier>tier</AccessTier>
<LeaseStatus>locked|unlocked</LeaseStatus>
<LeaseState>available | leased | expired | breaking | broken</LeaseState>
<LeaseDuration>infinite | fixed</LeaseDuration>
<CopyId>id</CopyId>
<CopyStatus>pending | success | aborted | failed </CopyStatus>
<CopySource>source url</CopySource>
<CopyProgress>bytes copied/bytes total</CopyProgress>
<CopyCompletionTime>datetime</CopyCompletionTime>
<CopyStatusDescription>error string</CopyStatusDescription>
<ServerEncrypted>true</ServerEncrypted>
<Content-Length>1112</Content-Length>
<Content-Type>blob-content-type</Content-Type>
<Content-Encoding />
<Content-Language />
<Content-MD5 />
<Cache-Control />
<x-ms-blob-sequence-number>sequence-number</x-ms-blob-sequence-number>
<BlobType>BlockBlob|PageBlob|AppendBlob</BlobType>
<AccessTier>tier</AccessTier>
<LeaseStatus>locked|unlocked</LeaseStatus>
<LeaseState>available | leased | expired | breaking | broken</LeaseState>
<LeaseDuration>infinite | fixed</LeaseDuration>
<CopyId>id</CopyId>
<CopyStatus>pending | success | aborted | failed </CopyStatus>
<CopySource>source url</CopySource>
<CopyProgress>bytes copied/bytes total</CopyProgress>
<CopyCompletionTime>datetime</CopyCompletionTime>
<CopyStatusDescription>error string</CopyStatusDescription>
<ServerEncrypted>true</ServerEncrypted>
<CustomerProvidedKeySha256>encryption-key-sha256</CustomerProvidedKeySha256>
<EncryptionScope>encryption-scope-name</EncryptionScope>
<IncrementalCopy>true</IncrementalCopy>
Expand All @@ -95,10 +95,10 @@ static constexpr std::string_view abs_payload = R"XML(
<TagCount>number of tags between 1 to 10</TagCount>
<RehydratePriority>rehydrate priority</RehydratePriority>
<Expiry-Time>date-time-value</Expiry-Time>
</Properties>
<Metadata>
<Name>value</Name>
</Metadata>
</Properties>
<Metadata>
<Name>value</Name>
</Metadata>
<Tags>
<TagSet>
<Tag>
Expand All @@ -108,53 +108,81 @@ static constexpr std::string_view abs_payload = R"XML(
</TagSet>
</Tags>
<OrMetadata />
</Blob>
<BlobPrefix>
<Name>blob-prefix</Name>
</BlobPrefix>
</Blobs>
<NextMarker />
</Blob>
<BlobPrefix>
<Name>blob-prefix</Name>
</BlobPrefix>
</Blobs>
<NextMarker />
</EnumerationResults>
)XML";

static constexpr std::string_view abs_payload_with_continuation = R"XML(
<EnumerationResults ServiceEndpoint="http://myaccount.blob.core.windows.net/" ContainerName="mycontainer">
<Prefix>prefix</Prefix>
<Marker>string-value</Marker>
<MaxResults>int-value</MaxResults>
<Delimiter>string-value</Delimiter>
<Blobs>
<Blob>
<Name>blob-name</Name>
<Snapshot>date-time-value</Snapshot>
<EnumerationResults ServiceEndpoint="http://myaccount.blob.core.windows.net/" ContainerName="mycontainer">
<Prefix>prefix</Prefix>
<Marker>string-value</Marker>
<MaxResults>int-value</MaxResults>
<Delimiter>string-value</Delimiter>
<Blobs>
<Blob>
<Name>blob-name</Name>
<Snapshot>date-time-value</Snapshot>
<VersionId>date-time-vlue</VersionId>
<IsCurrentVersion>true</IsCurrentVersion>
<Deleted>true</Deleted>
<Properties>
<Properties>
<Creation-Time>date-time-value</Creation-Time>
<Last-Modified>2021-01-10T02:00:00.000Z</Last-Modified>
<Last-Modified>2021-01-10T02:00:00.000Z</Last-Modified>
<Etag>etag</Etag>
<Acl>access control list</Acl>
<ResourceType>file | directory</ResourceType>
<Placeholder>true</Placeholder>
<Content-Length>1112</Content-Length>
<Content-Type>blob-content-type</Content-Type>
<Content-Encoding />
<Content-Language />
<Content-MD5 />
<Cache-Control />
<x-ms-blob-sequence-number>sequence-number</x-ms-blob-sequence-number>
<BlobType>BlockBlob|PageBlob|AppendBlob</BlobType>
<Content-Length>1112</Content-Length>
<Content-Type>blob-content-type</Content-Type>
<Content-Encoding />
<Content-Language />
<Content-MD5 />
<Cache-Control />
<x-ms-blob-sequence-number>sequence-number</x-ms-blob-sequence-number>
<BlobType>BlockBlob|PageBlob|AppendBlob</BlobType>
<RemainingRetentionDays>no-of-days</RemainingRetentionDays>
<TagCount>number of tags between 1 to 10</TagCount>
<Expiry-Time>date-time-value</Expiry-Time>
</Properties>
</Blob>
<BlobPrefix>
<Name>blob-prefix</Name>
</BlobPrefix>
</Blobs>
<NextMarker>nnn</NextMarker>
</Properties>
</Blob>
<BlobPrefix>
<Name>blob-prefix</Name>
</BlobPrefix>
</Blobs>
<NextMarker>nnn</NextMarker>
</EnumerationResults>
)XML";

static constexpr std::string_view abs_payload_with_blob_prefix = R"XML(
<EnumerationResults ServiceEndpoint="http://myaccount.blob.core.windows.net/" ContainerName="mycontainer">
<Prefix>prefix</Prefix>
<Marker>string-value</Marker>
<MaxResults>int-value</MaxResults>
<Delimiter>string-value</Delimiter>
<Blobs>
<BlobPrefix>
<Name>cluster_metadata/bb7527f1-3227-4d55-86da-c133ec955ea9/manifests/2/</Name>
<Properties>
<Creation-Time>Thu, 25 Jul 2024 14:07:26 GMT</Creation-Time>
<Last-Modified>Thu, 25 Jul 2024 14:07:26 GMT</Last-Modified>
<Etag>0x8DCACB31CE7DB5C</Etag>
<ResourceType>directory</ResourceType>
<Content-Length>0</Content-Length>
<BlobType>BlockBlob</BlobType>
<AccessTier>Hot</AccessTier>
<AccessTierInferred>true</AccessTierInferred>
<LeaseStatus>unlocked</LeaseStatus>
<LeaseState>available</LeaseState>
<ServerEncrypted>true</ServerEncrypted>
</Properties>
</BlobPrefix>
</Blobs>
<NextMarker />
</EnumerationResults>
)XML";

Expand Down Expand Up @@ -247,3 +275,20 @@ BOOST_AUTO_TEST_CASE(test_parse_abs_with_continuation) {
BOOST_REQUIRE_EQUAL(result.is_truncated, true);
BOOST_REQUIRE_EQUAL(result.next_continuation_token, "nnn");
}

BOOST_AUTO_TEST_CASE(test_parse_abs_with_blob_prefix) {
cloud_storage_clients::xml_sax_parser p{};
ss::temporary_buffer<char> buffer(
abs_payload_with_blob_prefix.data(), abs_payload_with_blob_prefix.size());

p.start_parse(std::make_unique<cloud_storage_clients::abs_parse_impl>());
p.parse_chunk(std::move(buffer));
p.end_parse();

auto result = p.result();
BOOST_REQUIRE(result.contents.empty());
BOOST_REQUIRE_EQUAL(result.common_prefixes.size(), 1);
BOOST_REQUIRE_EQUAL(
result.common_prefixes[0],
"cluster_metadata/bb7527f1-3227-4d55-86da-c133ec955ea9/manifests/2/");
}
4 changes: 3 additions & 1 deletion src/v/cloud_storage_clients/xml_sax_parser.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -190,7 +190,9 @@ bool abs_parse_impl::is_in_blob_prefixes() const {
}

void abs_parse_impl::handle_start_element(std::string_view element_name) {
if (element_name == abs_tags::blob && _tags.size() == 2) {
if (
(element_name == abs_tags::blob || element_name == abs_tags::blob_prefix)
&& _tags.size() == 2) {
// Reinitialize the item in preparation for next values
_current_item.emplace();
} else if (element_name == abs_tags::name) {
Expand Down

0 comments on commit 0182e7b

Please sign in to comment.