Skip to content

Commit

Permalink
PageCtl: Support getting blob data from PageStorage instance (release…
Browse files Browse the repository at this point in the history 
…-8.1) (#9720)

close #9716

PageCtl: Support getting blob data from PageStorage instance
And output the "PageEntryV3.offset" in decimal instead of hexadecimal

Signed-off-by: JaySon-Huang <tshent@qq.com>
  • Loading branch information
@JaySon-Huang
JaySon-Huang authored Dec 11, 2024
1 parent 6925b61 commit bd3762c
Show file tree
Hide file tree
Showing 3 changed files with 59 additions and 8 deletions.
2 changes: 1 addition & 1 deletion dbms/src/Storages/Page/V3/PageEntry.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -135,7 +135,7 @@ struct fmt::formatter<DB::PS::V3::PageEntryV3>

return fmt::format_to(
ctx.out(),
"PageEntry{{file: {}, offset: 0x{:X}, size: {}, checksum: 0x{:X}, tag: {}, field_offsets: [{}], "
"PageEntry{{file: {}, offset: {}, size: {}, checksum: 0x{:X}, tag: {}, field_offsets: [{}], "
"checkpoint_info: {}}}",
entry.file_id,
entry.offset,
Expand Down
62 changes: 55 additions & 7 deletions dbms/src/Storages/Page/tools/PageCtl/PageStorageCtlV3.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,10 +17,13 @@
#include <Poco/ConsoleChannel.h>
#include <Poco/PatternFormatter.h>
#include <Server/CLIService.h>
#include <Storages/Page/PageDefinesBase.h>
#include <Storages/Page/V3/PageDefines.h>
#include <Storages/Page/V3/PageDirectory.h>
#include <Storages/Page/V3/PageDirectoryFactory.h>
#include <Storages/Page/V3/PageStorageImpl.h>
#include <Storages/Page/V3/Universal/RaftDataReader.h>
#include <Storages/Page/V3/Universal/UniversalPageId.h>
#include <Storages/Page/V3/Universal/UniversalPageIdFormatImpl.h>
#include <Storages/Page/V3/Universal/UniversalPageStorage.h>
#include <Storages/PathPool.h>
Expand All @@ -29,6 +32,7 @@
#include <common/types.h>

#include <boost/program_options.hpp>
#include <cstdint>
#include <magic_enum.hpp>
#include <unordered_set>

Expand All @@ -47,12 +51,15 @@ struct ControlOptions
CHECK_ALL_DATA_CRC = 4,
DISPLAY_WAL_ENTRIES = 5,
DISPLAY_REGION_INFO = 6,
DISPLAY_BLOB_DATA = 7,
};

std::vector<std::string> paths;
DisplayType mode = DisplayType::DISPLAY_SUMMARY_INFO;
UInt64 page_id = UINT64_MAX;
UInt32 blob_id = UINT32_MAX;
BlobFileId blob_id = INVALID_BLOBFILE_ID;
BlobFileOffset blob_offset = INVALID_BLOBFILE_OFFSET;
size_t blob_size = UINT64_MAX;
UInt64 namespace_id = DB::TEST_NAMESPACE_ID;
StorageType storage_type = StorageType::Unknown; // only useful for universal page storage
UInt32 keyspace_id = NullspaceID; // only useful for universal page storage
Expand Down Expand Up @@ -85,6 +92,7 @@ ControlOptions ControlOptions::parse(int argc, char ** argv)
4 is check every data is valid
5 is dump entries in WAL log files
6 is display all region info
7 is display blob data (in hex)
)") //
("show_entries",
value<bool>()->default_value(true),
Expand All @@ -106,8 +114,14 @@ ControlOptions ControlOptions::parse(int argc, char ** argv)
value<UInt64>()->default_value(UINT64_MAX),
"Query a single Page id, and print its version chain.") //
("blob_id,B",
value<UInt32>()->default_value(UINT32_MAX),
"Query a single Blob id, and print its data distribution.") //
value<BlobFileId>()->default_value(INVALID_BLOBFILE_ID),
"Specify the blob_id") //
("blob_offset",
value<BlobFileOffset>()->default_value(INVALID_BLOBFILE_OFFSET),
"Specify the offset.") //
("blob_size",
value<size_t>()->default_value(0),
"Specify the size.") //
//
("imitative,I",
value<bool>()->default_value(true),
Expand Down Expand Up @@ -140,7 +154,9 @@ ControlOptions ControlOptions::parse(int argc, char ** argv)
opt.paths = options["paths"].as<std::vector<std::string>>();
auto mode_int = options["mode"].as<int>();
opt.page_id = options["page_id"].as<UInt64>();
opt.blob_id = options["blob_id"].as<UInt32>();
opt.blob_id = options["blob_id"].as<BlobFileId>();
opt.blob_offset = options["blob_offset"].as<BlobFileOffset>();
opt.blob_size = options["blob_size"].as<size_t>();
opt.show_entries = options["show_entries"].as<bool>();
opt.check_fields = options["check_fields"].as<bool>();
auto storage_type_int = options["storage_type"].as<int>();
Expand Down Expand Up @@ -346,6 +362,12 @@ class PageStorageControlV3
}
break;
}
case ControlOptions::DisplayType::DISPLAY_BLOB_DATA:
{
String hex_data = getBlobData(blob_store, opts.blob_id, opts.blob_offset, opts.blob_size);
fmt::print("hex:{}\n", hex_data);
break;
}
default:
std::cout << "Invalid display mode." << std::endl;
break;
Expand All @@ -372,7 +394,7 @@ class PageStorageControlV3
return 0;
}

static String getBlobsInfo(typename Trait::BlobStore & blob_store, UInt32 blob_id)
static String getBlobsInfo(typename Trait::BlobStore & blob_store, BlobFileId blob_id)
{
auto stat_info = [](const BlobStats::BlobStatPtr & stat, const String & path) {
FmtBuffer stat_str;
Expand Down Expand Up @@ -402,7 +424,7 @@ class PageStorageControlV3
{
for (const auto & stat : stats)
{
if (blob_id != UINT32_MAX)
if (blob_id != INVALID_BLOBFILE_ID)
{
if (stat->id == blob_id)
{
Expand All @@ -416,7 +438,7 @@ class PageStorageControlV3
}
}

if (blob_id != UINT32_MAX)
if (blob_id != INVALID_BLOBFILE_ID)
{
stats_info.fmtAppend(" no found blob {}", blob_id);
}
Expand Down Expand Up @@ -821,6 +843,32 @@ class PageStorageControlV3
return error_msg.toString();
}

static String getBlobData(
typename Trait::BlobStore & blob_store,
BlobFileId blob_id,
BlobFileOffset offset,
size_t size)
{
auto page_id = []() {
if constexpr (std::is_same_v<Trait, u128::PageStorageControlV3Trait>)
return PageIdV3Internal(0, 0);
else
return UniversalPageId("");
}();
char * buffer = new char[size];
blob_store.read(page_id, blob_id, offset, buffer, size, nullptr, false);

using ChecksumClass = Digest::CRC64;
ChecksumClass digest;
digest.update(buffer, size);
auto checksum = digest.checksum();
fmt::print("checksum: 0x{:X}\n", checksum);

auto hex_str = Redact::keyToHexString(buffer, size);
delete[] buffer;
return hex_str;
}

private:
ControlOptions options;
};
Expand Down
3 changes: 3 additions & 0 deletions release-centos7-llvm/env/.gitignore
View file
Original file line number Diff line number Diff line change
@@ -1,3 +1,6 @@
tiflash-env
tiflash-env-*.tar.xz
llvm-project
sysroot
*.log

0 comments on commit bd3762c

Please sign in to comment.