Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

fix pagestorage v2 restore (#5384) #5389

Closed
wants to merge 3 commits into from
Closed

fix pagestorage v2 restore (#5384) #5389

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
4f3b456
This is an automated cherry-pick of #5384
lidezhu Jul 15, 2022
581ec06
fix conflict
lidezhu Jul 15, 2022
d3c5c01
Merge branch 'release-5.2' into cherry-pick-5384-to-release-5.2
JaySon-Huang Dec 9, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 6 additions & 3 deletions dbms/src/Storages/Page/PageStorage.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -361,13 +361,15 @@ void PageStorage::restore()
}

// Fill write_files
PageFileIdAndLevel max_page_file_id_lvl{0, 0};
{
const size_t num_delta_paths = delegator->numPaths();
std::vector<size_t> next_write_fill_idx(num_delta_paths);
std::iota(next_write_fill_idx.begin(), next_write_fill_idx.end(), 0);
// Only insert location of PageFile when it storing delta data
for (auto & page_file : page_files)
{
max_page_file_id_lvl = std::max(max_page_file_id_lvl, page_file.fileIdLevel());
// Checkpoint file is always stored on `delegator`'s default path, so no need to insert it's location here
size_t idx_in_delta_paths = delegator->addPageFileUsedSize(
page_file.fileIdLevel(),
Expand All @@ -391,7 +393,7 @@ void PageStorage::restore()
std::vector<String> store_paths = delegator->listPaths();
for (size_t i = 0; i < write_files.size(); ++i)
{
auto writer = checkAndRenewWriter(write_files[i], /*parent_path_hint=*/store_paths[i % store_paths.size()]);
auto writer = checkAndRenewWriter(write_files[i], max_page_file_id_lvl, /*parent_path_hint=*/store_paths[i % store_paths.size()]);
idle_writers.emplace_back(std::move(writer));
}
#endif
Expand Down Expand Up @@ -451,6 +453,7 @@ PageEntry PageStorage::getEntry(PageId page_id, SnapshotPtr snapshot)
// The <id,level> of the new `page_file` is <max_id + 1, 0> of all `write_files`
PageStorage::WriterPtr PageStorage::checkAndRenewWriter( //
WritingPageFile & writing_file,
PageFileIdAndLevel max_page_file_id_lvl_hint,
const String & parent_path_hint,
PageStorage::WriterPtr && old_writer,
const String & logging_msg)
Expand Down Expand Up @@ -492,7 +495,7 @@ PageStorage::WriterPtr PageStorage::checkAndRenewWriter( //
pf_parent_path = parent_path_hint;
}

PageFileIdAndLevel max_writing_id_lvl{0, 0};
PageFileIdAndLevel max_writing_id_lvl{max_page_file_id_lvl_hint};
for (const auto & wf : write_files)
max_writing_id_lvl = std::max(max_writing_id_lvl, wf.file.fileIdLevel());
delegator->addPageFileUsedSize( //
Expand Down Expand Up @@ -603,7 +606,7 @@ void PageStorage::write(WriteBatch && wb, const WriteLimiterPtr & write_limiter)

// Check whether we need to roll to new PageFile and its writer
const auto logging_msg = " PageFile_" + DB::toString(writing_file.file.getFileId()) + "_0 is full,";
file_to_write = checkAndRenewWriter(writing_file, "", std::move(file_to_write), logging_msg);
file_to_write = checkAndRenewWriter(writing_file, {0, 0}, "", std::move(file_to_write), logging_msg);

idle_writers.emplace_back(std::move(file_to_write));

Expand Down
9 changes: 5 additions & 4 deletions dbms/src/Storages/Page/PageStorage.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -266,10 +266,11 @@ class PageStorage : private boost::noncopyable
TiFlashMetricsPtr metrics;

private:
WriterPtr checkAndRenewWriter(WritingPageFile & page_file,
const String & parent_path_hint,
WriterPtr && old_writer = nullptr,
const String & logging_msg = "");
WriterPtr checkAndRenewWriter(WritingPageFile & page_file,
PageFileIdAndLevel max_page_file_id_lvl_hint,
const String & parent_path_hint,
WriterPtr && old_writer = nullptr,
const String & logging_msg = "");
};

class PageReader : private boost::noncopyable
Expand Down
32 changes: 32 additions & 0 deletions dbms/src/Storages/Page/tests/gtest_page_storage.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -423,6 +423,38 @@ try
}
CATCH

TEST_F(PageStorage_test, RenewWriter)
try
{
constexpr size_t buf_sz = 100;
char c_buff[buf_sz];

{
WriteBatch wb;
memset(c_buff, 0xf, buf_sz);
auto buf = std::make_shared<ReadBufferFromMemory>(c_buff, sizeof(c_buff));
wb.putPage(1, 0, buf, buf_sz);
storage->write(std::move(wb));
}

{
PageStorage::ListPageFilesOption opt;
auto page_files = storage->listAllPageFiles(file_provider, storage->delegator, storage->log, opt);
ASSERT_EQ(page_files.size(), 1UL);
}

PageStorage::Config config;
config.file_roll_size = 10; // make it easy to renew a new page file for write
storage = reopenWithConfig(config);

{
PageStorage::ListPageFilesOption opt;
auto page_files = storage->listAllPageFiles(file_provider, storage->delegator, storage->log, opt);
ASSERT_EQ(page_files.size(), 2UL);
}
}
CATCH

/// Check if we can correctly do read / write after restore from disk.
TEST_F(PageStorage_test, WriteReadRestore)
try
Expand Down