Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Disagg: Fix the problem of missing remote page id in MemTableSet. (#8437) #8440

Merged
merged 4 commits into from
Dec 15, 2023
Merged

Disagg: Fix the problem of missing remote page id in MemTableSet. (#8437) #8440

Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
0134126
This is an automated cherry-pick of #8437
JinheLin Nov 29, 2023
4c3c94f
resolve
JinheLin Dec 12, 2023
2d82767
gtest_segment_read_task is not supportted in 7.5
JinheLin Dec 15, 2023
89f51c8
Merge branch 'release-7.5' into cherry-pick-8437-to-release-7.5
ti-chi-bot[bot] Dec 15, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
36 changes: 27 additions & 9 deletions dbms/src/Storages/DeltaMerge/Remote/RNReadTask.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -84,25 +84,38 @@ RNReadSegmentTaskPtr RNReadSegmentTask::buildFromEstablishResp(
std::vector<UInt64> delta_tinycf_ids;
std::vector<size_t> delta_tinycf_sizes;
{
// The number of ColumnFileTiny of MemTableSet is unknown, but there is a very high probability that it is zero.
// So ignoring the number of ColumnFileTiny of MemTableSet is better than always adding all the number of ColumnFile of MemTableSet when reserving.
auto persisted_cfs = segment_snap->delta->getPersistedFileSetSnapshot();
delta_tinycf_ids.reserve(persisted_cfs->getColumnFileCount());
delta_tinycf_sizes.reserve(persisted_cfs->getColumnFileCount());
for (const auto & cfs : persisted_cfs->getColumnFiles())
}
auto extract_remote_pages = [&delta_tinycf_ids, &delta_tinycf_sizes](const ColumnFiles & cfs) {
UInt64 count = 0;
for (const auto & cf : cfs)
{
if (auto * tiny = cfs->tryToTinyFile(); tiny)
if (auto * tiny = cf->tryToTinyFile(); tiny)
{
delta_tinycf_ids.emplace_back(tiny->getDataPageId());
delta_tinycf_sizes.emplace_back(tiny->getDataPageSize());
++count;
}
}
}
return count;
};
auto memory_page_count = extract_remote_pages(segment_snap->delta->getMemTableSetSnapshot()->getColumnFiles());
auto persisted_page_count
= extract_remote_pages(segment_snap->delta->getPersistedFileSetSnapshot()->getColumnFiles());

LOG_DEBUG(
segment_snap->log,
"Build RNReadSegmentTask: memtable_cfs={} persisted_cfs={} delta_index={}",
"Build RNReadSegmentTask: memtable_cfs={} persisted_cfs={} delta_index={} memory_page_count={} "
"persisted_page_count={}",
segment_snap->delta->getMemTableSetSnapshot()->getColumnFileCount(),
segment_snap->delta->getPersistedFileSetSnapshot()->getColumnFileCount(),
segment_snap->delta->getSharedDeltaIndex()->toString());
segment_snap->delta->getSharedDeltaIndex()->toString(),
memory_page_count,
persisted_page_count);

return std::shared_ptr<RNReadSegmentTask>(new RNReadSegmentTask(RNReadSegmentMeta{
.keyspace_id = keyspace_id,
Expand All @@ -124,15 +137,20 @@ RNReadSegmentTaskPtr RNReadSegmentTask::buildFromEstablishResp(

void RNReadSegmentTask::initColumnFileDataProvider(const RNLocalPageCacheGuardPtr & pages_guard)
{
auto & data_provider = meta.segment_snap->delta->getPersistedFileSetSnapshot()->data_provider;
RUNTIME_CHECK(std::dynamic_pointer_cast<ColumnFileDataProviderNop>(data_provider));

auto page_cache = meta.dm_context->db_context.getSharedContextDisagg()->rn_page_cache;
data_provider = std::make_shared<ColumnFileDataProviderRNLocalPageCache>(
auto data_provider = std::make_shared<ColumnFileDataProviderRNLocalPageCache>(
page_cache,
pages_guard,
meta.store_id,
KeyspaceTableID{meta.keyspace_id, meta.physical_table_id});

auto & persisted_cf_set_data_provider = meta.segment_snap->delta->getPersistedFileSetSnapshot()->data_provider;
RUNTIME_CHECK(std::dynamic_pointer_cast<ColumnFileDataProviderNop>(persisted_cf_set_data_provider));
persisted_cf_set_data_provider = data_provider;

auto & memory_cf_set_data_provider = meta.segment_snap->delta->getMemTableSetSnapshot()->data_provider;
RUNTIME_CHECK(std::dynamic_pointer_cast<ColumnFileDataProviderNop>(memory_cf_set_data_provider));
memory_cf_set_data_provider = data_provider;
}

void RNReadSegmentTask::initInputStream(
Expand Down