Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[FLASH-298] Import DecodedTiKVKey and TableRowIDMinMax #218

Merged
merged 2 commits into from
Sep 4, 2019
Merged

[FLASH-298] Import DecodedTiKVKey and TableRowIDMinMax #218

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
7b86205
FLASH-298
solotzg Sep 3, 2019
7efd325
move modules.
solotzg Sep 3, 2019
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions dbms/src/Debug/DBGInvoker.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -64,6 +64,7 @@ DBGInvoker::DBGInvoker()
regFunc("try_flush_region", dbgFuncTryFlushRegion);

regFunc("dump_all_region", dbgFuncDumpAllRegion);
regFunc("dump_all_mock_region", dbgFuncDumpAllMockRegion);

regFunc("enable_schema_sync_service", dbgFuncEnableSchemaSyncService);
regFunc("refresh_schemas", dbgFuncRefreshSchemas);
Expand Down
60 changes: 38 additions & 22 deletions dbms/src/Debug/dbgFuncRegion.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -143,13 +143,8 @@ void dbgFuncRegionSnapshotWithData(Context & context, const ASTs & args, DBGInvo
TiKVValue value = RecordKVFormat::EncodeRow(table->table_info, fields);
UInt64 commit_ts = tso;
UInt64 prewrite_ts = tso;
TiKVValue commit_value;

if (del)
commit_value = RecordKVFormat::encodeWriteCfValue(Region::DelFlag, prewrite_ts);
else
commit_value = RecordKVFormat::encodeWriteCfValue(Region::PutFlag, prewrite_ts, value);

TiKVValue commit_value = del ? RecordKVFormat::encodeWriteCfValue(Region::DelFlag, prewrite_ts)
: RecordKVFormat::encodeWriteCfValue(Region::PutFlag, prewrite_ts, value);
TiKVKey commit_key = RecordKVFormat::appendTs(key, commit_ts);

region->insert(Region::write_cf_name, std::move(commit_key), std::move(commit_value));
Expand Down Expand Up @@ -225,7 +220,7 @@ std::string getRegionKeyString(const TiKVRange::Handle s, const TiKVKey & k)
{
if (s.type != TiKVHandle::HandleIDType::NORMAL)
{
String raw_key = k.empty() ? "" : RecordKVFormat::decodeTiKVKey(k);
auto raw_key = k.empty() ? "" : RecordKVFormat::decodeTiKVKey(k);
bool is_record = RecordKVFormat::isRecord(raw_key);
std::stringstream ss;
if (is_record)
Expand Down Expand Up @@ -273,20 +268,10 @@ std::string getEndKeyString(TableID table_id, const TiKVKey & end_key)
}
}

void dbgFuncDumpAllRegion(Context & context, const ASTs & args, DBGInvoker::Printer output)
void dbgFuncDumpAllRegion(Context & context, TableID table_id, bool ignore_none, bool dump_status, DBGInvoker::Printer & output)
{
if (args.size() < 1)
throw Exception("Args not matched, should be: table_id", ErrorCodes::BAD_ARGUMENTS);

auto & tmt = context.getTMTContext();
TableID table_id = (TableID)safeGet<UInt64>(typeid_cast<const ASTLiteral &>(*args[0]).value);

bool ignore_none = false;
if (args.size() > 1)
ignore_none = (std::string(typeid_cast<const ASTIdentifier &>(*args[1]).name) == "true");

size_t size = 0;
tmt.getKVStore()->traverseRegions([&](const RegionID region_id, const RegionPtr & region) {
context.getTMTContext().getKVStore()->traverseRegions([&](const RegionID region_id, const RegionPtr & region) {
std::ignore = region_id;
auto range = region->getHandleRangeByTable(table_id);
size += 1;
Expand All @@ -295,15 +280,46 @@ void dbgFuncDumpAllRegion(Context & context, const ASTs & args, DBGInvoker::Prin
if (range.first >= range.second && ignore_none)
return;

ss << "table #" << table_id << " " << region->toString();
ss << region->toString(dump_status);
if (range.first >= range.second)
ss << " [none], ";
else
ss << " ranges: [" << range.first.toString() << ", " << range.second.toString() << "), ";
ss << region->dataInfo();
if (auto s = region->dataInfo(); s.size() > 2)
ss << ", " << s;
output(ss.str());
});
output("total size: " + toString(size));
}

void dbgFuncDumpAllRegion(Context & context, const ASTs & args, DBGInvoker::Printer output)
{
if (args.size() < 1)
throw Exception("Args not matched, should be: table_id", ErrorCodes::BAD_ARGUMENTS);

TableID table_id = (TableID)safeGet<UInt64>(typeid_cast<const ASTLiteral &>(*args[0]).value);

bool ignore_none = false;
if (args.size() > 1)
ignore_none = (std::string(typeid_cast<const ASTIdentifier &>(*args[1]).name) == "true");

bool dump_status = true;
if (args.size() > 2)
dump_status = (std::string(typeid_cast<const ASTIdentifier &>(*args[2]).name) == "true");

output("table #" + toString(table_id));
dbgFuncDumpAllRegion(context, table_id, ignore_none, dump_status, output);
}

void dbgFuncDumpAllMockRegion(Context & context, const ASTs & args, DBGInvoker::Printer output)
{
const String & database_name = typeid_cast<const ASTIdentifier &>(*args[0]).name;
const String & table_name = typeid_cast<const ASTIdentifier &>(*args[1]).name;

auto table = MockTiDB::instance().getTableByName(database_name, table_name);
auto table_id = table->id();

dbgFuncDumpAllRegion(context, table_id, false, false, output);
}

} // namespace DB
5 changes: 5 additions & 0 deletions dbms/src/Debug/dbgFuncRegion.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,11 @@ void dbgFuncRegionSnapshotWithData(Context & context, const ASTs & args, DBGInvo
// ./storage-client.sh "DBGInvoke dump_all_region(table_id)"
void dbgFuncDumpAllRegion(Context & context, const ASTs & args, DBGInvoker::Printer output);

// Dump all region ranges for specific table
// Usage:
// ./storage-client.sh "DBGInvoke dump_all_mock_region(table_id)"
void dbgFuncDumpAllMockRegion(Context & context, const ASTs & args, DBGInvoker::Printer output);

// Try flush regions
// Usage:
// ./storage-client.sh "DBGInvoke try_flush()"
Expand Down
31 changes: 3 additions & 28 deletions dbms/src/Storages/Transaction/Region.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -42,7 +42,7 @@ TableID Region::insert(const std::string & cf, TiKVKey key, TiKVValue value)

TableID Region::doInsert(const std::string & cf, TiKVKey && key, TiKVValue && value)
{
std::string raw_key = RecordKVFormat::decodeTiKVKey(key);
auto raw_key = RecordKVFormat::decodeTiKVKey(key);
auto table_id = checkRecordAndValidTable(raw_key);
if (table_id == InvalidTableID)
return InvalidTableID;
Expand All @@ -59,7 +59,7 @@ TableID Region::remove(const std::string & cf, const TiKVKey & key)

TableID Region::doRemove(const std::string & cf, const TiKVKey & key)
{
std::string raw_key = RecordKVFormat::decodeTiKVKey(key);
auto raw_key = RecordKVFormat::decodeTiKVKey(key);
auto table_id = checkRecordAndValidTable(raw_key);
if (table_id == InvalidTableID)
return InvalidTableID;
Expand Down Expand Up @@ -121,16 +121,6 @@ void Region::execChangePeer(
meta.execChangePeer(request, response, index, term);
}

const metapb::Peer & findPeer(const metapb::Region & region, UInt64 store_id)
{
for (const auto & peer : region.peers())
{
if (peer.store_id() == store_id)
return peer;
}
throw Exception("[findPeer] peer with store_id " + DB::toString(store_id) + " not found", ErrorCodes::LOGICAL_ERROR);
}

Regions Region::execBatchSplit(
const raft_cmdpb::AdminRequest &, const raft_cmdpb::AdminResponse & response, const UInt64 index, const UInt64 term)
{
Expand Down Expand Up @@ -585,7 +575,7 @@ void Region::compareAndCompleteSnapshot(HandleMap & handle_map, const TableID ta
if (ori_ts >= safe_point)
throw Exception("[Region::compareAndCompleteSnapshot] original ts >= gc safe point", ErrorCodes::LOGICAL_ERROR);

std::string raw_key = RecordKVFormat::genRawKey(table_id, handle);
auto raw_key = RecordKVFormat::genRawKey(table_id, handle);
TiKVKey key = RecordKVFormat::encodeAsTiKVKey(raw_key);
TiKVKey commit_key = RecordKVFormat::appendTs(key, ori_ts);
TiKVValue value = RecordKVFormat::encodeWriteCfValue(DelFlag, 0);
Expand Down Expand Up @@ -655,21 +645,6 @@ void Region::doDeleteRange(const std::string & cf, const TiKVKey & start_key, co

std::tuple<RegionVersion, RegionVersion, RegionRange> Region::dumpVersionRange() const { return meta.dumpVersionRange(); }

void tryPreDecodeTiKVValue(std::optional<ExtraCFDataQueue> && values)
{
if (!values)
return;

for (const auto & val : *values)
{
auto & decoded_row_info = val->extraInfo();
if (decoded_row_info.load())
continue;
DecodedRow * decoded_row = ValueExtraInfo<>::computeDecodedRow(val->getStr());
decoded_row_info.atomicUpdate(decoded_row);
}
}

void Region::tryPreDecodeTiKVValue()
{
DB::tryPreDecodeTiKVValue(data.defaultCF().getExtra().popAll());
Expand Down
8 changes: 6 additions & 2 deletions dbms/src/Storages/Transaction/RegionCFDataBase.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,10 @@
namespace DB
{

const std::string RegionWriteCFDataTrait::name = "write";
const std::string RegionDefaultCFDataTrait::name = "default";
const std::string RegionLockCFDataTrait::name = "lock";

template <typename Trait>
const TiKVKey & RegionCFDataBase<Trait>::getTiKVKey(const Value & val)
{
Expand All @@ -25,12 +29,12 @@ const TiKVValue & RegionCFDataBase<Trait>::getTiKVValue(const Value & val)
template <typename Trait>
TableID RegionCFDataBase<Trait>::insert(TiKVKey && key, TiKVValue && value)
{
const String & raw_key = RecordKVFormat::decodeTiKVKey(key);
const auto & raw_key = RecordKVFormat::decodeTiKVKey(key);
return insert(std::move(key), std::move(value), raw_key);
}

template <typename Trait>
TableID RegionCFDataBase<Trait>::insert(TiKVKey && key, TiKVValue && value, const String & raw_key)
TableID RegionCFDataBase<Trait>::insert(TiKVKey && key, TiKVValue && value, const DecodedTiKVKey & raw_key)
{
Pair kv_pair = Trait::genKVPair(std::move(key), raw_key, std::move(value));
if (shouldIgnoreInsert(kv_pair.second))
Expand Down
2 changes: 1 addition & 1 deletion dbms/src/Storages/Transaction/RegionCFDataBase.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -37,7 +37,7 @@ struct RegionCFDataBase

TableID insert(TiKVKey && key, TiKVValue && value);
TableID insert(const TableID table_id, std::pair<Key, Value> && kv_pair);
TableID insert(TiKVKey && key, TiKVValue && value, const String & raw_key);
TableID insert(TiKVKey && key, TiKVValue && value, const DecodedTiKVKey & raw_key);

static size_t calcTiKVKeyValueSize(const Value & value);

Expand Down
9 changes: 6 additions & 3 deletions dbms/src/Storages/Transaction/RegionCFDataTrait.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,12 +19,13 @@ struct CFKeyHasher

struct RegionWriteCFDataTrait
{
static const std::string name;
Copy link
Contributor

@innerr innerr Sep 3, 2019 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

If my memories are not wrong, we could define static const std::string name = 'foobar' in .h, just work for static vars.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I've test and this didn't work. std::string need constructor.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

name is a static variables of struct RegionWriteCFDataTrait, if we want to access it out of this struct, we must initialize it out of RegionWriteCFDataTrait.

using DecodedWriteCFValue = RecordKVFormat::DecodedWriteCFValue;
using Key = std::pair<HandleID, Timestamp>;
using Value = std::tuple<std::shared_ptr<const TiKVKey>, std::shared_ptr<const TiKVValue>, DecodedWriteCFValue>;
using Map = std::map<Key, Value>;

static Map::value_type genKVPair(TiKVKey && key, const String & raw_key, TiKVValue && value)
static Map::value_type genKVPair(TiKVKey && key, const DecodedTiKVKey & raw_key, TiKVValue && value)
{
HandleID handle_id = RecordKVFormat::getHandle(raw_key);
Timestamp ts = RecordKVFormat::getTs(key);
Expand All @@ -42,11 +43,12 @@ struct RegionWriteCFDataTrait

struct RegionDefaultCFDataTrait
{
static const std::string name;
using Key = std::pair<HandleID, Timestamp>;
using Value = std::tuple<std::shared_ptr<const TiKVKey>, std::shared_ptr<const TiKVValue>>;
using Map = std::map<Key, Value>;

static Map::value_type genKVPair(TiKVKey && key, const String & raw_key, TiKVValue && value)
static Map::value_type genKVPair(TiKVKey && key, const DecodedTiKVKey & raw_key, TiKVValue && value)
{
HandleID handle_id = RecordKVFormat::getHandle(raw_key);
Timestamp ts = RecordKVFormat::getTs(key);
Expand All @@ -57,12 +59,13 @@ struct RegionDefaultCFDataTrait

struct RegionLockCFDataTrait
{
static const std::string name;
using DecodedLockCFValue = RecordKVFormat::DecodedLockCFValue;
using Key = HandleID;
using Value = std::tuple<std::shared_ptr<const TiKVKey>, std::shared_ptr<const TiKVValue>, DecodedLockCFValue>;
using Map = std::unordered_map<Key, Value>;

static Map::value_type genKVPair(TiKVKey && key, const String & raw_key, TiKVValue && value)
static Map::value_type genKVPair(TiKVKey && key, const DecodedTiKVKey & raw_key, TiKVValue && value)
{
HandleID handle_id = RecordKVFormat::getHandle(raw_key);
auto decoded_val = RecordKVFormat::decodeLockCfValue(value);
Expand Down
8 changes: 4 additions & 4 deletions dbms/src/Storages/Transaction/RegionData.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@
namespace DB
{

TableID RegionData::insert(ColumnFamilyType cf, TiKVKey && key, const String & raw_key, TiKVValue && value)
TableID RegionData::insert(ColumnFamilyType cf, TiKVKey && key, const DecodedTiKVKey & raw_key, TiKVValue && value)
{
switch (cf)
{
Expand Down Expand Up @@ -33,20 +33,20 @@ TableID RegionData::insert(ColumnFamilyType cf, TiKVKey && key, const String & r
}
}

void RegionData::removeLockCF(const TableID & table_id, const String & raw_key)
void RegionData::removeLockCF(const TableID & table_id, const DecodedTiKVKey & raw_key)
{
HandleID handle_id = RecordKVFormat::getHandle(raw_key);
lock_cf.remove(table_id, handle_id);
}

void RegionData::removeDefaultCF(const TableID & table_id, const TiKVKey & key, const String & raw_key)
void RegionData::removeDefaultCF(const TableID & table_id, const TiKVKey & key, const DecodedTiKVKey & raw_key)
{
HandleID handle_id = RecordKVFormat::getHandle(raw_key);
Timestamp ts = RecordKVFormat::getTs(key);
cf_data_size -= default_cf.remove(table_id, RegionDefaultCFData::Key{handle_id, ts}, true);
}

void RegionData::removeWriteCF(const TableID & table_id, const TiKVKey & key, const String & raw_key)
void RegionData::removeWriteCF(const TableID & table_id, const TiKVKey & key, const DecodedTiKVKey & raw_key)
{
HandleID handle_id = RecordKVFormat::getHandle(raw_key);
Timestamp ts = RecordKVFormat::getTs(key);
Expand Down
8 changes: 4 additions & 4 deletions dbms/src/Storages/Transaction/RegionData.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,11 +25,11 @@ class RegionData
using WriteCFIter = RegionWriteCFData::Map::iterator;
using ConstWriteCFIter = RegionWriteCFData::Map::const_iterator;

TableID insert(ColumnFamilyType cf, TiKVKey && key, const String & raw_key, TiKVValue && value);
TableID insert(ColumnFamilyType cf, TiKVKey && key, const DecodedTiKVKey & raw_key, TiKVValue && value);

void removeLockCF(const TableID & table_id, const String & raw_key);
void removeDefaultCF(const TableID & table_id, const TiKVKey & key, const String & raw_key);
void removeWriteCF(const TableID & table_id, const TiKVKey & key, const String & raw_key);
void removeLockCF(const TableID & table_id, const DecodedTiKVKey & raw_key);
void removeDefaultCF(const TableID & table_id, const TiKVKey & key, const DecodedTiKVKey & raw_key);
void removeWriteCF(const TableID & table_id, const TiKVKey & key, const DecodedTiKVKey & raw_key);

WriteCFIter removeDataByWriteIt(const TableID & table_id, const WriteCFIter & write_it);

Expand Down
27 changes: 26 additions & 1 deletion dbms/src/Storages/Transaction/RegionHelper.hpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@ namespace DB
{

/// Ignoring all keys other than records.
inline TableID checkRecordAndValidTable(const std::string & raw_key)
inline TableID checkRecordAndValidTable(const DecodedTiKVKey & raw_key)
{
// Ignoring all keys other than records.
if (!RecordKVFormat::isRecord(raw_key))
Expand All @@ -17,4 +17,29 @@ inline TableID checkRecordAndValidTable(const std::string & raw_key)
return table_id;
}

void tryPreDecodeTiKVValue(std::optional<ExtraCFDataQueue> && values)
{
if (!values)
return;

for (const auto & val : *values)
{
auto & decoded_row_info = val->extraInfo();
if (decoded_row_info.load())
continue;
DecodedRow * decoded_row = ValueExtraInfo<>::computeDecodedRow(val->getStr());
decoded_row_info.atomicUpdate(decoded_row);
}
}

const metapb::Peer & findPeer(const metapb::Region & region, UInt64 store_id)
{
for (const auto & peer : region.peers())
{
if (peer.store_id() == store_id)
return peer;
}
throw Exception("[findPeer] peer with store_id " + DB::toString(store_id) + " not found", ErrorCodes::LOGICAL_ERROR);
}

} // namespace DB
18 changes: 18 additions & 0 deletions dbms/src/Storages/Transaction/TableRowIDMinMax.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,18 @@
#include <Storages/Transaction/TableRowIDMinMax.h>

namespace DB
{

std::unordered_map<TableID, TableRowIDMinMax> TableRowIDMinMax::data;
std::mutex TableRowIDMinMax::mutex;

const TableRowIDMinMax & TableRowIDMinMax::getMinMax(const TableID table_id)
{
std::lock_guard<std::mutex> lock(mutex);

if (auto it = data.find(table_id); it != data.end())
return it->second;
return data.try_emplace(table_id, table_id).first->second;
}

} // namespace DB
Loading