Skip to content

Commit

Permalink
schema : make update to partition tables when 'set tiflash replica' (#…
Browse files Browse the repository at this point in the history 
…5267)

close #5266
  • Loading branch information
@hongyunyan
hongyunyan authored Jul 7, 2022
1 parent cb69d5c commit 597f8b8
Show file tree
Hide file tree
Showing 8 changed files with 185 additions and 43 deletions.
2 changes: 2 additions & 0 deletions dbms/src/Debug/DBGInvoker.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -118,6 +118,8 @@ DBGInvoker::DBGInvoker()
regSchemalessFunc("mapped_database", dbgFuncMappedDatabase);
regSchemalessFunc("mapped_table", dbgFuncMappedTable);
regSchemafulFunc("query_mapped", dbgFuncQueryMapped);
regSchemalessFunc("get_tiflash_replica_count", dbgFuncGetTiflashReplicaCount);
regSchemalessFunc("get_partition_tables_tiflash_replica_count", dbgFuncGetPartitionTablesTiflashReplicaCount);

regSchemalessFunc("search_log_for_key", dbgFuncSearchLogForKey);
regSchemalessFunc("tidb_dag", dbgFuncTiDBQueryFromNaturalDag);
Expand Down
2 changes: 2 additions & 0 deletions dbms/src/Debug/dbgFuncSchema.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@
#include <Storages/IManageableStorage.h>
#include <Storages/Transaction/TMTContext.h>
#include <Storages/Transaction/TiDB.h>
#include <TiDB/Schema/SchemaNameMapper.h>
#include <TiDB/Schema/SchemaSyncService.h>
#include <TiDB/Schema/SchemaSyncer.h>
#include <fmt/core.h>
Expand Down Expand Up @@ -137,4 +138,5 @@ void dbgFuncIsTombstone(Context & context, const ASTs & args, DBGInvoker::Printe
output(fmt_buf.toString());
}


} // namespace DB
1 change: 0 additions & 1 deletion dbms/src/Debug/dbgFuncSchema.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -46,5 +46,4 @@ void dbgFuncResetSchemas(Context & context, const ASTs & args, DBGInvoker::Print
// Usage:
// ./storage-client.sh "DBGInvoke is_tombstone(db_name, table_name)"
void dbgFuncIsTombstone(Context & context, const ASTs & args, DBGInvoker::Printer output);

} // namespace DB
53 changes: 53 additions & 0 deletions dbms/src/Debug/dbgFuncSchemaName.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -128,4 +128,57 @@ BlockInputStreamPtr dbgFuncQueryMapped(Context & context, const ASTs & args)
return executeQuery(query, context, true).in;
}


void dbgFuncGetTiflashReplicaCount(Context & context, const ASTs & args, DBGInvoker::Printer output)
{
if (args.empty() || args.size() != 2)
throw Exception("Args not matched, should be: database-name[, table-name]", ErrorCodes::BAD_ARGUMENTS);

const String & database_name = typeid_cast<const ASTIdentifier &>(*args[0]).name;
FmtBuffer fmt_buf;

const String & table_name = typeid_cast<const ASTIdentifier &>(*args[1]).name;
auto mapped = mappedTable(context, database_name, table_name);
auto storage = context.getTable(mapped->first, mapped->second);
auto managed_storage = std::dynamic_pointer_cast<IManageableStorage>(storage);
if (!managed_storage)
throw Exception(database_name + "." + table_name + " is not ManageableStorage", ErrorCodes::BAD_ARGUMENTS);

fmt_buf.append((std::to_string(managed_storage->getTableInfo().replica_info.count)));

output(fmt_buf.toString());
}

void dbgFuncGetPartitionTablesTiflashReplicaCount(Context & context, const ASTs & args, DBGInvoker::Printer output)
{
if (args.empty() || args.size() != 2)
throw Exception("Args not matched, should be: database-name[, table-name]", ErrorCodes::BAD_ARGUMENTS);

const String & database_name = typeid_cast<const ASTIdentifier &>(*args[0]).name;
FmtBuffer fmt_buf;

const String & table_name = typeid_cast<const ASTIdentifier &>(*args[1]).name;
auto mapped = mappedTable(context, database_name, table_name);
auto storage = context.getTable(mapped->first, mapped->second);
auto managed_storage = std::dynamic_pointer_cast<IManageableStorage>(storage);
if (!managed_storage)
throw Exception(database_name + "." + table_name + " is not ManageableStorage", ErrorCodes::BAD_ARGUMENTS);

auto table_info = managed_storage->getTableInfo();

if (!table_info.isLogicalPartitionTable())
throw Exception(database_name + "." + table_name + " is not logical partition table", ErrorCodes::BAD_ARGUMENTS);

SchemaNameMapper name_mapper;
for (const auto & part_def : table_info.partition.definitions)
{
auto paritition_table_info = table_info.producePartitionTableInfo(part_def.id, name_mapper);
auto partition_storage = context.getTMTContext().getStorages().get(paritition_table_info->id);
fmt_buf.append((std::to_string(partition_storage->getTableInfo().replica_info.count)));
fmt_buf.append("/");
}

output(fmt_buf.toString());
}

} // namespace DB
10 changes: 10 additions & 0 deletions dbms/src/Debug/dbgFuncSchemaName.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -40,4 +40,14 @@ void dbgFuncMappedTable(Context & context, const ASTs & args, DBGInvoker::Printe
// ./storage-client.sh "DBGInvoke query_mapped('select * from $d.$t', database_name[, table_name])"
BlockInputStreamPtr dbgFuncQueryMapped(Context & context, const ASTs & args);

// Get table's tiflash replica counts with mapped table name
// Usage:
// ./storage-client.sh "DBGInvoke get_tiflash_replica_count(db_name, table_name)"
void dbgFuncGetTiflashReplicaCount(Context & context, const ASTs & args, DBGInvoker::Printer output);

// Get the logical table's partition tables' tiflash replica counts with mapped table name
// Usage:
// ./storage-client.sh "DBGInvoke get_partition_tables_tiflash_replica_count(db_name, table_name)"
void dbgFuncGetPartitionTablesTiflashReplicaCount(Context & context, const ASTs & args, DBGInvoker::Printer output);

} // namespace DB
74 changes: 47 additions & 27 deletions dbms/src/TiDB/Schema/SchemaBuilder.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -320,7 +320,7 @@ inline SchemaChanges detectSchemaChanges(
}

template <typename Getter, typename NameMapper>
void SchemaBuilder<Getter, NameMapper>::applyAlterPhysicalTable(DBInfoPtr db_info, TableInfoPtr table_info, ManageableStoragePtr storage)
void SchemaBuilder<Getter, NameMapper>::applyAlterPhysicalTable(const DBInfoPtr & db_info, const TableInfoPtr & table_info, const ManageableStoragePtr & storage)
{
LOG_FMT_INFO(log, "Altering table {}", name_mapper.debugCanonicalName(*db_info, *table_info));

Expand Down Expand Up @@ -394,7 +394,7 @@ void SchemaBuilder<Getter, NameMapper>::applyAlterPhysicalTable(DBInfoPtr db_inf
}

template <typename Getter, typename NameMapper>
void SchemaBuilder<Getter, NameMapper>::applyAlterTable(DBInfoPtr db_info, TableID table_id)
void SchemaBuilder<Getter, NameMapper>::applyAlterTable(const DBInfoPtr & db_info, TableID table_id)
{
auto table_info = getter.getTableInfo(db_info->id, table_id);
if (table_info == nullptr)
Expand All @@ -413,7 +413,7 @@ void SchemaBuilder<Getter, NameMapper>::applyAlterTable(DBInfoPtr db_info, Table
}

template <typename Getter, typename NameMapper>
void SchemaBuilder<Getter, NameMapper>::applyAlterLogicalTable(DBInfoPtr db_info, TableInfoPtr table_info, ManageableStoragePtr storage)
void SchemaBuilder<Getter, NameMapper>::applyAlterLogicalTable(const DBInfoPtr & db_info, const TableInfoPtr & table_info, const ManageableStoragePtr & storage)
{
// Alter logical table first.
applyAlterPhysicalTable(db_info, table_info, storage);
Expand Down Expand Up @@ -569,7 +569,7 @@ void SchemaBuilder<Getter, NameMapper>::applyDiff(const SchemaDiff & diff)
}

template <typename Getter, typename NameMapper>
void SchemaBuilder<Getter, NameMapper>::applyPartitionDiff(TiDB::DBInfoPtr db_info, TableID table_id)
void SchemaBuilder<Getter, NameMapper>::applyPartitionDiff(const TiDB::DBInfoPtr & db_info, TableID table_id)
{
auto table_info = getter.getTableInfo(db_info->id, table_id);
if (table_info == nullptr)
Expand All @@ -593,7 +593,7 @@ void SchemaBuilder<Getter, NameMapper>::applyPartitionDiff(TiDB::DBInfoPtr db_in
}

template <typename Getter, typename NameMapper>
void SchemaBuilder<Getter, NameMapper>::applyPartitionDiff(TiDB::DBInfoPtr db_info, TableInfoPtr table_info, ManageableStoragePtr storage)
void SchemaBuilder<Getter, NameMapper>::applyPartitionDiff(const TiDB::DBInfoPtr & db_info, const TableInfoPtr & table_info, const ManageableStoragePtr & storage)
{
const auto & orig_table_info = storage->getTableInfo();
if (!orig_table_info.isLogicalPartitionTable())
Expand Down Expand Up @@ -659,7 +659,7 @@ void SchemaBuilder<Getter, NameMapper>::applyPartitionDiff(TiDB::DBInfoPtr db_in
}

template <typename Getter, typename NameMapper>
void SchemaBuilder<Getter, NameMapper>::applyRenameTable(DBInfoPtr new_db_info, TableID table_id)
void SchemaBuilder<Getter, NameMapper>::applyRenameTable(const DBInfoPtr & new_db_info, TableID table_id)
{
auto new_table_info = getter.getTableInfo(new_db_info->id, table_id);
if (new_table_info == nullptr)
Expand All @@ -679,9 +679,9 @@ void SchemaBuilder<Getter, NameMapper>::applyRenameTable(DBInfoPtr new_db_info,

template <typename Getter, typename NameMapper>
void SchemaBuilder<Getter, NameMapper>::applyRenameLogicalTable(
DBInfoPtr new_db_info,
TableInfoPtr new_table_info,
ManageableStoragePtr storage)
const DBInfoPtr & new_db_info,
const TableInfoPtr & new_table_info,
const ManageableStoragePtr & storage)
{
applyRenamePhysicalTable(new_db_info, *new_table_info, storage);

Expand All @@ -703,9 +703,9 @@ void SchemaBuilder<Getter, NameMapper>::applyRenameLogicalTable(

template <typename Getter, typename NameMapper>
void SchemaBuilder<Getter, NameMapper>::applyRenamePhysicalTable(
DBInfoPtr new_db_info,
TableInfo & new_table_info,
ManageableStoragePtr storage)
const DBInfoPtr & new_db_info,
const TableInfo & new_table_info,
const ManageableStoragePtr & storage)
{
const auto old_mapped_db_name = storage->getDatabaseName();
const auto new_mapped_db_name = name_mapper.mapDatabaseName(*new_db_info);
Expand Down Expand Up @@ -908,7 +908,7 @@ String createDatabaseStmt(Context & context, const DBInfo & db_info, const Schem
}

template <typename Getter, typename NameMapper>
void SchemaBuilder<Getter, NameMapper>::applyCreateSchema(TiDB::DBInfoPtr db_info)
void SchemaBuilder<Getter, NameMapper>::applyCreateSchema(const TiDB::DBInfoPtr & db_info)
{
GET_METRIC(tiflash_schema_internal_ddl_count, type_create_db).Increment();
LOG_FMT_INFO(log, "Creating database {}", name_mapper.debugDatabaseName(*db_info));
Expand Down Expand Up @@ -1047,7 +1047,7 @@ String createTableStmt(
}

template <typename Getter, typename NameMapper>
void SchemaBuilder<Getter, NameMapper>::applyCreatePhysicalTable(DBInfoPtr db_info, TableInfoPtr table_info)
void SchemaBuilder<Getter, NameMapper>::applyCreatePhysicalTable(const DBInfoPtr & db_info, const TableInfoPtr & table_info)
{
GET_METRIC(tiflash_schema_internal_ddl_count, type_create_table).Increment();
LOG_FMT_INFO(log, "Creating table {}", name_mapper.debugCanonicalName(*db_info, *table_info));
Expand Down Expand Up @@ -1109,7 +1109,7 @@ void SchemaBuilder<Getter, NameMapper>::applyCreatePhysicalTable(DBInfoPtr db_in
}

template <typename Getter, typename NameMapper>
void SchemaBuilder<Getter, NameMapper>::applyCreateTable(TiDB::DBInfoPtr db_info, TableID table_id)
void SchemaBuilder<Getter, NameMapper>::applyCreateTable(const TiDB::DBInfoPtr & db_info, TableID table_id)
{
auto table_info = getter.getTableInfo(db_info->id, table_id);
if (table_info == nullptr)
Expand All @@ -1123,7 +1123,7 @@ void SchemaBuilder<Getter, NameMapper>::applyCreateTable(TiDB::DBInfoPtr db_info
}

template <typename Getter, typename NameMapper>
void SchemaBuilder<Getter, NameMapper>::applyCreateLogicalTable(TiDB::DBInfoPtr db_info, TableInfoPtr table_info)
void SchemaBuilder<Getter, NameMapper>::applyCreateLogicalTable(const TiDB::DBInfoPtr & db_info, const TableInfoPtr & table_info)
{
if (table_info->isLogicalPartitionTable())
{
Expand Down Expand Up @@ -1169,7 +1169,7 @@ void SchemaBuilder<Getter, NameMapper>::applyDropPhysicalTable(const String & db
}

template <typename Getter, typename NameMapper>
void SchemaBuilder<Getter, NameMapper>::applyDropTable(DBInfoPtr db_info, TableID table_id)
void SchemaBuilder<Getter, NameMapper>::applyDropTable(const DBInfoPtr & db_info, TableID table_id)
{
auto & tmt_context = context.getTMTContext();
auto * storage = tmt_context.getStorages().get(table_id).get();
Expand All @@ -1193,13 +1193,14 @@ void SchemaBuilder<Getter, NameMapper>::applyDropTable(DBInfoPtr db_info, TableI
}

template <typename Getter, typename NameMapper>
void SchemaBuilder<Getter, NameMapper>::applySetTiFlashReplica(TiDB::DBInfoPtr db_info, TableID table_id)
void SchemaBuilder<Getter, NameMapper>::applySetTiFlashReplica(const TiDB::DBInfoPtr & db_info, TableID table_id)
{
auto latest_table_info = getter.getTableInfo(db_info->id, table_id);
if (unlikely(latest_table_info == nullptr))
{
throw TiFlashException(fmt::format("miss table in TiKV : {}", table_id), Errors::DDL::StaleSchema);
}

auto & tmt_context = context.getTMTContext();
auto storage = tmt_context.getStorages().get(latest_table_info->id);
if (unlikely(storage == nullptr))
Expand All @@ -1208,18 +1209,37 @@ void SchemaBuilder<Getter, NameMapper>::applySetTiFlashReplica(TiDB::DBInfoPtr d
Errors::DDL::MissingTable);
}

auto managed_storage = std::dynamic_pointer_cast<IManageableStorage>(storage);
if (unlikely(!managed_storage))
throw Exception(fmt::format("{} is not a ManageableStorage", name_mapper.debugCanonicalName(*db_info, *latest_table_info)));
applySetTiFlashReplicaOnLogicalTable(db_info, latest_table_info, storage);
}

template <typename Getter, typename NameMapper>
void SchemaBuilder<Getter, NameMapper>::applySetTiFlashReplicaOnLogicalTable(const TiDB::DBInfoPtr & db_info, const TiDB::TableInfoPtr & table_info, const ManageableStoragePtr & storage)
{
applySetTiFlashReplicaOnPhysicalTable(db_info, table_info, storage);

if (table_info->isLogicalPartitionTable())
{
auto & tmt_context = context.getTMTContext();

applySetTiFlashReplica(db_info, latest_table_info, managed_storage);
for (const auto & part_def : table_info->partition.definitions)
{
auto new_part_table_info = table_info->producePartitionTableInfo(part_def.id, name_mapper);
auto part_storage = tmt_context.getStorages().get(new_part_table_info->id);
if (unlikely(part_storage == nullptr))
{
throw TiFlashException(fmt::format("miss table in TiFlash : {}", name_mapper.debugCanonicalName(*db_info, *new_part_table_info)),
Errors::DDL::MissingTable);
}
applySetTiFlashReplicaOnPhysicalTable(db_info, new_part_table_info, part_storage);
}
}
}

template <typename Getter, typename NameMapper>
void SchemaBuilder<Getter, NameMapper>::applySetTiFlashReplica(
TiDB::DBInfoPtr db_info,
TiDB::TableInfoPtr latest_table_info,
ManageableStoragePtr storage)
void SchemaBuilder<Getter, NameMapper>::applySetTiFlashReplicaOnPhysicalTable(
const TiDB::DBInfoPtr & db_info,
const TiDB::TableInfoPtr & latest_table_info,
const ManageableStoragePtr & storage)
{
if (storage->getTableInfo().replica_info.count == latest_table_info->replica_info.count)
return;
Expand Down Expand Up @@ -1306,7 +1326,7 @@ void SchemaBuilder<Getter, NameMapper>::syncAllSchema()
/// Rename if needed.
applyRenameLogicalTable(db, table, storage);
/// Update replica info if needed.
applySetTiFlashReplica(db, table, storage);
applySetTiFlashReplicaOnLogicalTable(db, table, storage);
/// Alter if needed.
applyAlterLogicalTable(db, table, storage);
LOG_FMT_DEBUG(log, "Table {} synced during sync all schemas", name_mapper.debugCanonicalName(*db, *table));
Expand Down
31 changes: 16 additions & 15 deletions dbms/src/TiDB/Schema/SchemaBuilder.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -55,39 +55,40 @@ struct SchemaBuilder

bool applyCreateSchema(DatabaseID schema_id);

void applyCreateSchema(TiDB::DBInfoPtr db_info);
void applyCreateSchema(const TiDB::DBInfoPtr & db_info);

void applyCreateTable(TiDB::DBInfoPtr db_info, TableID table_id);
void applyCreateTable(const TiDB::DBInfoPtr & db_info, TableID table_id);

void applyCreateLogicalTable(TiDB::DBInfoPtr db_info, TiDB::TableInfoPtr table_info);
void applyCreateLogicalTable(const TiDB::DBInfoPtr & db_info, const TiDB::TableInfoPtr & table_info);

void applyCreatePhysicalTable(TiDB::DBInfoPtr db_info, TiDB::TableInfoPtr table_info);
void applyCreatePhysicalTable(const TiDB::DBInfoPtr & db_info, const TiDB::TableInfoPtr & table_info);

void applyDropTable(TiDB::DBInfoPtr db_info, TableID table_id);
void applyDropTable(const TiDB::DBInfoPtr & db_info, TableID table_id);

/// Parameter schema_name should be mapped.
void applyDropPhysicalTable(const String & db_name, TableID table_id);

void applyPartitionDiff(TiDB::DBInfoPtr db_info, TableID table_id);
void applyPartitionDiff(const TiDB::DBInfoPtr & db_info, TableID table_id);

void applyPartitionDiff(TiDB::DBInfoPtr db_info, TiDB::TableInfoPtr table_info, ManageableStoragePtr storage);
void applyPartitionDiff(const TiDB::DBInfoPtr & db_info, const TiDB::TableInfoPtr & table_info, const ManageableStoragePtr & storage);

void applyAlterTable(TiDB::DBInfoPtr db_info, TableID table_id);
void applyAlterTable(const TiDB::DBInfoPtr & db_info, TableID table_id);

void applyAlterLogicalTable(TiDB::DBInfoPtr db_info, TiDB::TableInfoPtr table_info, ManageableStoragePtr storage);
void applyAlterLogicalTable(const TiDB::DBInfoPtr & db_info, const TiDB::TableInfoPtr & table_info, const ManageableStoragePtr & storage);

void applyAlterPhysicalTable(TiDB::DBInfoPtr db_info, TiDB::TableInfoPtr table_info, ManageableStoragePtr storage);
void applyAlterPhysicalTable(const TiDB::DBInfoPtr & db_info, const TiDB::TableInfoPtr & table_info, const ManageableStoragePtr & storage);

void applyRenameTable(TiDB::DBInfoPtr new_db_info, TiDB::TableID table_id);
void applyRenameTable(const TiDB::DBInfoPtr & new_db_info, TiDB::TableID table_id);

void applyRenameLogicalTable(TiDB::DBInfoPtr new_db_info, TiDB::TableInfoPtr new_table_info, ManageableStoragePtr storage);
void applyRenameLogicalTable(const TiDB::DBInfoPtr & new_db_info, const TiDB::TableInfoPtr & new_table_info, const ManageableStoragePtr & storage);

void applyRenamePhysicalTable(TiDB::DBInfoPtr new_db_info, TiDB::TableInfo & new_table_info, ManageableStoragePtr storage);
void applyRenamePhysicalTable(const TiDB::DBInfoPtr & new_db_info, const TiDB::TableInfo & new_table_info, const ManageableStoragePtr & storage);

void applyExchangeTablePartition(const SchemaDiff & diff);

void applySetTiFlashReplica(TiDB::DBInfoPtr db_info, TableID table_id);
void applySetTiFlashReplica(TiDB::DBInfoPtr db_info, TiDB::TableInfoPtr table_info, ManageableStoragePtr storage);
void applySetTiFlashReplica(const TiDB::DBInfoPtr & db_info, TableID table_id);
void applySetTiFlashReplicaOnLogicalTable(const TiDB::DBInfoPtr & db_info, const TiDB::TableInfoPtr & table_info, const ManageableStoragePtr & storage);
void applySetTiFlashReplicaOnPhysicalTable(const TiDB::DBInfoPtr & db_info, const TiDB::TableInfoPtr & table_info, const ManageableStoragePtr & storage);
};

} // namespace DB
Loading

0 comments on commit 597f8b8

Please sign in to comment.