[FLASH-406] CI run gtest cases

dbms/CMakeLists.txt

@@ -266,7 +266,8 @@ if (ENABLE_TESTS)
 
     # attach all dbms gtest sources
     grep_gtest_sources(${ClickHouse_SOURCE_DIR}/dbms dbms_gtest_sources)
-    add_executable(unit_tests_dbms ${dbms_gtest_sources})
-    target_link_libraries(unit_tests_dbms gtest_main dbms)
-    add_check(unit_tests_dbms)
+    add_executable(gtests_dbms EXCLUDE_FROM_ALL ${dbms_gtest_sources})
+    target_link_libraries(gtests_dbms gtest_main dbms clickhouse_functions)
+    target_compile_options(gtests_dbms PRIVATE -Wno-unknown-pragmas)
+    add_check(gtests_dbms)
 endif ()

dbms/src/Analyzers/CMakeLists.txt

@@ -1,3 +1,3 @@
 if(ENABLE_TESTS)
-    add_subdirectory(tests)
+    add_subdirectory(tests EXCLUDE_FROM_ALL)
 endif()

dbms/src/Common/CMakeLists.txt

@@ -4,5 +4,5 @@ add_subdirectory(StringUtils)
 #add_subdirectory(ConfigProcessor)
 
 if (ENABLE_TESTS)
-    add_subdirectory (tests)
+    add_subdirectory (tests EXCLUDE_FROM_ALL)
 endif ()

dbms/src/Common/ZooKeeper/CMakeLists.txt

@@ -7,5 +7,5 @@ add_library(clickhouse_common_zookeeper ${SPLIT_SHARED} ${clickhouse_common_zook
 target_link_libraries (clickhouse_common_zookeeper clickhouse_common_io)
 
 if (ENABLE_TESTS)
-    add_subdirectory (tests)
+    add_subdirectory (tests EXCLUDE_FROM_ALL)
 endif ()

dbms/src/Common/ZooKeeper/tests/gtest_zkutil_test_multi_exception.cpp

@@ -15,7 +15,9 @@
 
 using namespace DB;
 
-TEST(zkutil, zookeeper_connected)
+/// Disable zookeeper tests since tiflash don't use zookeeper
+
+TEST(zkutil, DISABLED_zookeeper_connected)
 {
     auto zookeeper = std::make_unique<zkutil::ZooKeeper>("localhost:2181");
     try
@@ -29,7 +31,7 @@ TEST(zkutil, zookeeper_connected)
     }
 }
 
-TEST(zkutil, multi_nice_exception_msg)
+TEST(zkutil, DISABLED_multi_nice_exception_msg)
 {
     auto zookeeper = std::make_unique<zkutil::ZooKeeper>("localhost:2181");
 
@@ -67,7 +69,7 @@ TEST(zkutil, multi_nice_exception_msg)
 }
 
 
-TEST(zkutil, multi_async)
+TEST(zkutil, DISABLED_multi_async)
 {
     auto zookeeper = std::make_unique<zkutil::ZooKeeper>("localhost:2181");
     zkutil::Requests ops;
@@ -129,7 +131,7 @@ TEST(zkutil, multi_async)
     }
 }
 
-TEST(zkutil, watch_get_children_with_chroot)
+TEST(zkutil, DISABLED_watch_get_children_with_chroot)
 {
     try
     {
@@ -160,7 +162,7 @@ TEST(zkutil, watch_get_children_with_chroot)
     }
 }
 
-TEST(zkutil, multi_create_sequential)
+TEST(zkutil, DISABLED_multi_create_sequential)
 {
     try
     {

dbms/src/Core/CMakeLists.txt

@@ -1,3 +1,3 @@
 if (ENABLE_TESTS)
-    add_subdirectory (tests)
+    add_subdirectory (tests EXCLUDE_FROM_ALL)
 endif ()

dbms/src/DataStreams/CMakeLists.txt

@@ -1,3 +1,3 @@
 if (ENABLE_TESTS)
-    add_subdirectory (tests)
+    add_subdirectory (tests EXCLUDE_FROM_ALL)
 endif ()

dbms/src/DataStreams/tests/CMakeLists.txt

@@ -3,28 +3,28 @@ include_directories (${CMAKE_CURRENT_BINARY_DIR})
 set(SRCS )
 
 add_executable (tab_separated_streams tab_separated_streams.cpp ${SRCS})
-target_link_libraries (tab_separated_streams dbms)
+target_link_libraries (tab_separated_streams dbms clickhouse_functions)
 
 add_executable (block_row_transforms block_row_transforms.cpp ${SRCS})
-target_link_libraries (block_row_transforms dbms)
+target_link_libraries (block_row_transforms dbms clickhouse_functions)
 
 add_executable (expression_stream expression_stream.cpp ${SRCS})
-target_link_libraries (expression_stream dbms clickhouse_storages_system)
+target_link_libraries (expression_stream dbms clickhouse_storages_system clickhouse_functions)
 
 add_executable (native_streams native_streams.cpp ${SRCS})
-target_link_libraries (native_streams dbms)
+target_link_libraries (native_streams dbms clickhouse_functions)
 
 add_executable (filter_stream filter_stream.cpp ${SRCS})
-target_link_libraries (filter_stream dbms clickhouse_storages_system)
+target_link_libraries (filter_stream dbms clickhouse_storages_system clickhouse_functions)
 
 add_executable (filter_stream_hitlog filter_stream_hitlog.cpp ${SRCS})
-target_link_libraries (filter_stream_hitlog dbms)
+target_link_libraries (filter_stream_hitlog dbms clickhouse_functions)
 
 add_executable (sorting_stream sorting_stream.cpp ${SRCS})
-target_link_libraries (sorting_stream dbms)
+target_link_libraries (sorting_stream dbms clickhouse_functions)
 
 add_executable (union_stream2 union_stream2.cpp ${SRCS})
-target_link_libraries (union_stream2 dbms)
+target_link_libraries (union_stream2 dbms clickhouse_functions)
 
 add_executable (collapsing_sorted_stream collapsing_sorted_stream.cpp ${SRCS})
-target_link_libraries (collapsing_sorted_stream dbms)
+target_link_libraries (collapsing_sorted_stream dbms clickhouse_functions)

dbms/src/DataTypes/CMakeLists.txt

@@ -1,3 +1,3 @@
 if (ENABLE_TESTS)
-    add_subdirectory (tests)
+    add_subdirectory (tests EXCLUDE_FROM_ALL)
 endif ()

dbms/src/Debug/DBGInvoker.cpp

@@ -34,9 +34,11 @@ DBGInvoker::DBGInvoker()
     regFunc("sleep", dbgFuncSleep);
 
     regFunc("mock_tidb_table", MockTiDBTable::dbgFuncMockTiDBTable);
+    regFunc("mock_tidb_db", MockTiDBTable::dbgFuncMockTiDBDB);
     regFunc("mock_tidb_partition", MockTiDBTable::dbgFuncMockTiDBPartition);
     regFunc("rename_table_for_partition", MockTiDBTable::dbgFuncRenameTableForPartition);
     regFunc("drop_tidb_table", MockTiDBTable::dbgFuncDropTiDBTable);
+    regFunc("drop_tidb_db", MockTiDBTable::dbgFuncDropTiDBDB);
     regFunc("add_column_to_tidb_table", MockTiDBTable::dbgFuncAddColumnToTiDBTable);
     regFunc("drop_column_from_tidb_table", MockTiDBTable::dbgFuncDropColumnFromTiDBTable);
     regFunc("modify_column_in_tidb_table", MockTiDBTable::dbgFuncModifyColumnInTiDBTable);
@@ -63,6 +65,7 @@ DBGInvoker::DBGInvoker()
 
     regFunc("enable_schema_sync_service", dbgFuncEnableSchemaSyncService);
     regFunc("refresh_schemas", dbgFuncRefreshSchemas);
+    regFunc("reset_schemas", dbgFuncResetSchemas);
 }
 
 void replaceSubstr(std::string & str, const std::string & target, const std::string & replacement)

dbms/src/Debug/MockTiDB.cpp

@@ -26,7 +26,25 @@ Table::Table(const String & database_name_, const String & table_name_, TableInf
     : table_info(std::move(table_info_)), database_name(database_name_), table_name(table_name_)
 {}
 
-void MockTiDB::dropTable(const String & database_name, const String & table_name)
+MockTiDB::MockTiDB() { databases["default"] = 0; }
+
+void MockTiDB::dropDB(const String & database_name)
+{
+    version++;
+
+    SchemaDiff diff;
+    diff.type = SchemaActionDropSchema;
+    if (databases.find(database_name) == databases.end())
+        diff.schema_id = -1;
+    else
+        diff.schema_id = databases[database_name];
+    diff.version = version;
+    version_diff[version] = diff;
+
+    databases.erase(database_name);
+}
+
+void MockTiDB::dropTable(const String & database_name, const String & table_name, bool is_drop_db)
 {
     std::lock_guard lock(tables_mutex);
 
@@ -47,14 +65,17 @@ void MockTiDB::dropTable(const String & database_name, const String & table_name
 
     tables_by_name.erase(it_by_name);
 
-    version++;
-
-    SchemaDiff diff;
-    diff.type = SchemaActionDropTable;
-    diff.schema_id = table->table_info.db_id;
-    diff.table_id = table->id();
-    diff.version = version;
-    version_diff[version] = diff;
+    if (!is_drop_db)
+    {
+        version++;
+
+        SchemaDiff diff;
+        diff.type = SchemaActionDropTable;
+        diff.schema_id = table->table_info.db_id;
+        diff.table_id = table->id();
+        diff.version = version;
+        version_diff[version] = diff;
+    }
 }
 
 ColumnInfo getColumnInfoFromColumn(const NameAndTypePair & column, ColumnID id)
@@ -112,6 +133,26 @@ ColumnInfo getColumnInfoFromColumn(const NameAndTypePair & column, ColumnID id)
     return column_info;
 }
 
+DatabaseID MockTiDB::newDataBase(const String & database_name)
+{
+    DatabaseID schema_id = 0;
+
+    if (databases.find(database_name) == databases.end())
+    {
+        schema_id = databases.size() + 1;
+        databases.emplace(database_name, schema_id);
+    }
+
+    version++;
+    SchemaDiff diff;
+    diff.type = SchemaActionCreateSchema;
+    diff.schema_id = schema_id;
+    diff.version = version;
+    version_diff[version] = diff;
+
+    return schema_id;
+}
+
 TableID MockTiDB::newTable(const String & database_name, const String & table_name, const ColumnsDescription & columns, Timestamp tso)
 {
     std::lock_guard lock(tables_mutex);
@@ -124,9 +165,9 @@ TableID MockTiDB::newTable(const String & database_name, const String & table_na
 
     TableInfo table_info;
 
-    if (databases.find(database_name) != databases.end())
+    if (databases.find(database_name) == databases.end())
     {
-        databases.emplace(database_name, databases.size());
+        throw Exception("MockTiDB not found db: " + database_name, ErrorCodes::LOGICAL_ERROR);
     }
     table_info.db_id = databases[database_name];
     table_info.db_name = database_name;

dbms/src/Debug/MockTiDB.h

@@ -23,6 +23,7 @@ class MockTiDB : public ext::singleton<MockTiDB>
     friend class ext::singleton<MockTiDB>;
 
 public:
+    MockTiDB();
     class Table
     {
         friend class MockTiDB;
@@ -65,9 +66,13 @@ class MockTiDB : public ext::singleton<MockTiDB>
 public:
     TableID newTable(const String & database_name, const String & table_name, const ColumnsDescription & columns, Timestamp tso);
 
+    DatabaseID newDataBase(const String & database_name);
+
     TableID newPartition(const String & database_name, const String & table_name, const String & partition_name, Timestamp tso);
 
-    void dropTable(const String & database_name, const String & table_name);
+    void dropTable(const String & database_name, const String & table_name, bool is_drop_db);
+
+    void dropDB(const String & database_name);
 
     void addColumnToTable(const String & database_name, const String & table_name, const NameAndTypePair & column);
 

dbms/src/Debug/dbgFuncMockTiDBTable.cpp

@@ -49,6 +49,20 @@ void MockTiDBTable::dbgFuncMockTiDBTable(Context & context, const ASTs & args, D
     output(ss.str());
 }
 
+void MockTiDBTable::dbgFuncMockTiDBDB(Context &, const ASTs & args, DBGInvoker::Printer output)
+{
+    if (args.size() != 1)
+        throw Exception("Args not matched, should be: database-name", ErrorCodes::BAD_ARGUMENTS);
+
+    const String & database_name = typeid_cast<const ASTIdentifier &>(*args[0]).name;
+
+    DatabaseID db_id = MockTiDB::instance().newDataBase(database_name);
+
+    std::stringstream ss;
+    ss << "mock db #" << db_id;
+    output(ss.str());
+}
+
 void MockTiDBTable::dbgFuncMockTiDBPartition(Context & context, const ASTs & args, DBGInvoker::Printer output)
 {
     if (args.size() != 3)
@@ -96,6 +110,26 @@ void MockTiDBTable::dbgFuncRenameTableForPartition(Context & context, const ASTs
     output(ss.str());
 }
 
+void MockTiDBTable::dbgFuncDropTiDBDB(Context & context, const ASTs & args, DBGInvoker::Printer output)
+{
+    if (args.size() != 1 && args.size() != 2)
+        throw Exception("Args not matched, should be: database-name [, drop-regions]", ErrorCodes::BAD_ARGUMENTS);
+
+    const String & database_name = typeid_cast<const ASTIdentifier &>(*args[0]).name;
+    bool drop_regions = true;
+    if (args.size() == 3)
+        drop_regions = typeid_cast<const ASTIdentifier &>(*args[1]).name == "true";
+
+    std::vector<String> table_names;
+    MockTiDB::instance().traverseTables([&](MockTiDB::TablePtr table) {
+        if (table->table_info.db_name == database_name)
+            table_names.push_back(table->table_info.name);
+    });
+    for (auto table_name : table_names)
+        dbgFuncDropTiDBTableImpl(context, database_name, table_name, drop_regions, true, output);
+    MockTiDB::instance().dropDB(database_name);
+}
+
 void MockTiDBTable::dbgFuncDropTiDBTable(Context & context, const ASTs & args, DBGInvoker::Printer output)
 {
     if (args.size() != 2 && args.size() != 3)
@@ -106,7 +140,12 @@ void MockTiDBTable::dbgFuncDropTiDBTable(Context & context, const ASTs & args, D
     bool drop_regions = true;
     if (args.size() == 3)
         drop_regions = typeid_cast<const ASTIdentifier &>(*args[1]).name == "true";
+    dbgFuncDropTiDBTableImpl(context, database_name, table_name, drop_regions, false, output);
+}
 
+void MockTiDBTable::dbgFuncDropTiDBTableImpl(
+    Context & context, String database_name, String table_name, bool drop_regions, bool is_drop_db, DBGInvoker::Printer output)
+{
     MockTiDB::TablePtr table = nullptr;
     TableID table_id = InvalidTableID;
     try
@@ -144,7 +183,7 @@ void MockTiDBTable::dbgFuncDropTiDBTable(Context & context, const ASTs & args, D
         region_table.mockDropRegionsInTable(table_id);
     }
 
-    MockTiDB::instance().dropTable(database_name, table_name);
+    MockTiDB::instance().dropTable(database_name, table_name, is_drop_db);
 
     std::stringstream ss;
     ss << "dropped table #" << table_id;

dbms/src/Debug/dbgFuncMockTiDBTable.h

@@ -17,6 +17,11 @@ struct MockTiDBTable
     //   ./storages-client.sh "DBGInvoke mock_tidb_table(database_name, table_name, 'col1 type1, col2 type2, ...')"
     static void dbgFuncMockTiDBTable(Context & context, const ASTs & args, DBGInvoker::Printer output);
 
+    // Inject mocked TiDB table.
+    // Usage:
+    //   ./storages-client.sh "DBGInvoke mock_tidb_db(database_name)
+    static void dbgFuncMockTiDBDB(Context & context, const ASTs & args, DBGInvoker::Printer output);
+
     // Inject a partition into mocked TiDB table.
     // Usage:
     //   ./storages-client.sh "DBGInvoke mock_tidb_partition(database_name, table_name, partition_name)"
@@ -34,6 +39,11 @@ struct MockTiDBTable
     //   ./storages-client.sh "DBGInvoke drop_tidb_table(database_name, table_name)"
     static void dbgFuncDropTiDBTable(Context & context, const ASTs & args, DBGInvoker::Printer output);
 
+    // Drop a mocked TiDB table.
+    // Usage:
+    //   ./storages-client.sh "DBGInvoke drop_tidb_db(database_name)"
+    static void dbgFuncDropTiDBDB(Context & context, const ASTs & args, DBGInvoker::Printer output);
+
     // Add a column to a mocked TiDB table.
     // Usage:
     //   ./storages-client.sh "DBGInvoke add_column_to_tidb_table(database_name, table_name, 'col type')"
@@ -58,6 +68,15 @@ struct MockTiDBTable
     // Usage:
     //   ./storages-client.sh "DBGInvoke truncate_tidb_table(database_name, table_name)"
     static void dbgFuncTruncateTiDBTable(Context & context, const ASTs & args, DBGInvoker::Printer output);
+
+    // Reset Schema Syncer.
+    // Usage:
+    //   ./storages-client.sh "DBGInvoke reset_syncer()"
+    static void dbgFuncResetSyncer(Context & context, const ASTs & args, DBGInvoker::Printer output);
+
+private:
+    static void dbgFuncDropTiDBTableImpl(
+        Context & context, String database_name, String table_name, bool drop_regions, bool is_drop_db, DBGInvoker::Printer output);
 };
 
 } // namespace DB

dbms/src/Debug/dbgFuncSchema.cpp

@@ -49,4 +49,15 @@ void dbgFuncRefreshSchemas(Context & context, const ASTs &, DBGInvoker::Printer
     output(ss.str());
 }
 
+void dbgFuncResetSchemas(Context & context, const ASTs &, DBGInvoker::Printer output)
+{
+    TMTContext & tmt = context.getTMTContext();
+    auto schema_syncer = tmt.getSchemaSyncer();
+    schema_syncer->reset();
+
+    std::stringstream ss;
+    ss << "reset schemas";
+    output(ss.str());
+}
+
 } // namespace DB