[FLASH-231] Refactor Structure And Module

dbms/src/DataStreams/RangesFilterBlockInputStream.cpp

@@ -10,9 +10,6 @@ namespace ErrorCodes
 extern const int LOGICAL_ERROR;
 }
 
-template class RangesFilterBlockInputStream<Int64>;
-template class RangesFilterBlockInputStream<UInt64>;
-
 struct PKColumnIterator : public std::iterator<std::random_access_iterator_tag, UInt64, size_t>
 {
     PKColumnIterator & operator++()
@@ -112,4 +109,7 @@ Block RangesFilterBlockInputStream<HandleType>::readImpl()
     }
 }
 
+template class RangesFilterBlockInputStream<Int64>;
+template class RangesFilterBlockInputStream<UInt64>;
+
 } // namespace DB

dbms/src/DataStreams/ReplacingTMTSortedBlockInputStream.cpp

@@ -3,8 +3,6 @@
 
 namespace DB
 {
-template class ReplacingTMTSortedBlockInputStream<Int64>;
-template class ReplacingTMTSortedBlockInputStream<UInt64>;
 
 template <typename HandleType>
 void ReplacingTMTSortedBlockInputStream<HandleType>::insertRow(MutableColumns & merged_columns, size_t & merged_rows)
@@ -181,4 +179,7 @@ void ReplacingTMTSortedBlockInputStream<HandleType>::logRowGoing(const std::stri
                    << ". same=" << ((toString(curr_pk) == next_pk) ? "true" : "false") << ". why{" << msg << "}, output: " << is_output);
 }
 
+template class ReplacingTMTSortedBlockInputStream<Int64>;
+template class ReplacingTMTSortedBlockInputStream<UInt64>;
+
 } // namespace DB

dbms/src/Debug/dbgFuncMockTiDBTable.cpp

@@ -1,21 +1,16 @@
+#include <Debug/MockTiDB.h>
+#include <Debug/dbgFuncMockTiDBTable.h>
+#include <Interpreters/InterpreterCreateQuery.h>
+#include <Interpreters/InterpreterRenameQuery.h>
 #include <Parsers/ASTExpressionList.h>
 #include <Parsers/ASTIdentifier.h>
 #include <Parsers/ASTLiteral.h>
 #include <Parsers/ParserCreateQuery.h>
 #include <Parsers/ParserRenameQuery.h>
 #include <Parsers/parseQuery.h>
-
-#include <Interpreters/InterpreterCreateQuery.h>
-#include <Interpreters/InterpreterRenameQuery.h>
-
 #include <Storages/Transaction/SchemaSyncer.h>
 #include <Storages/Transaction/TMTContext.h>
 
-#include <Raft/RaftContext.h>
-
-#include <Debug/MockTiDB.h>
-#include <Debug/dbgFuncMockTiDBTable.h>
-
 namespace DB
 {
 

dbms/src/Debug/dbgFuncRegion.cpp

@@ -1,23 +1,19 @@
 #include <Common/typeid_cast.h>
-
+#include <Debug/MockTiDB.h>
+#include <Debug/dbgFuncRegion.h>
+#include <Debug/dbgTools.h>
+#include <Interpreters/executeQuery.h>
 #include <Parsers/ASTIdentifier.h>
 #include <Parsers/ASTLiteral.h>
-
 #include <Storages/MutableSupport.h>
+#include <Storages/StorageMergeTree.h>
 #include <Storages/Transaction/KVStore.h>
-#include <Storages/Transaction/TMTContext.h>
-#include <Storages/Transaction/applySnapshot.h>
-
 #include <Storages/Transaction/Region.h>
+#include <Storages/Transaction/TMTContext.h>
 #include <Storages/Transaction/TiKVRange.h>
+#include <Storages/Transaction/applySnapshot.h>
 #include <Storages/Transaction/tests/region_helper.h>
 
-#include <Debug/MockTiDB.h>
-#include <Debug/dbgFuncRegion.h>
-#include <Debug/dbgTools.h>
-
-#include <Interpreters/executeQuery.h>
-
 namespace DB
 {
 

dbms/src/Debug/dbgTools.cpp

@@ -1,13 +1,12 @@
 #include <random>
 
-#include <Parsers/ASTLiteral.h>
-
-#include <Storages/Transaction/Codec.h>
-
+#include <Common/typeid_cast.h>
 #include <Debug/MockTiKV.h>
 #include <Debug/dbgTools.h>
+#include <Interpreters/Context.h>
+#include <Parsers/ASTLiteral.h>
 #include <Raft/RaftContext.h>
-
+#include <Storages/Transaction/Codec.h>
 #include <Storages/Transaction/KVStore.h>
 #include <Storages/Transaction/Region.h>
 #include <Storages/Transaction/TMTContext.h>

dbms/src/Debug/dbgTools.h

@@ -1,5 +1,12 @@
 #pragma once
 
+#include <Parsers/IAST.h>
+
+namespace TiDB
+{
+struct TableInfo;
+}
+
 namespace DB
 {
 class Context;

dbms/src/Raft/RaftService.cpp

@@ -1,5 +1,6 @@
 #include <Interpreters/Context.h>
 #include <Raft/RaftService.h>
+#include <Storages/MergeTree/BackgroundProcessingPool.h>
 #include <Storages/Transaction/KVStore.h>
 #include <Storages/Transaction/TMTContext.h>
 #include <Storages/Transaction/applySnapshot.h>

dbms/src/Raft/RaftService.h

@@ -1,7 +1,6 @@
 #pragma once
 
 #include <memory>
-#include <string>
 
 #include <common/logger_useful.h>
 #include <boost/noncopyable.hpp>

dbms/src/Storages/StorageMergeTree.h

@@ -2,16 +2,19 @@
 
 #include <ext/shared_ptr_helper.h>
 
+#include <Common/SimpleIncrement.h>
 #include <Storages/IStorage.h>
+#include <Storages/MergeTree/BackgroundProcessingPool.h>
+#include <Storages/MergeTree/DiskSpaceMonitor.h>
 #include <Storages/MergeTree/MergeTreeData.h>
+#include <Storages/MergeTree/MergeTreeDataMerger.h>
 #include <Storages/MergeTree/MergeTreeDataSelectExecutor.h>
 #include <Storages/MergeTree/MergeTreeDataWriter.h>
-#include <Storages/MergeTree/MergeTreeDataMerger.h>
-#include <Storages/MergeTree/DiskSpaceMonitor.h>
-#include <Storages/MergeTree/BackgroundProcessingPool.h>
-#include <Storages/Transaction/TiDB.h>
-#include <Common/SimpleIncrement.h>
 
+namespace TiDB
+{
+struct TableInfo;
+}
 
 namespace DB
 {
@@ -20,8 +23,8 @@ namespace DB
   */
 class StorageMergeTree : public ext::shared_ptr_helper<StorageMergeTree>, public IStorage
 {
-friend class MergeTreeBlockOutputStream;
-friend class TxnMergeTreeBlockOutputStream;
+    friend class MergeTreeBlockOutputStream;
+    friend class TxnMergeTreeBlockOutputStream;
 
     using TableInfo = TiDB::TableInfo;
 
@@ -30,10 +33,7 @@ friend class TxnMergeTreeBlockOutputStream;
     void shutdown() override;
     ~StorageMergeTree() override;
 
-    std::string getName() const override
-    {
-        return data.merging_params.getModeName() + "MergeTree";
-    }
+    std::string getName() const override { return data.merging_params.getModeName() + "MergeTree"; }
 
     std::string getTableName() const override { return table_name; }
     std::string getDatabaseName() const { return database_name; }
@@ -47,18 +47,11 @@ friend class TxnMergeTreeBlockOutputStream;
     const ColumnsDescription & getColumns() const override { return data.getColumns(); }
     void setColumns(ColumnsDescription columns_) override { return data.setColumns(std::move(columns_)); }
 
-    NameAndTypePair getColumn(const String & column_name) const override
-    {
-        return data.getColumn(column_name);
-    }
+    NameAndTypePair getColumn(const String & column_name) const override { return data.getColumn(column_name); }
 
-    bool hasColumn(const String & column_name) const override
-    {
-        return data.hasColumn(column_name);
-    }
+    bool hasColumn(const String & column_name) const override { return data.hasColumn(column_name); }
 
-    BlockInputStreams read(
-        const Names & column_names,
+    BlockInputStreams read(const Names & column_names,
         const SelectQueryInfo & query_info,
         const Context & context,
         QueryProcessingStage::Enum & processed_stage,
@@ -121,7 +114,7 @@ friend class TxnMergeTreeBlockOutputStream;
 
     Logger * log;
 
-    std::atomic<bool> shutdown_called {false};
+    std::atomic<bool> shutdown_called{false};
 
     BackgroundProcessingPool::TaskHandle merge_task_handle;
 
@@ -134,7 +127,7 @@ friend class TxnMergeTreeBlockOutputStream;
       * Returns true if merge is finished successfully.
       */
     bool merge(size_t aio_threshold, bool aggressive, const String & partition_id, bool final, bool deduplicate,
-               String * out_disable_reason = nullptr);
+        String * out_disable_reason = nullptr);
 
     bool mergeTask();
 
@@ -148,8 +141,7 @@ friend class TxnMergeTreeBlockOutputStream;
       * date_column_name      - if not empty, the name of the column with the date used for partitioning by month;
           otherwise, partition_expr_ast is used as the partitioning expression;
       */
-    StorageMergeTree(
-        const String & path_,
+    StorageMergeTree(const String & path_,
         const String & database_name_,
         const String & table_name_,
         const ColumnsDescription & columns_,
@@ -166,4 +158,4 @@ friend class TxnMergeTreeBlockOutputStream;
         bool has_force_restore_data_flag);
 };
 
-}
+} // namespace DB

dbms/src/Storages/Transaction/KVStore.cpp

@@ -1,3 +1,4 @@
+#include <Interpreters/Context.h>
 #include <Raft/RaftContext.h>
 #include <Storages/Transaction/KVStore.h>
 #include <Storages/Transaction/Region.h>
@@ -69,7 +70,7 @@ void KVStore::onSnapshot(RegionPtr new_region, RegionTable * region_table)
                 LOG_DEBUG(log, "KVStore::onSnapshot: discard new region because of index is outdated");
                 return;
             }
-            old_region->reset(std::move(*new_region));
+            old_region->assignRegion(std::move(*new_region));
             new_region = old_region;
         }
         else
@@ -229,7 +230,7 @@ bool KVStore::tryPersistAndReport(RaftContext & context, const Seconds kvstore_t
     traverseRegions([&](const RegionID region_id, const RegionPtr & region) {
         if (now < (region->lastPersistTime() + region_persist_period))
             return;
-        if (region->persistParm() == 0)
+        if (region->dirtyFlag() == 0)
             return;
         all_region_copy[region_id] = region;
     });
@@ -244,7 +245,7 @@ bool KVStore::tryPersistAndReport(RaftContext & context, const Seconds kvstore_t
 
         region_persister.persist(region, response);
 
-        ss << "(" << region_id << "," << region->persistParm() << ") ";
+        ss << "(" << region_id << "," << region->dirtyFlag() << ") ";
     }
 
     if (persist_job)

dbms/src/Storages/Transaction/PartitionStreams.cpp

@@ -41,7 +41,7 @@ std::tuple<BlockInputStreamPtr, RegionTable::RegionReadStatus, size_t> RegionTab
     const Names & ordered_columns,
     bool learner_read,
     bool resolve_locks,
-    UInt64 start_ts,
+    Timestamp start_ts,
     RegionDataReadInfoList * data_list_for_remove)
 {
     if (!region)