Merge branch 'hongyunyan_mark_cache' of https://github.com/hongyunyan…

…/tiflash into hongyunyan_mark_cache

dbms/src/Flash/LogSearch.cpp

@@ -410,9 +410,11 @@ bool FilterFileByDatetime(
     const int64_t start_time)
 {
     static const std::string date_format_example = "0000-00-00-00:00:00.000";
-    static const std::string raftstore_proxy_date_format_example = "0000-00-00-00:00:00.000000000";
     static const char * date_format = "%d-%d-%d-%d:%d:%d.%d";
 
+    static const std::string raftstore_proxy_date_format_example = "0000-00-00T00:00:00.000.log";
+    static const char * raftstore_proxy_date_format = "%d-%d-%dT%d-%d-%d.%d";
+
     for (const auto & ignore_log_file_prefix : ignore_log_file_prefixes)
     {
         if (!ignore_log_file_prefix.empty() && startsWith(path, ignore_log_file_prefix))
@@ -441,7 +443,7 @@ bool FilterFileByDatetime(
     else
     {
         // filter proxy log end datetime
-        return filterLogEndDatetime(path, raftstore_proxy_date_format_example, date_format, start_time);
+        return filterLogEndDatetime(path, raftstore_proxy_date_format_example, raftstore_proxy_date_format, start_time);
     }
 }
 

dbms/src/Flash/tests/gtest_log_search.cpp

@@ -123,7 +123,7 @@ TEST_F(LogSearchTest, SearchDir)
     ASSERT_TRUE(FilterFileByDatetime("/1/server.log.2021-10-09-14:50:55.481.gz", {"/1/test-err.log"}, 1633855377000)); // 1633855377000 : 2021-10-10 16:42:57
     ASSERT_FALSE(FilterFileByDatetime("/1/server.log.2021-10-10-16:43:57.123.gz", {"/1/test-err.log"}, 1633855377000));
 
-    ASSERT_TRUE(FilterFileByDatetime("/1/proxy.log.2021-10-09-14:50:55.123456789", {"/1/test-err.log"}, 1633855377000));
+    ASSERT_TRUE(FilterFileByDatetime("/1/tiflash_tikv.2021-10-09T14-50-55.123.log", {"/1/test-err.log"}, 1633855377000));
 
     {
         const std::string example_data = "[2020/04/23 13:11:02.329 +08:00] [DEBUG] [\"Application : Load metadata done.\"]\n";

dbms/src/Storages/Transaction/TiDB.cpp

@@ -525,10 +525,40 @@ try
 
     auto defs_json = json->getArray("definitions");
     definitions.clear();
+    std::unordered_set<TableID> part_id_set;
     for (size_t i = 0; i < defs_json->size(); i++)
     {
         PartitionDefinition definition(defs_json->getObject(i));
         definitions.emplace_back(definition);
+        part_id_set.emplace(definition.id);
+    }
+
+    auto add_defs_json = json->getArray("adding_definitions");
+    if (!add_defs_json.isNull())
+    {
+        for (size_t i = 0; i < add_defs_json->size(); i++)
+        {
+            PartitionDefinition definition(add_defs_json->getObject(i));
+            if (part_id_set.count(definition.id) == 0)
+            {
+                definitions.emplace_back(definition);
+                part_id_set.emplace(definition.id);
+            }
+        }
+    }
+
+    auto drop_defs_json = json->getArray("dropping_definitions");
+    if (!drop_defs_json.isNull())
+    {
+        for (size_t i = 0; i < drop_defs_json->size(); i++)
+        {
+            PartitionDefinition definition(drop_defs_json->getObject(i));
+            if (part_id_set.count(definition.id) == 0)
+            {
+                definitions.emplace_back(definition);
+                part_id_set.emplace(definition.id);
+            }
+        }
     }
 
     num = json->getValue<UInt64>("num");

dbms/src/TiDB/Schema/SchemaBuilder.cpp

@@ -534,6 +534,7 @@ void SchemaBuilder<Getter, NameMapper>::applyDiff(const SchemaDiff & diff)
     case SchemaActionType::AddTablePartition:
     case SchemaActionType::DropTablePartition:
     case SchemaActionType::TruncateTablePartition:
+    case SchemaActionType::ActionReorganizePartition:
     {
         applyPartitionDiff(db_info, diff.table_id);
         break;

tests/fullstack-test2/ddl/reorganize_partition.test

@@ -0,0 +1,101 @@
+# Copyright 2023 PingCAP, Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+mysql> drop table if exists test.t;
+mysql> create table test.t (a int primary key, b varchar(255), c int, key (b), key (c,b)) partition by range (a) (partition p0 values less than (1000000), partition p1M values less than (2000000));
+mysql> analyze table test.t;
+mysql> alter table test.t set tiflash replica 1;
+
+func> wait_table test t
+
+# check table info in tiflash
+>> select tidb_database,tidb_name from system.tables where tidb_database = 'test' and tidb_name = 't' and is_tombstone = 0
+┌─tidb_database─┬─tidb_name─┐
+│ test          │ t         │
+└───────────────┴───────────┘
+
+mysql> insert into test.t values (1,"1",-1);
+mysql> insert into test.t select a+1,a+1,-(a+1) from test.t;
+mysql> insert into test.t select a+2,a+2,-(a+2) from test.t;
+mysql> insert into test.t select a+500000,a+500000,-(a+500000) from test.t;
+mysql> insert into test.t select a+1000000,a+1000000,-(a+1000000) from test.t;
+mysql> select /*+ READ_FROM_STORAGE(TIKV[t]) */ count(*) from test.t partition (p0);
++----------+
+| count(*) |
++----------+
+|        8 |
++----------+
+
+mysql> show warnings;
+mysql> select /*+ READ_FROM_STORAGE(TIFLASH[t]) */ count(*) from test.t partition (p0);
++----------+
+| count(*) |
++----------+
+|        8 |
++----------+
+
+mysql> show warnings;
+mysql> select /*+ READ_FROM_STORAGE(TIKV[t]) */ count(*) from test.t partition (p0);
++----------+
+| count(*) |
++----------+
+|        8 |
++----------+
+
+mysql> select /*+ READ_FROM_STORAGE(TIFLASH[t]) */ count(*) from test.t partition (p0);
++----------+
+| count(*) |
++----------+
+|        8 |
++----------+
+
+mysql> show warnings;
+
+mysql> alter table test.t reorganize partition p0 INTO (partition p0 values less than (500000), partition p500k values less than (1000000));
+
+mysql> select /*+ READ_FROM_STORAGE(TIFLASH[t]) */ count(*) from test.t partition (p0);
++----------+
+| count(*) |
++----------+
+|        4 |
++----------+
+
+mysql> show warnings;
+
+mysql> select /*+ READ_FROM_STORAGE(TIFLASH[t]) */ count(*) from test.t partition (p500k);
++----------+
+| count(*) |
++----------+
+|        4 |
++----------+
+
+mysql> show warnings;
+
+mysql> select /*+ READ_FROM_STORAGE(TIKV[t]) */ count(*) from test.t partition (p0);
++----------+
+| count(*) |
++----------+
+|        4 |
++----------+
+
+mysql> select /*+ READ_FROM_STORAGE(TIKV[t]) */ count(*) from test.t partition (p500k);
++----------+
+| count(*) |
++----------+
+|        4 |
++----------+
+
+mysql> show warnings;
+