From 959bd31183dc24cd5dec7163197e1d13f79b8f8e Mon Sep 17 00:00:00 2001 From: zhangdong <493738387@qq.com> Date: Fri, 1 Nov 2024 19:22:28 +0800 Subject: [PATCH] [fix](mtmv) cannot refresh mtmv when base table recreating (#41762) (#42340) pick from master #41762 fix in the scenario of recreating a table, the materialized view may assume that the data has not changed --- .../org/apache/doris/catalog/OlapTable.java | 2 +- .../mtmv/MTMVRefreshPartitionSnapshot.java | 34 +++++++ .../data/mtmv_p0/test_recreate_table_mtmv.out | 13 +++ .../mtmv_p0/test_recreate_table_mtmv.groovy | 91 +++++++++++++++++++ 4 files changed, 139 insertions(+), 1 deletion(-) create mode 100644 regression-test/data/mtmv_p0/test_recreate_table_mtmv.out create mode 100644 regression-test/suites/mtmv_p0/test_recreate_table_mtmv.groovy diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java index 5743a5e5de15e4..f5ea56991d0417 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java @@ -3163,7 +3163,7 @@ public MTMVSnapshotIf getPartitionSnapshot(String partitionName, MTMVRefreshCont public MTMVSnapshotIf getTableSnapshot(MTMVRefreshContext context) { Map tableVersions = context.getBaseVersions().getTableVersions(); long visibleVersion = tableVersions.containsKey(id) ? tableVersions.get(id) : getVisibleVersion(); - return new MTMVVersionSnapshot(visibleVersion); + return new MTMVVersionSnapshot(visibleVersion, id); } @Override diff --git a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRefreshPartitionSnapshot.java b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRefreshPartitionSnapshot.java index fa17ed766661d0..a8de5b6597bc49 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRefreshPartitionSnapshot.java +++ b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRefreshPartitionSnapshot.java @@ -20,6 +20,7 @@ import org.apache.doris.catalog.MTMV; import org.apache.doris.catalog.OlapTable; import org.apache.doris.catalog.Partition; +import org.apache.doris.catalog.TableIf; import org.apache.doris.common.AnalysisException; import com.google.common.collect.Maps; @@ -89,6 +90,13 @@ public void compatible(MTMV mtmv) { } catch (Throwable e) { LOG.warn("MTMV compatibleTables failed, mtmv: {}", mtmv.getName(), e); } + + try { + // snapshot add tableId resolve problem of recreate table + compatibleTablesSnapshot(); + } catch (Throwable e) { + LOG.warn("MTMV compatibleTables failed, mtmv: {}", mtmv.getName(), e); + } } private void compatiblePartitions(MTMV mtmv) throws AnalysisException { @@ -116,6 +124,32 @@ private boolean checkHasDataWithoutPartitionId() { return false; } + private void compatibleTablesSnapshot() { + if (!checkHasDataWithoutTableId()) { + return; + } + for (Entry entry : tablesInfo.entrySet()) { + MTMVVersionSnapshot versionSnapshot = (MTMVVersionSnapshot) entry.getValue(); + if (versionSnapshot.getId() == 0) { + try { + TableIf table = MTMVUtil.getTable(entry.getKey()); + versionSnapshot.setId(table.getId()); + } catch (AnalysisException e) { + LOG.warn("MTMV compatibleTablesSnapshot failed, can not get table by: {}", entry.getKey()); + } + } + } + } + + private boolean checkHasDataWithoutTableId() { + for (MTMVSnapshotIf snapshot : tablesInfo.values()) { + if (snapshot instanceof MTMVVersionSnapshot && ((MTMVVersionSnapshot) snapshot).getId() == 0) { + return true; + } + } + return false; + } + private void compatibleTables(MTMV mtmv) { if (tables.size() == tablesInfo.size()) { return; diff --git a/regression-test/data/mtmv_p0/test_recreate_table_mtmv.out b/regression-test/data/mtmv_p0/test_recreate_table_mtmv.out new file mode 100644 index 00000000000000..688518a2a4de03 --- /dev/null +++ b/regression-test/data/mtmv_p0/test_recreate_table_mtmv.out @@ -0,0 +1,13 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !mv1 -- +21 mysql test1 + +-- !mv2 -- +mysql 1 + +-- !mv1_recreate -- +21 doris test1 + +-- !mv2_recreate -- +doris 1 + diff --git a/regression-test/suites/mtmv_p0/test_recreate_table_mtmv.groovy b/regression-test/suites/mtmv_p0/test_recreate_table_mtmv.groovy new file mode 100644 index 00000000000000..e9f7f339a0c6a7 --- /dev/null +++ b/regression-test/suites/mtmv_p0/test_recreate_table_mtmv.groovy @@ -0,0 +1,91 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import org.junit.Assert; + +suite("test_recreate_table_mtmv","mtmv") { + String suiteName = "test_recreate_table_mtmv" + String tableName1 = "${suiteName}_table1" + String tableName2 = "${suiteName}_table2" + String mvName1 = "${suiteName}_mv1" + String mvName2 = "${suiteName}_mv2" + sql """drop table if exists `${tableName1}`""" + sql """drop table if exists `${tableName2}`""" + sql """drop materialized view if exists ${mvName1};""" + sql """drop materialized view if exists ${mvName2};""" + + sql """ + CREATE TABLE `${tableName1}` ( + `id` int NULL, + `first_name` varchar(255) NULL, + `last_name` varchar(255) NULL + ) ENGINE=OLAP + AGGREGATE KEY(`id`, `first_name`, `last_name`) + COMMENT 'OLAP' + DISTRIBUTED BY HASH(`id`) BUCKETS 2 + PROPERTIES ('replication_num' = '1') ; + """ + sql """ + INSERT INTO `${tableName1}` (`id`, `first_name`, `last_name`) VALUES (21, 'mysql', 'test1'); + """ + sql """ + CREATE TABLE `${tableName2}` ( + `id` int NULL, + `first_name` varchar(255) NULL, + `last_name` varchar(255) NULL + ) ENGINE=OLAP + AGGREGATE KEY(`id`, `first_name`, `last_name`) + COMMENT 'OLAP' + DISTRIBUTED BY HASH(`id`) BUCKETS 2 + PROPERTIES ('replication_num' = '1') ; + """ + sql """ + INSERT INTO `${tableName2}` (`id`, `first_name`, `last_name`) VALUES (21, 'doris', 'test1'); + """ + sql """ + CREATE MATERIALIZED VIEW IF NOT EXISTS ${mvName1} BUILD IMMEDIATE REFRESH AUTO ON COMMIT DISTRIBUTED BY RANDOM BUCKETS 2 + PROPERTIES ( + "replication_allocation" = "tag.location.default: 1" + ) AS (SELECT * FROM ${tableName1} ); + """ + waitingMTMVTaskFinishedByMvName(mvName1); + order_qt_mv1 "SELECT * FROM ${mvName1}" + sql """ + CREATE MATERIALIZED VIEW IF NOT EXISTS ${mvName2} BUILD IMMEDIATE REFRESH AUTO ON COMMIT DISTRIBUTED BY RANDOM BUCKETS 2 + PROPERTIES ( + "replication_allocation" = "tag.location.default: 1" + ) AS (SELECT first_name,count(last_name) FROM ${mvName1} GROUP BY first_name); + """ + waitingMTMVTaskFinishedByMvName(mvName2); + order_qt_mv2 "SELECT * FROM ${mvName2}" + sql """drop materialized view if exists ${mvName1};""" + sql """ + CREATE MATERIALIZED VIEW IF NOT EXISTS ${mvName1} BUILD IMMEDIATE REFRESH AUTO ON COMMIT DISTRIBUTED BY RANDOM BUCKETS 2 + PROPERTIES ( + "replication_allocation" = "tag.location.default: 1" + ) AS (SELECT * FROM ${tableName2} ); + """ + waitingMTMVTaskFinishedByMvName(mvName1); + order_qt_mv1_recreate "SELECT * FROM ${mvName1}" + waitingMTMVTaskFinishedByMvName(mvName2); + order_qt_mv2_recreate "SELECT * FROM ${mvName2}" + + sql """drop table if exists `${tableName1}`""" + sql """drop table if exists `${tableName2}`""" + sql """drop materialized view if exists ${mvName1};""" + sql """drop materialized view if exists ${mvName2};""" +}