diff --git a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPlanUtil.java b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPlanUtil.java index a8bc43a159e4532..ab87e834409d186 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPlanUtil.java +++ b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPlanUtil.java @@ -38,6 +38,8 @@ import org.apache.doris.nereids.trees.plans.visitor.TableCollector.TableCollectorContext; import org.apache.doris.qe.ConnectContext; +import com.google.common.collect.Sets; + import java.util.List; import java.util.Set; @@ -77,12 +79,7 @@ private static Set getBaseTables(Plan plan) { } private static Set getBaseViews(Plan plan) { - TableCollectorContext collectorContext = - new TableCollector.TableCollectorContext( - com.google.common.collect.Sets.newHashSet(TableType.VIEW)); - plan.accept(TableCollector.INSTANCE, collectorContext); - List collectedTables = collectorContext.getCollectedTables(); - return transferTableIfToInfo(collectedTables); + return Sets.newHashSet(); } private static Set transferTableIfToInfo(List tables) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateMTMVInfo.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateMTMVInfo.java index 6d81191c8bed900..0ed1bfba7be27a3 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateMTMVInfo.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateMTMVInfo.java @@ -30,6 +30,7 @@ import org.apache.doris.catalog.PartitionType; import org.apache.doris.catalog.TableIf; import org.apache.doris.catalog.TableIf.TableType; +import org.apache.doris.catalog.View; import org.apache.doris.common.DdlException; import org.apache.doris.common.ErrorCode; import org.apache.doris.common.FeNameFormat; @@ -56,6 +57,7 @@ import org.apache.doris.nereids.trees.plans.commands.ExplainCommand.ExplainLevel; import org.apache.doris.nereids.trees.plans.logical.LogicalPlan; import org.apache.doris.nereids.trees.plans.logical.LogicalSink; +import org.apache.doris.nereids.trees.plans.logical.LogicalSubQueryAlias; import org.apache.doris.nereids.trees.plans.visitor.NondeterministicFunctionCollector; import org.apache.doris.nereids.trees.plans.visitor.TableCollector; import org.apache.doris.nereids.trees.plans.visitor.TableCollector.TableCollectorContext; @@ -66,6 +68,8 @@ import com.google.common.collect.Maps; import com.google.common.collect.Sets; import org.apache.commons.collections.CollectionUtils; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import java.util.ArrayList; import java.util.List; @@ -80,6 +84,8 @@ * MTMV info in creating MTMV. */ public class CreateMTMVInfo { + public static final Logger LOG = LogManager.getLogger(CreateMTMVInfo.class); + private final boolean ifNotExists; private final TableNameInfo mvName; private List keys; @@ -208,7 +214,7 @@ public void analyzeQuery(ConnectContext ctx) { throw new AnalysisException("at least contain one table"); } // can not contain VIEW or MTMV - analyzeBaseTables(plan); + analyzeBaseTables(planner.getAnalyzedPlan()); // can not contain Random function analyzeExpressions(planner.getAnalyzedPlan()); // can not contain partition or tablets @@ -282,11 +288,28 @@ private PartitionDesc generatePartitionDesc(OlapTable relatedTable) { private void analyzeBaseTables(Plan plan) { TableCollectorContext collectorContext = - new TableCollector.TableCollectorContext(Sets.newHashSet(TableType.MATERIALIZED_VIEW, TableType.VIEW)); + new TableCollector.TableCollectorContext(Sets.newHashSet(TableType.MATERIALIZED_VIEW)); plan.accept(TableCollector.INSTANCE, collectorContext); List collectedTables = collectorContext.getCollectedTables(); if (!CollectionUtils.isEmpty(collectedTables)) { - throw new AnalysisException("can not contain MATERIALIZED_VIEW or VIEW"); + throw new AnalysisException("can not contain MATERIALIZED_VIEW"); + } + + List subQuerys = plan.collectToList(node -> node instanceof LogicalSubQueryAlias); + for (Object subquery : subQuerys) { + List qualifier = ((LogicalSubQueryAlias) subquery).getQualifier(); + if (!CollectionUtils.isEmpty(qualifier) && qualifier.size() == 3) { + try { + TableIf table = Env.getCurrentEnv().getCatalogMgr() + .getCatalogOrAnalysisException(qualifier.get(0)) + .getDbOrAnalysisException(qualifier.get(1)).getTableOrAnalysisException(qualifier.get(2)); + if (table instanceof View) { + throw new AnalysisException("can not contain VIEW"); + } + } catch (org.apache.doris.common.AnalysisException e) { + LOG.warn("can not get table, ", e); + } + } } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalSubQueryAlias.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalSubQueryAlias.java index c1bbfdaf93070a8..73ae70e62bc1a90 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalSubQueryAlias.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalSubQueryAlias.java @@ -183,6 +183,10 @@ public RelationId getRelationId() { return relationId; } + public List getQualifier() { + return qualifier; + } + @Override public Set getInputRelations() { Set relationIdSet = Sets.newHashSet(); diff --git a/regression-test/data/mtmv_p0/test_build_mtmv.out b/regression-test/data/mtmv_p0/test_build_mtmv.out index bc43296bd4813cc..ad0100fe2742954 100644 --- a/regression-test/data/mtmv_p0/test_build_mtmv.out +++ b/regression-test/data/mtmv_p0/test_build_mtmv.out @@ -45,9 +45,7 @@ lisi 300 zhangsang 200 -- !select -- -clz 200 -lisi 300 -zhangsang 200 +{grace_period=3333} -- !select -- clz 200 @@ -59,3 +57,6 @@ clz 200 lisi 300 zhangsang 200 +-- !select_union -- +11 111 + diff --git a/regression-test/suites/mtmv_p0/test_build_mtmv.groovy b/regression-test/suites/mtmv_p0/test_build_mtmv.groovy index e7b8e89ce5a3ed6..882a7eff22e1326 100644 --- a/regression-test/suites/mtmv_p0/test_build_mtmv.groovy +++ b/regression-test/suites/mtmv_p0/test_build_mtmv.groovy @@ -19,15 +19,18 @@ import java.time.LocalDateTime; import java.time.format.DateTimeFormatter; import java.time.Instant; import java.time.ZoneId; +import org.junit.Assert; suite("test_build_mtmv") { def tableName = "t_test_create_mtmv_user" def tableNamePv = "t_test_create_mtmv_user_pv" def mvName = "multi_mv_test_create_mtmv" + def viewName = "multi_mv_test_create_view" def mvNameRenamed = "multi_mv_test_create_mtmv_renamed" sql """drop table if exists `${tableName}`""" sql """drop table if exists `${tableNamePv}`""" + sql """drop view if exists `${viewName}`""" sql """ CREATE TABLE IF NOT EXISTS `${tableName}` ( @@ -59,6 +62,10 @@ suite("test_build_mtmv") { INSERT INTO ${tableNamePv} VALUES("2022-10-26",1,200),("2022-10-28",2,200),("2022-10-28",3,300); """ + sql """ + create view if not exists ${viewName} as select * from ${tableName}; + """ + sql """drop materialized view if exists ${mvName};""" sql """drop materialized view if exists ${mvNameRenamed};""" @@ -69,7 +76,10 @@ suite("test_build_mtmv") { BUILD DEFERRED REFRESH COMPLETE ON MANUAL COMMENT "comment1" DISTRIBUTED BY RANDOM BUCKETS 2 - PROPERTIES ('replication_num' = '1') + PROPERTIES ( + 'replication_num' = '1', + "grace_period"="333" + ) AS SELECT id, username FROM ${tableName}; """ @@ -77,23 +87,67 @@ suite("test_build_mtmv") { def showCreateTableResult = sql """show create table ${mvName}""" logger.info("showCreateTableResult: " + showCreateTableResult.toString()) assertTrue(showCreateTableResult.toString().contains("CREATE MATERIALIZED VIEW `multi_mv_test_create_mtmv` (\n `aa` BIGINT NULL COMMENT 'aaa',\n `bb` VARCHAR(20) NULL\n) ENGINE=MATERIALIZED_VIEW\nCOMMENT 'comment1'\nDISTRIBUTED BY RANDOM BUCKETS 2\nPROPERTIES")) - sql """ - DROP MATERIALIZED VIEW ${mvName} - """ - // IMMEDIATE MANUAL - sql """ - CREATE MATERIALIZED VIEW ${mvName} - BUILD IMMEDIATE REFRESH COMPLETE ON MANUAL - DISTRIBUTED BY RANDOM BUCKETS 2 - PROPERTIES ('replication_num' = '1') - AS - SELECT ${tableName}.username, ${tableNamePv}.pv FROM ${tableName}, ${tableNamePv} WHERE ${tableName}.id=${tableNamePv}.id; - """ - def jobName = getJobName("regression_test_mtmv_p0", mvName); - println jobName - waitingMTMVTaskFinished(jobName) - order_qt_select "SELECT * FROM ${mvName}" + // if not exist + try { + sql """ + CREATE MATERIALIZED VIEW IF NOT EXISTS ${mvName} + BUILD DEFERRED REFRESH COMPLETE ON MANUAL + DISTRIBUTED BY RANDOM BUCKETS 2 + PROPERTIES ('replication_num' = '1') + AS + SELECT * from ${tableName}; + """ + } catch (Exception e) { + log.info(e.getMessage()) + Assert.fail(); + } + + // not use `if not exist` + try { + sql """ + CREATE MATERIALIZED VIEW ${mvName} + BUILD DEFERRED REFRESH COMPLETE ON MANUAL + DISTRIBUTED BY RANDOM BUCKETS 2 + PROPERTIES ('replication_num' = '1') + AS + SELECT * from ${mvName}; + """ + Assert.fail(); + } catch (Exception e) { + log.info(e.getMessage()) + } + + // not allow create mv use other mv + try { + sql """ + CREATE MATERIALIZED VIEW ${mvNameRenamed} + BUILD DEFERRED REFRESH COMPLETE ON MANUAL + DISTRIBUTED BY RANDOM BUCKETS 2 + PROPERTIES ('replication_num' = '1') + AS + SELECT * from ${mvName}; + """ + Assert.fail(); + } catch (Exception e) { + log.info(e.getMessage()) + } + + // not allow create mv use view + try { + sql """ + CREATE MATERIALIZED VIEW ${mvNameRenamed} + BUILD DEFERRED REFRESH COMPLETE ON MANUAL + DISTRIBUTED BY RANDOM BUCKETS 2 + PROPERTIES ('replication_num' = '1') + AS + SELECT * from ${viewName}; + """ + Assert.fail(); + } catch (Exception e) { + log.info(e.getMessage()) + } + sql """ DROP MATERIALIZED VIEW ${mvName} """ @@ -205,6 +259,111 @@ suite("test_build_mtmv") { log.info(e.getMessage()) } + // now + try { + sql """ + CREATE MATERIALIZED VIEW ${mvName} + BUILD IMMEDIATE REFRESH COMPLETE ON SCHEDULE EVERY 10 SECOND STARTS "2023-12-13 21:07:09" + DISTRIBUTED BY RANDOM BUCKETS 2 + PROPERTIES ('replication_num' = '1') + AS + SELECT now() as dd, ${tableName}.username, ${tableNamePv}.pv FROM ${tableName}, ${tableNamePv} WHERE ${tableName}.id=${tableNamePv}.id; + """ + Assert.fail(); + } catch (Exception e) { + log.info(e.getMessage()) + } + + // uuid + try { + sql """ + CREATE MATERIALIZED VIEW ${mvName} + BUILD IMMEDIATE REFRESH COMPLETE ON SCHEDULE EVERY 10 SECOND STARTS "2023-12-13 21:07:09" + DISTRIBUTED BY RANDOM BUCKETS 2 + PROPERTIES ('replication_num' = '1') + AS + SELECT uuid() as dd, ${tableName}.username, ${tableNamePv}.pv FROM ${tableName}, ${tableNamePv} WHERE ${tableName}.id=${tableNamePv}.id; + """ + Assert.fail(); + } catch (Exception e) { + log.info(e.getMessage()) + } + + // unix_timestamp + try { + sql """ + CREATE MATERIALIZED VIEW ${mvName} + BUILD IMMEDIATE REFRESH COMPLETE ON SCHEDULE EVERY 10 SECOND STARTS "2023-12-13 21:07:09" + DISTRIBUTED BY RANDOM BUCKETS 2 + PROPERTIES ('replication_num' = '1') + AS + SELECT unix_timestamp() as dd, ${tableName}.username, ${tableNamePv}.pv FROM ${tableName}, ${tableNamePv} WHERE ${tableName}.id=${tableNamePv}.id; + """ + Assert.fail(); + } catch (Exception e) { + log.info(e.getMessage()) + } + + // utc_timestamp + try { + sql """ + CREATE MATERIALIZED VIEW ${mvName} + BUILD IMMEDIATE REFRESH COMPLETE ON SCHEDULE EVERY 10 SECOND STARTS "2023-12-13 21:07:09" + DISTRIBUTED BY RANDOM BUCKETS 2 + PROPERTIES ('replication_num' = '1') + AS + SELECT utc_timestamp() as dd, ${tableName}.username, ${tableNamePv}.pv FROM ${tableName}, ${tableNamePv} WHERE ${tableName}.id=${tableNamePv}.id; + """ + Assert.fail(); + } catch (Exception e) { + log.info(e.getMessage()) + } + + // CURDATE + try { + sql """ + CREATE MATERIALIZED VIEW ${mvName} + BUILD IMMEDIATE REFRESH COMPLETE ON SCHEDULE EVERY 10 SECOND STARTS "2023-12-13 21:07:09" + DISTRIBUTED BY RANDOM BUCKETS 2 + PROPERTIES ('replication_num' = '1') + AS + SELECT CURDATE() as dd, ${tableName}.username, ${tableNamePv}.pv FROM ${tableName}, ${tableNamePv} WHERE ${tableName}.id=${tableNamePv}.id; + """ + Assert.fail(); + } catch (Exception e) { + log.info(e.getMessage()) + } + + // uuid_numeric + try { + sql """ + CREATE MATERIALIZED VIEW ${mvName} + BUILD IMMEDIATE REFRESH COMPLETE ON SCHEDULE EVERY 10 SECOND STARTS "2023-12-13 21:07:09" + DISTRIBUTED BY RANDOM BUCKETS 2 + PROPERTIES ('replication_num' = '1') + AS + SELECT uuid_numeric() as dd, ${tableName}.username, ${tableNamePv}.pv FROM ${tableName}, ${tableNamePv} WHERE ${tableName}.id=${tableNamePv}.id; + """ + Assert.fail(); + } catch (Exception e) { + log.info(e.getMessage()) + } + + // current_time + try { + sql """ + CREATE MATERIALIZED VIEW ${mvName} + BUILD IMMEDIATE REFRESH COMPLETE ON SCHEDULE EVERY 10 SECOND STARTS "2023-12-13 21:07:09" + DISTRIBUTED BY RANDOM BUCKETS 2 + PROPERTIES ('replication_num' = '1') + AS + SELECT current_time() as dd, ${tableName}.username, ${tableNamePv}.pv FROM ${tableName}, ${tableNamePv} WHERE ${tableName}.id=${tableNamePv}.id; + """ + Assert.fail(); + } catch (Exception e) { + log.info(e.getMessage()) + } + // repeat cols try { sql """ @@ -257,6 +416,74 @@ suite("test_build_mtmv") { waitingMTMVTaskFinished(jobName) order_qt_select "SELECT * FROM ${mvName}" + // alter mv property + sql """ + alter Materialized View ${mvName} set("grace_period"="3333"); + """ + order_qt_select "select MvProperties from mv_infos('database'='regression_test_mtmv_p0') where Name = '${mvName}'" + + // use alter table + // not allow rename + try { + sql """ + alter table ${mvName} rename ${mvNameRenamed} + """ + Assert.fail(); + } catch (Exception e) { + log.info(e.getMessage()) + } + + + // not allow modify `grace_period` + try { + sql """ + alter table ${mvName} set("grace_period"="3333"); + """ + Assert.fail(); + } catch (Exception e) { + log.info(e.getMessage()) + } + + // allow modify comment + try { + sql """ + alter table ${mvName} MODIFY COMMENT "new table comment"; + """ + } catch (Exception e) { + log.info(e.getMessage()) + Assert.fail(); + } + + // not allow modify column + try { + sql """ + alter table ${mvName} DROP COLUMN pv; + """ + Assert.fail(); + } catch (Exception e) { + log.info(e.getMessage()) + } + + // not allow replace + try { + sql """ + alter table ${mvName} REPLACE WITH TABLE ${tableName}; + """ + Assert.fail(); + } catch (Exception e) { + log.info(e.getMessage()) + } + + // not allow use mv modify property of table + try { + sql """ + alter Materialized View ${mvName} set("replication_num" = "1"); + """ + Assert.fail(); + } catch (Exception e) { + log.info(e.getMessage()) + } + // alter rename sql """ alter Materialized View ${mvName} rename ${mvNameRenamed}; @@ -304,4 +531,36 @@ suite("test_build_mtmv") { println tasks assertEquals(tasks.get(0).get(0), 0); + // test bitmap + sql """drop table if exists `${tableName}`""" + sql """ + CREATE TABLE IF NOT EXISTS `${tableName}` ( + id BIGINT, + user_id bitmap + ) + DUPLICATE KEY(id) + DISTRIBUTED BY HASH(id) BUCKETS 2 + PROPERTIES ( + "replication_num" = "1" + ); + """ + sql """ + insert into ${tableName} values(11,to_bitmap(111)) + """ + + sql """ + CREATE MATERIALIZED VIEW ${mvName} + BUILD IMMEDIATE REFRESH COMPLETE ON MANUAL + DISTRIBUTED BY RANDOM BUCKETS 2 + PROPERTIES ('replication_num' = '1') + AS + select id,BITMAP_UNION(user_id) as bb from ${tableName} group by id; + """ + jobName = getJobName("regression_test_mtmv_p0", mvName); + waitingMTMVTaskFinished(jobName) + order_qt_select_union "SELECT id,bitmap_to_string(bb) FROM ${mvName}" + + sql """ + DROP MATERIALIZED VIEW ${mvName} + """ }