From ac60193a3606a5dc30c67d043ec57055a8fe1cc1 Mon Sep 17 00:00:00 2001 From: morningman Date: Wed, 27 Feb 2019 10:51:21 +0800 Subject: [PATCH 1/4] Remove the restrict that delete stmt must specify partition even for unpartitioned table --- .../Data Manipulation/manipulation_stmt.md | 8 +++---- fe/src/main/cup/sql_parser.cup | 5 ++-- .../org/apache/doris/analysis/DeleteStmt.java | 23 ++++--------------- .../main/java/org/apache/doris/load/Load.java | 12 ++++++++++ 4 files changed, 22 insertions(+), 26 deletions(-) diff --git a/docs/help/Contents/Data Manipulation/manipulation_stmt.md b/docs/help/Contents/Data Manipulation/manipulation_stmt.md index 59b8f6025ca138..63906e3f7b1fb6 100644 --- a/docs/help/Contents/Data Manipulation/manipulation_stmt.md +++ b/docs/help/Contents/Data Manipulation/manipulation_stmt.md @@ -293,22 +293,22 @@ 该语句用于按条件删除指定 table（base index） partition 中的数据。 该操作会同时删除和此 base index 相关的 rollup index 的数据。 语法： - DELETE FROM table_name PARTITION partition_name + DELETE FROM table_name [PARTITION partition_name] WHERE column_name1 op value[ AND column_name2 op value ...]; 说明： 1) op 的可选类型包括：=, >, <, >=, <=, != 2) 只能指定 key 列上的条件。 + 2) 当选定的 key 列不存在于某个 rollup 中时，无法进行 delete。 3) 条件之间只能是“与”的关系。 - 若希望达成“或”的关系，需要将条件分写在两个 DELETE语句中。 - 4) 如果为单分区表，partition_name 同 table_name。 + 若希望达成“或”的关系，需要将条件分写在两个 DELETE 语句中。 + 4) 如果为RANGE分区表，则必须指定 PARTITION。如果是单分区表，可以不指定。 注意： 该语句可能会降低执行后一段时间内的查询效率。 影响程度取决于语句中指定的删除条件的数量。 指定的条件越多，影响越大。 - ## example diff --git a/fe/src/main/cup/sql_parser.cup b/fe/src/main/cup/sql_parser.cup index df1e1e143f8a3d..b54ae1b2e3ad2b 100644 --- a/fe/src/main/cup/sql_parser.cup +++ b/fe/src/main/cup/sql_parser.cup @@ -2063,10 +2063,9 @@ cancel_param ::= // Delete stmt delete_stmt ::= - KW_DELETE KW_FROM table_name:table KW_PARTITION ident:partition where_clause:wherePredicate - opt_properties:properties + KW_DELETE KW_FROM table_name:table opt_partition_name:partition where_clause:wherePredicate {: - RESULT = new DeleteStmt(table, partition, wherePredicate, properties); + RESULT = new DeleteStmt(table, partition, wherePredicate); :} ; diff --git a/fe/src/main/java/org/apache/doris/analysis/DeleteStmt.java b/fe/src/main/java/org/apache/doris/analysis/DeleteStmt.java index f64858d661142b..b6360ce163be59 100644 --- a/fe/src/main/java/org/apache/doris/analysis/DeleteStmt.java +++ b/fe/src/main/java/org/apache/doris/analysis/DeleteStmt.java @@ -23,7 +23,6 @@ import org.apache.doris.common.ErrorCode; import org.apache.doris.common.ErrorReport; import org.apache.doris.common.UserException; -import org.apache.doris.common.util.PrintableMap; import org.apache.doris.mysql.privilege.PrivPredicate; import org.apache.doris.qe.ConnectContext; @@ -39,15 +38,12 @@ public class DeleteStmt extends DdlStmt { private Expr wherePredicate; private List deleteConditions; - private Map properties; public DeleteStmt(TableName tableName, String partitionName, Expr wherePredicate, Map properties) { this.tbl = tableName; - this.partitionName = partitionName; + this.partitionName = Strings.emptyToNull(partitionName); this.wherePredicate = wherePredicate; this.deleteConditions = new LinkedList(); - - this.properties = properties; } public String getTableName() { @@ -66,10 +62,6 @@ public List getDeleteConditions() { return deleteConditions; } - public Map getProperties() { - return properties; - } - @Override public void analyze(Analyzer analyzer) throws AnalysisException, UserException { super.analyze(analyzer); @@ -79,10 +71,6 @@ public void analyze(Analyzer analyzer) throws AnalysisException, UserException { } tbl.analyze(analyzer); - - if (Strings.isNullOrEmpty(partitionName)) { - throw new AnalysisException("Partition is not set"); - } if (wherePredicate == null) { throw new AnalysisException("Where clause is not set"); @@ -136,13 +124,10 @@ private void analyzePredicate(Expr predicate) throws AnalysisException { public String toSql() { StringBuilder sb = new StringBuilder(); sb.append("DELETE FROM ").append(tbl.toSql()); - sb.append(" PARTITION ").append(partitionName); - sb.append(" WHERE ").append(wherePredicate.toSql()); - if (properties != null && !properties.isEmpty()) { - sb.append("\nPROPERTIES ("); - sb.append(new PrintableMap(properties, "=", true, false)); - sb.append(")"); + if (!Strings.isNullOrEmpty(partitionName)) { + sb.append(" PARTITION ").append(partitionName); } + sb.append(" WHERE ").append(wherePredicate.toSql()); return sb.toString(); } diff --git a/fe/src/main/java/org/apache/doris/load/Load.java b/fe/src/main/java/org/apache/doris/load/Load.java index 2187e61e99b3b5..7ade948951a32d 100644 --- a/fe/src/main/java/org/apache/doris/load/Load.java +++ b/fe/src/main/java/org/apache/doris/load/Load.java @@ -39,6 +39,7 @@ import org.apache.doris.catalog.OlapTable.OlapTableState; import org.apache.doris.catalog.Partition; import org.apache.doris.catalog.Partition.PartitionState; +import org.apache.doris.catalog.PartitionType; import org.apache.doris.catalog.PrimitiveType; import org.apache.doris.catalog.Replica; import org.apache.doris.catalog.Table; @@ -3089,6 +3090,16 @@ public void delete(DeleteStmt stmt) throws DdlException { } tableId = olapTable.getId(); + if (partitionName == null) { + if (olapTable.getPartitionInfo().getType() == PartitionType.RANGE) { + throw new DdlException("This is range partitioned table." + + " You should specify partition in delete stmt"); + } else { + // this is a unpartitioned table, use table name as partition name + partitionName = olapTable.getName(); + } + } + Partition partition = olapTable.getPartition(partitionName); if (partition == null) { throw new DdlException("Partition does not exist. name: " + partitionName); @@ -3170,6 +3181,7 @@ public void delete(DeleteStmt stmt) throws DdlException { } } + @Deprecated public void deleteOld(DeleteStmt stmt) throws DdlException { String dbName = stmt.getDbName(); String tableName = stmt.getTableName(); From eb175abc3deadea2d844b170ccb2858b296111cd Mon Sep 17 00:00:00 2001 From: morningman Date: Wed, 27 Feb 2019 11:06:27 +0800 Subject: [PATCH 2/4] bug fix --- .../org/apache/doris/analysis/DeleteStmt.java | 3 +-- .../apache/doris/analysis/DeleteStmtTest.java | 16 ++++++++-------- 2 files changed, 9 insertions(+), 10 deletions(-) diff --git a/fe/src/main/java/org/apache/doris/analysis/DeleteStmt.java b/fe/src/main/java/org/apache/doris/analysis/DeleteStmt.java index b6360ce163be59..aa6f601a67bfe7 100644 --- a/fe/src/main/java/org/apache/doris/analysis/DeleteStmt.java +++ b/fe/src/main/java/org/apache/doris/analysis/DeleteStmt.java @@ -30,7 +30,6 @@ import java.util.LinkedList; import java.util.List; -import java.util.Map; public class DeleteStmt extends DdlStmt { private final TableName tbl; @@ -39,7 +38,7 @@ public class DeleteStmt extends DdlStmt { private List deleteConditions; - public DeleteStmt(TableName tableName, String partitionName, Expr wherePredicate, Map properties) { + public DeleteStmt(TableName tableName, String partitionName, Expr wherePredicate) { this.tbl = tableName; this.partitionName = Strings.emptyToNull(partitionName); this.wherePredicate = wherePredicate; diff --git a/fe/src/test/java/org/apache/doris/analysis/DeleteStmtTest.java b/fe/src/test/java/org/apache/doris/analysis/DeleteStmtTest.java index 310514223b979c..cb67f235042f3a 100644 --- a/fe/src/test/java/org/apache/doris/analysis/DeleteStmtTest.java +++ b/fe/src/test/java/org/apache/doris/analysis/DeleteStmtTest.java @@ -55,7 +55,7 @@ public void setUp() { public void getMethodTest() { BinaryPredicate wherePredicate = new BinaryPredicate(Operator.EQ, new SlotRef(null, "k1"), new StringLiteral("abc")); - DeleteStmt deleteStmt = new DeleteStmt(new TableName("testDb", "testTbl"), "partition", wherePredicate, null); + DeleteStmt deleteStmt = new DeleteStmt(new TableName("testDb", "testTbl"), "partition", wherePredicate); Assert.assertEquals("testDb", deleteStmt.getDbName()); Assert.assertEquals("testTbl", deleteStmt.getTableName()); @@ -70,7 +70,7 @@ public void testAnalyze() { LikePredicate likePredicate = new LikePredicate(org.apache.doris.analysis.LikePredicate.Operator.LIKE, new SlotRef(null, "k1"), new StringLiteral("abc")); - DeleteStmt deleteStmt = new DeleteStmt(new TableName("testDb", "testTbl"), "partition", likePredicate, null); + DeleteStmt deleteStmt = new DeleteStmt(new TableName("testDb", "testTbl"), "partition", likePredicate); try { deleteStmt.analyze(analyzer); } catch (UserException e) { @@ -84,7 +84,7 @@ public void testAnalyze() { new CompoundPredicate(org.apache.doris.analysis.CompoundPredicate.Operator.OR, binaryPredicate, binaryPredicate); - deleteStmt = new DeleteStmt(new TableName("testDb", "testTbl"), "partition", compoundPredicate, null); + deleteStmt = new DeleteStmt(new TableName("testDb", "testTbl"), "partition", compoundPredicate); try { deleteStmt.analyze(analyzer); @@ -97,7 +97,7 @@ public void testAnalyze() { binaryPredicate, likePredicate); - deleteStmt = new DeleteStmt(new TableName("testDb", "testTbl"), "partition", compoundPredicate, null); + deleteStmt = new DeleteStmt(new TableName("testDb", "testTbl"), "partition", compoundPredicate); try { deleteStmt.analyze(analyzer); } catch (UserException e) { @@ -111,7 +111,7 @@ public void testAnalyze() { binaryPredicate, binaryPredicate); - deleteStmt = new DeleteStmt(new TableName("testDb", "testTbl"), "partition", compoundPredicate, null); + deleteStmt = new DeleteStmt(new TableName("testDb", "testTbl"), "partition", compoundPredicate); try { deleteStmt.analyze(analyzer); } catch (UserException e) { @@ -125,7 +125,7 @@ public void testAnalyze() { binaryPredicate, binaryPredicate); - deleteStmt = new DeleteStmt(new TableName("testDb", "testTbl"), "partition", compoundPredicate, null); + deleteStmt = new DeleteStmt(new TableName("testDb", "testTbl"), "partition", compoundPredicate); try { deleteStmt.analyze(analyzer); } catch (UserException e) { @@ -139,7 +139,7 @@ public void testAnalyze() { binaryPredicate, binaryPredicate); - deleteStmt = new DeleteStmt(new TableName("testDb", "testTbl"), null, compoundPredicate, null); + deleteStmt = new DeleteStmt(new TableName("testDb", "testTbl"), null, compoundPredicate); try { deleteStmt.analyze(analyzer); } catch (UserException e) { @@ -157,7 +157,7 @@ public void testAnalyze() { binaryPredicate, compoundPredicate2); - deleteStmt = new DeleteStmt(new TableName("testDb", "testTbl"), "partition", compoundPredicate, null); + deleteStmt = new DeleteStmt(new TableName("testDb", "testTbl"), "partition", compoundPredicate); try { deleteStmt.analyze(analyzer); } catch (UserException e) { From 04875bb7b1867d031e85ae3eec733ef8e5f72f92 Mon Sep 17 00:00:00 2001 From: morningman Date: Wed, 27 Feb 2019 11:22:03 +0800 Subject: [PATCH 3/4] bug fix --- fe/src/main/java/org/apache/doris/load/Load.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fe/src/main/java/org/apache/doris/load/Load.java b/fe/src/main/java/org/apache/doris/load/Load.java index 7ade948951a32d..9081d6cf0b34b8 100644 --- a/fe/src/main/java/org/apache/doris/load/Load.java +++ b/fe/src/main/java/org/apache/doris/load/Load.java @@ -3092,7 +3092,7 @@ public void delete(DeleteStmt stmt) throws DdlException { tableId = olapTable.getId(); if (partitionName == null) { if (olapTable.getPartitionInfo().getType() == PartitionType.RANGE) { - throw new DdlException("This is range partitioned table." + throw new DdlException("This is a range partitioned table." + " You should specify partition in delete stmt"); } else { // this is a unpartitioned table, use table name as partition name From 5c76906d78890d0a707f4a2efea37bedd3541929 Mon Sep 17 00:00:00 2001 From: morningman Date: Wed, 27 Feb 2019 11:32:21 +0800 Subject: [PATCH 4/4] bug fix --- .../test/java/org/apache/doris/analysis/DeleteStmtTest.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/fe/src/test/java/org/apache/doris/analysis/DeleteStmtTest.java b/fe/src/test/java/org/apache/doris/analysis/DeleteStmtTest.java index cb67f235042f3a..484418805dbb8b 100644 --- a/fe/src/test/java/org/apache/doris/analysis/DeleteStmtTest.java +++ b/fe/src/test/java/org/apache/doris/analysis/DeleteStmtTest.java @@ -133,8 +133,7 @@ public void testAnalyze() { } // case 6 partition is null - binaryPredicate = new BinaryPredicate(Operator.EQ, new StringLiteral("abc"), - new SlotRef(null, "k1")); + binaryPredicate = new BinaryPredicate(Operator.EQ, new SlotRef(null, "k1"), new StringLiteral("abc")); compoundPredicate = new CompoundPredicate(org.apache.doris.analysis.CompoundPredicate.Operator.AND, binaryPredicate, binaryPredicate); @@ -143,6 +142,7 @@ public void testAnalyze() { try { deleteStmt.analyze(analyzer); } catch (UserException e) { + e.printStackTrace(); Assert.assertTrue(e.getMessage().contains("Partition is not set")); }