[fix](statistics)Fix drop stats fail silently bug. (apache#28635)

Drop stats use IN predicate to filter the column stats to delete. The default length of IN predicate is 1024, drop table stats with more than 1024 columns may fail. This pr is to split the delete sql based on the IN predicate length.
HappenLee · Jan 12, 2024 · 17626fa · 17626fa
1 parent 864db7d
commit 17626fa
Show file tree

Hide file tree

Showing 3 changed files with 1,147 additions and 4 deletions.
diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsRepository.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsRepository.java
@@ -38,6 +38,7 @@
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.HashSet;
+import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
@@ -192,10 +193,34 @@ public static void dropStatistics(long tblId, Set<String> colNames) throws DdlEx
     public static void dropStatisticsByColName(long tblId, Set<String> colNames, String statsTblName)
             throws DdlException {
         Map<String, String> params = new HashMap<>();
-        String right = colNames.stream().map(s -> "'" + s + "'").collect(Collectors.joining(","));
-        String inPredicate = String.format("tbl_id = %s AND %s IN (%s)", tblId, "col_id", right);
         params.put("tblName", statsTblName);
-        params.put("condition", inPredicate);
+        Iterator<String> iterator = colNames.iterator();
+        int columnCount = 0;
+        StringBuilder inPredicate = new StringBuilder();
+        while (iterator.hasNext()) {
+            inPredicate.append("'");
+            inPredicate.append(iterator.next());
+            inPredicate.append("'");
+            inPredicate.append(",");
+            columnCount++;
+            if (columnCount == Config.max_allowed_in_element_num_of_delete) {
+                executeDropSql(inPredicate, tblId, params);
+                columnCount = 0;
+                inPredicate.setLength(0);
+            }
+        }
+        if (inPredicate.length() > 0) {
+            executeDropSql(inPredicate, tblId, params);
+        }
+    }
+
+    public static void executeDropSql(StringBuilder inPredicate, long tblId, Map<String, String> params)
+            throws DdlException {
+        if (inPredicate.length() > 0) {
+            inPredicate.delete(inPredicate.length() - 1, inPredicate.length());
+        }
+        String predicate = String.format("tbl_id = '%s' AND %s IN (%s)", tblId, "col_id", inPredicate);
+        params.put("condition", predicate);
         try {
             StatisticsUtil.execUpdate(new StringSubstitutor(params).replace(DROP_TABLE_STATISTICS_TEMPLATE));
         } catch (Exception e) {

diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/util/StatisticsUtil.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/util/StatisticsUtil.java
@@ -148,7 +148,11 @@ public static QueryState execUpdate(String sql) throws Exception {
             StmtExecutor stmtExecutor = new StmtExecutor(r.connectContext, sql);
             r.connectContext.setExecutor(stmtExecutor);
             stmtExecutor.execute();
-            return r.connectContext.getState();
+            QueryState state = r.connectContext.getState();
+            if (state.getStateType().equals(QueryState.MysqlStateType.ERR)) {
+                throw new Exception(state.getErrorMessage());
+            }
+            return state;
         }
     }