[fix](nereids)unnest in-subquery with agg node in proper condition

apache · Oct 23, 2023 · 5c95fbd · 5c95fbd
1 parent 9006e2b
commit 5c95fbd
Show file tree

Hide file tree

Showing 5 changed files with 134 additions and 13 deletions.
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/SubExprAnalyzer.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/SubExprAnalyzer.java
@@ -89,8 +89,7 @@ public Expression visitInSubquery(InSubquery expr, CascadesContext context) {
         AnalyzedResult analyzedResult = analyzeSubquery(expr);
 
         checkOutputColumn(analyzedResult.getLogicalPlan());
-        checkHasNotAgg(analyzedResult);
-        checkHasGroupBy(analyzedResult);
+        checkNoCorrelatedSlotsUnderAgg(analyzedResult);
         checkRootIsLimit(analyzedResult);
 
         return new InSubquery(
@@ -105,7 +104,7 @@ public Expression visitScalarSubquery(ScalarSubquery scalar, CascadesContext con
 
         checkOutputColumn(analyzedResult.getLogicalPlan());
         checkHasAgg(analyzedResult);
-        checkHasGroupBy(analyzedResult);
+        checkHasNoGroupBy(analyzedResult);
 
         return new ScalarSubquery(analyzedResult.getLogicalPlan(), analyzedResult.getCorrelatedSlots());
     }
@@ -135,7 +134,7 @@ private void checkHasAgg(AnalyzedResult analyzedResult) {
         }
     }
 
-    private void checkHasGroupBy(AnalyzedResult analyzedResult) {
+    private void checkHasNoGroupBy(AnalyzedResult analyzedResult) {
         if (!analyzedResult.isCorrelated()) {
             return;
         }
@@ -145,13 +144,11 @@ private void checkHasGroupBy(AnalyzedResult analyzedResult) {
         }
     }
 
-    private void checkHasNotAgg(AnalyzedResult analyzedResult) {
-        if (!analyzedResult.isCorrelated()) {
-            return;
-        }
-        if (analyzedResult.hasAgg()) {
-            throw new AnalysisException("Unsupported correlated subquery with grouping and/or aggregation "
-                + analyzedResult.getLogicalPlan());
+    private void checkNoCorrelatedSlotsUnderAgg(AnalyzedResult analyzedResult) {
+        if (analyzedResult.hasCorrelatedSlotsUnderAgg()) {
+            throw new AnalysisException(
+                    "Unsupported correlated subquery with grouping and/or aggregation "
+                            + analyzedResult.getLogicalPlan());
         }
     }
 
@@ -223,6 +220,15 @@ public boolean hasGroupBy() {
             return false;
         }
 
+        public boolean hasCorrelatedSlotsUnderAgg() {
+            if (!correlatedSlots.isEmpty()) {
+                return logicalPlan.anyMatch(planTreeNode -> planTreeNode instanceof LogicalAggregate
+                        && ((LogicalAggregate<?>) planTreeNode)
+                                .containsSlots(ImmutableSet.copyOf(correlatedSlots)));
+            }
+            return false;
+        }
+
         public boolean rootIsLimit() {
             return logicalPlan instanceof LogicalLimit;
         }

diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/SubqueryToApply.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/SubqueryToApply.java
@@ -34,10 +34,12 @@
 import org.apache.doris.nereids.trees.expressions.Slot;
 import org.apache.doris.nereids.trees.expressions.SlotReference;
 import org.apache.doris.nereids.trees.expressions.SubqueryExpr;
+import org.apache.doris.nereids.trees.expressions.functions.agg.AggregateFunction;
 import org.apache.doris.nereids.trees.expressions.literal.BooleanLiteral;
 import org.apache.doris.nereids.trees.expressions.visitor.DefaultExpressionRewriter;
 import org.apache.doris.nereids.trees.plans.Plan;
 import org.apache.doris.nereids.trees.plans.algebra.Aggregate;
+import org.apache.doris.nereids.trees.plans.logical.LogicalAggregate;
 import org.apache.doris.nereids.trees.plans.logical.LogicalApply;
 import org.apache.doris.nereids.trees.plans.logical.LogicalFilter;
 import org.apache.doris.nereids.trees.plans.logical.LogicalJoin;
@@ -269,8 +271,13 @@ private LogicalPlan subqueryToApply(List<SubqueryExpr> subqueryExprs, LogicalPla
     private boolean nonMarkJoinExistsWithAgg(SubqueryExpr exists,
                  Map<SubqueryExpr, Optional<MarkJoinSlotReference>> subqueryToMarkJoinSlot) {
         return exists instanceof Exists
-            && exists.getQueryPlan().anyMatch(Aggregate.class::isInstance)
-            && !subqueryToMarkJoinSlot.get(exists).isPresent();
+                && exists.getQueryPlan()
+                        .anyMatch(planTreeNode -> planTreeNode instanceof LogicalAggregate
+                                && ((LogicalAggregate<?>) planTreeNode).getOutputExpressions()
+                                        .stream()
+                                        .anyMatch(namedExpression -> namedExpression
+                                                .anyMatch(AggregateFunction.class::isInstance)))
+                && !subqueryToMarkJoinSlot.get(exists).isPresent();
     }
 
     private LogicalPlan addApply(SubqueryExpr subquery, LogicalPlan childPlan,

diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/Plan.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/Plan.java
@@ -65,6 +65,12 @@ default boolean hasUnboundExpression() {
         return getExpressions().stream().anyMatch(Expression::hasUnbound);
     }
 
+    default boolean containsSlots(ImmutableSet<Slot> slots) {
+        return getExpressions().stream().anyMatch(
+                expression -> !Sets.intersection(slots, expression.getInputSlots()).isEmpty()
+                        || children().stream().anyMatch(plan -> plan.containsSlots(slots)));
+    }
+
     default LogicalProperties computeLogicalProperties() {
         throw new IllegalStateException("Not support compute logical properties for " + getClass().getName());
     }

diff --git a/regression-test/data/nereids_syntax_p0/sub_query_correlated.out b/regression-test/data/nereids_syntax_p0/sub_query_correlated.out
@@ -465,3 +465,18 @@ true
 \N
 \N
 
+-- !cir_5218_in_ok --
+4
+
+-- !cir_5218_exists_ok_1 --
+13
+
+-- !cir_5218_exists_ok_2 --
+3
+
+-- !cir_5218_exists_ok_3 --
+5
+
+-- !cir_5218_exists_ok_4 --
+13
+
diff --git a/regression-test/suites/nereids_syntax_p0/sub_query_correlated.groovy b/regression-test/suites/nereids_syntax_p0/sub_query_correlated.groovy
@@ -532,6 +532,93 @@ suite ("sub_query_correlated") {
         select sub_query_correlated_subquery8.k1 in (select sub_query_correlated_subquery9.k3 from sub_query_correlated_subquery9) from sub_query_correlated_subquery8 order by k1, k2;
     """
 
+    qt_cir_5218_in_ok """
+        SELECT count(*)
+        FROM sub_query_correlated_subquery6
+        WHERE k1 IN 
+            (SELECT k1
+            FROM 
+                (SELECT k1,
+                sum(k3) AS bbb,
+                count(k2) AS aaa
+                FROM sub_query_correlated_subquery7
+                WHERE k1 > 0
+                        AND k3 > 0
+                GROUP BY  k1 ) y
+                WHERE y.aaa>0
+                        AND k1>1); 
+    """
+
+    qt_cir_5218_exists_ok_1 """
+        SELECT count(*)
+        FROM sub_query_correlated_subquery6
+        WHERE exists 
+            (SELECT k1
+            FROM 
+                (SELECT k1,
+                sum(k3) AS bbb,
+                count(k2) AS aaa
+                FROM sub_query_correlated_subquery7
+                WHERE k1 > 0
+                        AND k3 > 0
+                GROUP BY  k1 ) y
+                WHERE y.aaa>0
+                        AND k1>1); 
+    """
+
+    qt_cir_5218_exists_ok_2 """
+        SELECT count(*)
+            FROM sub_query_correlated_subquery6
+            WHERE exists
+                (SELECT k1
+                FROM 
+                    (SELECT k1
+                    FROM sub_query_correlated_subquery7
+                    WHERE sub_query_correlated_subquery6.k1 > 7
+                    GROUP BY  k1 ) y);
+    """
+
+    qt_cir_5218_exists_ok_3 """
+        SELECT count(*)
+            FROM sub_query_correlated_subquery6
+            WHERE exists
+                (SELECT k1
+                FROM 
+                    (SELECT k1
+                    FROM sub_query_correlated_subquery7
+                    WHERE sub_query_correlated_subquery6.k1 > sub_query_correlated_subquery7.k3
+                    GROUP BY  k1 ) y);
+    """
+
+    qt_cir_5218_exists_ok_4 """
+        SELECT count(*)
+            FROM sub_query_correlated_subquery6
+            WHERE exists
+                (SELECT sum(k3)
+                FROM 
+                    sub_query_correlated_subquery7
+                    WHERE sub_query_correlated_subquery6.k1 > sub_query_correlated_subquery7.k3);
+    """
+
+    test {
+        sql """
+                SELECT count(*)
+                    FROM sub_query_correlated_subquery6
+                    WHERE k1 IN 
+                        (SELECT k1
+                        FROM 
+                            (SELECT k1,
+                            sum(k3) AS bbb,
+                            count(k2) AS aaa
+                            FROM sub_query_correlated_subquery7
+                            WHERE k1 > 0
+                                    AND k3 > 0 and sub_query_correlated_subquery6.k1 > 2
+                            GROUP BY  k1 ) y
+                            WHERE y.aaa>0
+                                    AND k1>1); """
+        exception "Unsupported correlated subquery with grouping and/or aggregation";
+    }
+
     // order_qt_doris_6937_2 """
     //     select * from sub_query_correlated_subquery1 where sub_query_correlated_subquery1.k1 not in (select sub_query_correlated_subquery3.k3 from sub_query_correlated_subquery3 where sub_query_correlated_subquery3.v2 > sub_query_correlated_subquery1.k2) or k1 < 10 order by k1, k2;
     // """