[fix](inverted index) in_list support inverted index apache#37921 apa…

…che#37842 (apache#38738)
mongo360 · Aug 5, 2024 · 471af2a · 471af2a
1 parent d31ab89
commit 471af2a
Show file tree

Hide file tree

Showing 4 changed files with 209 additions and 10 deletions.
diff --git a/be/src/olap/rowset/segment_v2/segment_iterator.cpp b/be/src/olap/rowset/segment_v2/segment_iterator.cpp
@@ -731,9 +731,16 @@ Status SegmentIterator::_execute_predicates_except_leafnode_of_andnode(
     } else if (_is_literal_node(node_type)) {
         auto v_literal_expr = std::dynamic_pointer_cast<doris::vectorized::VLiteral>(expr);
         _column_predicate_info->query_values.insert(v_literal_expr->value());
-    } else if (node_type == TExprNodeType::BINARY_PRED || node_type == TExprNodeType::MATCH_PRED) {
+    } else if (node_type == TExprNodeType::BINARY_PRED || node_type == TExprNodeType::MATCH_PRED ||
+               node_type == TExprNodeType::IN_PRED) {
         if (node_type == TExprNodeType::MATCH_PRED) {
             _column_predicate_info->query_op = "match";
+        } else if (node_type == TExprNodeType::IN_PRED) {
+            if (expr->op() == TExprOpcode::type::FILTER_IN) {
+                _column_predicate_info->query_op = "in";
+            } else {
+                _column_predicate_info->query_op = "not_in";
+            }
         } else {
             _column_predicate_info->query_op = expr->fn().name.function_name;
         }
@@ -866,6 +873,10 @@ Status SegmentIterator::_apply_index_except_leafnode_of_andnode() {
                           pred_type == PredicateType::LT || pred_type == PredicateType::LE ||
                           pred_type == PredicateType::GT || pred_type == PredicateType::GE ||
                           pred_type == PredicateType::MATCH;
+        if (_opts.runtime_state->query_options().enable_inverted_index_compound_inlist) {
+            is_support |= (pred_type == PredicateType::IN_LIST ||
+                           pred_type == PredicateType::NOT_IN_LIST);
+        }
         if (!is_support) {
             _need_read_data_indices[column_id] = true;
             continue;
@@ -2516,6 +2527,12 @@ void SegmentIterator::_calculate_pred_in_remaining_conjunct_root(
     } else {
         if (node_type == TExprNodeType::MATCH_PRED) {
             _column_predicate_info->query_op = "match";
+        } else if (node_type == TExprNodeType::IN_PRED) {
+            if (expr->op() == TExprOpcode::type::FILTER_IN) {
+                _column_predicate_info->query_op = "in";
+            } else {
+                _column_predicate_info->query_op = "not_in";
+            }
         } else if (node_type != TExprNodeType::COMPOUND_PRED) {
             _column_predicate_info->query_op = expr->fn().name.function_name;
         }

diff --git a/be/src/vec/exec/scan/vscan_node.cpp b/be/src/vec/exec/scan/vscan_node.cpp
@@ -1133,21 +1133,23 @@ Status VScanNode::_normalize_in_and_not_in_compound_predicate(vectorized::VExpr*
         std::string fn_name =
                 expr->op() == TExprOpcode::type::FILTER_IN ? "in_list" : "not_in_list";
 
+        for (const auto& child_expr : expr->children()) {
+            if (child_expr->node_type() == TExprNodeType::NULL_LITERAL) {
+                *pdt = PushDownType::UNACCEPTABLE;
+                return Status::OK();
+            }
+        }
+
         HybridSetBase::IteratorBase* iter = nullptr;
         auto hybrid_set = expr->get_set_func();
 
         if (hybrid_set != nullptr) {
-            if (hybrid_set->size() <= _max_pushdown_conditions_per_column) {
-                iter = hybrid_set->begin();
-            } else {
-                _filter_predicates.in_filters.emplace_back(slot->col_name(), expr->get_set_func());
-                *pdt = PushDownType::ACCEPTABLE;
-                return Status::OK();
-            }
+            *pdt = PushDownType::UNACCEPTABLE;
+            return Status::OK();
         } else {
-            VInPredicate* pred = static_cast<VInPredicate*>(expr);
+            auto* pred = static_cast<vectorized::VInPredicate*>(expr);
 
-            InState* state = reinterpret_cast<InState*>(
+            auto* state = reinterpret_cast<vectorized::InState*>(
                     expr_ctx->fn_context(pred->fn_context_index())
                             ->get_function_state(FunctionContext::FRAGMENT_LOCAL));
 

diff --git a/regression-test/data/inverted_index_p0/test_index_rqg_bug4.out b/regression-test/data/inverted_index_p0/test_index_rqg_bug4.out
@@ -0,0 +1,18 @@
+-- This file is automatically generated. You should know what you did if you want to edit this
+-- !sql --
+\N
+a
+b
+f
+h
+i
+j
+k
+l
+o
+p
+q
+v
+y
+z
+