From 5e7f2c35328041bced33094268bb4ffc1b29078f Mon Sep 17 00:00:00 2001 From: xufei Date: Tue, 3 Dec 2019 16:31:56 +0800 Subject: [PATCH 1/7] TiDB requires row__id when use -1 as column id in table scan executor --- dbms/src/Debug/dbgFuncCoprocessor.cpp | 57 +++++++++---------- dbms/src/Flash/Coprocessor/DAGContext.h | 2 - dbms/src/Flash/Coprocessor/DAGDriver.cpp | 7 +-- dbms/src/Flash/Coprocessor/DAGQuerySource.cpp | 12 +--- dbms/src/Flash/Coprocessor/DAGQuerySource.h | 2 +- dbms/src/Flash/Coprocessor/InterpreterDAG.cpp | 43 ++++++-------- tests/mutable-test/txn_dag/table_scan.test | 5 ++ 7 files changed, 54 insertions(+), 74 deletions(-) diff --git a/dbms/src/Debug/dbgFuncCoprocessor.cpp b/dbms/src/Debug/dbgFuncCoprocessor.cpp index 2909350fe74..95b71548c7e 100644 --- a/dbms/src/Debug/dbgFuncCoprocessor.cpp +++ b/dbms/src/Debug/dbgFuncCoprocessor.cpp @@ -388,6 +388,21 @@ std::tuple compileQuery(Context & context, hijackTiDBTypeForMockTest(ci); ts_output.emplace_back(std::make_pair(column_info.name, std::move(ci))); } + for (const auto & expr : ast_query.select_expression_list->children) + { + if (ASTIdentifier * identifier = typeid_cast(expr.get())) + { + if (identifier->getColumnName() == VOID_COL_NAME) + { + ColumnInfo ci; + ci.tp = TiDB::TypeLongLong; + ci.setPriKeyFlag(); + ci.setNotNullFlag(); + hijackTiDBTypeForMockTest(ci); + ts_output.emplace_back(std::make_pair(VOID_COL_NAME, std::move(ci))); + } + } + } executor_ctx_map.emplace( ts_exec, ExecutorCtx{nullptr, std::move(ts_output), std::unordered_map>{}}); last_executor = ts_exec; @@ -450,7 +465,10 @@ std::tuple compileQuery(Context & context, for (const auto & info : executor_ctx.output) { tipb::ColumnInfo * ci = ts->add_columns(); - ci->set_column_id(table_info.getColumnID(info.first)); + if (info.first == VOID_COL_NAME) + ci->set_column_id(-1); + else + ci->set_column_id(table_info.getColumnID(info.first)); ci->set_tp(info.second.tp); ci->set_flag(info.second.flag); ci->set_columnlen(info.second.flen); @@ -590,37 +608,14 @@ std::tuple compileQuery(Context & context, const auto & last_output = executor_ctx_map[last_executor].output; - // For testing VOID column, ignore any other select expressions, unless table contains it. - if (std::find(final_output.begin(), final_output.end(), VOID_COL_NAME) != final_output.end() - && std::find_if( - last_output.begin(), last_output.end(), [&](const auto & last_field) { return last_field.first == VOID_COL_NAME; }) - == last_output.end()) - { - dag_request.add_output_offsets(0); - - // Set column ID to -1 to trigger `void` column in DAG processing. - tipb::ColumnInfo * ci = ts->add_columns(); - ci->set_column_id(-1); - - // Set column name to VOID and tp to Nullable(UInt64), - // as chunk decoding doesn't do strict field type check so Nullable(UInt64) should be enough. - ColumnInfo ti_ci; - ti_ci.name = VOID_COL_NAME; - ti_ci.tp = TiDB::TypeLongLong; - ti_ci.setNotNullFlag(); - schema.emplace_back(DAGColumnInfo{VOID_COL_NAME, std::move(ti_ci)}); - } - else + for (const auto & field : final_output) { - for (const auto & field : final_output) - { - auto iter = std::find_if( - last_output.begin(), last_output.end(), [&](const auto & last_field) { return last_field.first == field; }); - if (iter == last_output.end()) - throw Exception("Column not found after pruning: " + field, ErrorCodes::LOGICAL_ERROR); - dag_request.add_output_offsets(iter - last_output.begin()); - schema.push_back(*iter); - } + auto iter + = std::find_if(last_output.begin(), last_output.end(), [&](const auto & last_field) { return last_field.first == field; }); + if (iter == last_output.end()) + throw Exception("Column not found after pruning: " + field, ErrorCodes::LOGICAL_ERROR); + dag_request.add_output_offsets(iter - last_output.begin()); + schema.push_back(*iter); } } diff --git a/dbms/src/Flash/Coprocessor/DAGContext.h b/dbms/src/Flash/Coprocessor/DAGContext.h index 453e6df219e..385fc48cd68 100644 --- a/dbms/src/Flash/Coprocessor/DAGContext.h +++ b/dbms/src/Flash/Coprocessor/DAGContext.h @@ -13,8 +13,6 @@ struct DAGContext { DAGContext(size_t profile_list_size) { profile_streams_list.resize(profile_list_size); }; std::vector profile_streams_list; - - tipb::FieldType void_result_ft; }; } // namespace DB diff --git a/dbms/src/Flash/Coprocessor/DAGDriver.cpp b/dbms/src/Flash/Coprocessor/DAGDriver.cpp index 604887fd1eb..984f99cec57 100644 --- a/dbms/src/Flash/Coprocessor/DAGDriver.cpp +++ b/dbms/src/Flash/Coprocessor/DAGDriver.cpp @@ -66,11 +66,8 @@ try // Only query is allowed, so streams.in must not be null and streams.out must be null throw Exception("DAG is not query.", ErrorCodes::LOGICAL_ERROR); - BlockOutputStreamPtr dag_output_stream = std::make_shared(dag_response, - context.getSettings().dag_records_per_chunk, - dag.getEncodeType(), - dag.getResultFieldTypes(dag.getDAGContext().void_result_ft), - streams.in->getHeader()); + BlockOutputStreamPtr dag_output_stream = std::make_shared( + dag_response, context.getSettings().dag_records_per_chunk, dag.getEncodeType(), dag.getResultFieldTypes(), streams.in->getHeader()); copyData(*streams.in, *dag_output_stream); if (!dag_request.has_collect_execution_summaries() || !dag_request.collect_execution_summaries()) diff --git a/dbms/src/Flash/Coprocessor/DAGQuerySource.cpp b/dbms/src/Flash/Coprocessor/DAGQuerySource.cpp index d6f2b257d43..b064c49f288 100644 --- a/dbms/src/Flash/Coprocessor/DAGQuerySource.cpp +++ b/dbms/src/Flash/Coprocessor/DAGQuerySource.cpp @@ -62,11 +62,11 @@ DAGQuerySource::DAGQuerySource(Context & context_, DAGContext & dag_context_, Re break; default: throw Exception( - "Unsupported executor in DAG request: " + dag_request.executors(i).DebugString(), ErrorCodes::NOT_IMPLEMENTED); + "Unsupported executor in DAG request: " + dag_request.executors(i).DebugString(), ErrorCodes::NOT_IMPLEMENTED); } } encode_type = dag_request.encode_type(); - if (encode_type == tipb::EncodeType::TypeArrow && hasUnsupportedTypeForArrowEncode(getResultFieldTypes({}))) + if (encode_type == tipb::EncodeType::TypeArrow && hasUnsupportedTypeForArrowEncode(getResultFieldTypes())) { encode_type = tipb::EncodeType::TypeDefault; } @@ -100,8 +100,6 @@ bool fillExecutorOutputFieldTypes(const tipb::Executor & executor, std::vector DAGQuerySource::getResultFieldTypes(const tipb::FieldType & void_result_ft) const +std::vector DAGQuerySource::getResultFieldTypes() const { std::vector executor_output; for (int i = dag_request.executors_size() - 1; i >= 0; i--) { if (fillExecutorOutputFieldTypes(dag_request.executors(i), executor_output)) - { - if (executor_output.empty()) - executor_output.push_back(void_result_ft); break; - } } if (executor_output.empty()) { diff --git a/dbms/src/Flash/Coprocessor/DAGQuerySource.h b/dbms/src/Flash/Coprocessor/DAGQuerySource.h index 0909c54adef..1dc8a0fb893 100644 --- a/dbms/src/Flash/Coprocessor/DAGQuerySource.h +++ b/dbms/src/Flash/Coprocessor/DAGQuerySource.h @@ -80,7 +80,7 @@ class DAGQuerySource : public IQuerySource }; const tipb::DAGRequest & getDAGRequest() const { return dag_request; }; - std::vector getResultFieldTypes(const tipb::FieldType & void_result_ft) const; + std::vector getResultFieldTypes() const; ASTPtr getAST() const { return ast; }; diff --git a/dbms/src/Flash/Coprocessor/InterpreterDAG.cpp b/dbms/src/Flash/Coprocessor/InterpreterDAG.cpp index e08fbab84db..216c4fe87d9 100644 --- a/dbms/src/Flash/Coprocessor/InterpreterDAG.cpp +++ b/dbms/src/Flash/Coprocessor/InterpreterDAG.cpp @@ -171,9 +171,24 @@ void InterpreterDAG::executeTS(const tipb::TableScan & ts, Pipeline & pipeline) ColumnID cid = ci.column_id(); if (cid == -1) - // Column ID -1 means TiDB expects no specific column, mostly it is for cases like `select count(*)`. - // This means we can return whatever column, we'll choose it later if no other columns are specified either. + { + // Column ID -1 return the handle column + if (auto pk_handle_col = storage->getTableInfo().getPKHandleColumn()) + { + required_columns.push_back(pk_handle_col->get().name); + auto pair = storage->getColumns().getPhysical(pk_handle_col->get().name); + source_columns.push_back(pair); + is_ts_column.push_back(false); + } + else + { + required_columns.push_back(MutableSupport::tidb_pk_column_name); + auto pair = storage->getColumns().getPhysical(MutableSupport::tidb_pk_column_name); + source_columns.push_back(pair); + is_ts_column.push_back(false); + } continue; + } String name = storage->getTableInfo().getColumnName(cid); required_columns.push_back(name); @@ -181,30 +196,6 @@ void InterpreterDAG::executeTS(const tipb::TableScan & ts, Pipeline & pipeline) source_columns.emplace_back(std::move(pair)); is_ts_column.push_back(ci.tp() == TiDB::TypeTimestamp); } - if (required_columns.empty()) - { - // No column specified, we choose the handle column as it will be emitted by storage read anyhow. - // Set `void` column field type correspondingly for further needs, i.e. encoding results. - if (auto pk_handle_col = storage->getTableInfo().getPKHandleColumn()) - { - required_columns.push_back(pk_handle_col->get().name); - auto pair = storage->getColumns().getPhysical(pk_handle_col->get().name); - source_columns.push_back(pair); - is_ts_column.push_back(false); - // For PK handle, use original column info of itself. - dag.getDAGContext().void_result_ft = columnInfoToFieldType(pk_handle_col->get()); - } - else - { - required_columns.push_back(MutableSupport::tidb_pk_column_name); - auto pair = storage->getColumns().getPhysical(MutableSupport::tidb_pk_column_name); - source_columns.push_back(pair); - is_ts_column.push_back(false); - // For implicit handle, reverse get a column info. - auto column_info = reverseGetColumnInfo(pair, -1, Field()); - dag.getDAGContext().void_result_ft = columnInfoToFieldType(column_info); - } - } analyzer = std::make_unique(std::move(source_columns), context); diff --git a/tests/mutable-test/txn_dag/table_scan.test b/tests/mutable-test/txn_dag/table_scan.test index ab824845d71..3fa30419965 100644 --- a/tests/mutable-test/txn_dag/table_scan.test +++ b/tests/mutable-test/txn_dag/table_scan.test @@ -36,6 +36,11 @@ │ 50 │ └───────┘ +=> DBGInvoke dag('select _void, col_1 from default.test') " --dag_planner="optree +┌─_void─┬─col_1─┐ +│ 50 │ test1 │ +└───────┴───────┘ + # Clean up. => DBGInvoke __drop_tidb_table(default, test) => drop table if exists default.test From 2bd6180ef66370f0c73139e20c29e365573ba061 Mon Sep 17 00:00:00 2001 From: xufei Date: Tue, 3 Dec 2019 16:41:28 +0800 Subject: [PATCH 2/7] check region read status in executeTS --- dbms/src/Flash/Coprocessor/InterpreterDAG.cpp | 18 +++++++++++++++--- dbms/src/Flash/Coprocessor/InterpreterDAG.h | 3 +++ 2 files changed, 18 insertions(+), 3 deletions(-) diff --git a/dbms/src/Flash/Coprocessor/InterpreterDAG.cpp b/dbms/src/Flash/Coprocessor/InterpreterDAG.cpp index 216c4fe87d9..556ce8e7417 100644 --- a/dbms/src/Flash/Coprocessor/InterpreterDAG.cpp +++ b/dbms/src/Flash/Coprocessor/InterpreterDAG.cpp @@ -139,6 +139,18 @@ bool checkKeyRanges(const std::vector> else return isAllValueCoveredByRanges(handle_ranges, region_handle_ranges); } + +RegionException::RegionReadStatus InterpreterDAG::getRegionReadStatus(RegionPtr current_region) +{ + if (!current_region) + return RegionException::NOT_FOUND; + if (current_region->version() != dag.getRegionVersion() || current_region->confVer() != dag.getRegionConfVersion()) + return RegionException::VERSION_ERROR; + if (current_region->isPendingRemove()) + return RegionException::PENDING_REMOVE; + return RegionException::OK; +} + // the flow is the same as executeFetchcolumns void InterpreterDAG::executeTS(const tipb::TableScan & ts, Pipeline & pipeline) { @@ -253,12 +265,12 @@ void InterpreterDAG::executeTS(const tipb::TableScan & ts, Pipeline & pipeline) info.version = dag.getRegionVersion(); info.conf_version = dag.getRegionConfVersion(); auto current_region = context.getTMTContext().getKVStore()->getRegion(info.region_id); - if (!current_region || current_region->version() != dag.getRegionVersion() || current_region->confVer() != dag.getRegionConfVersion()) + auto region_read_status = getRegionReadStatus(current_region); + if (region_read_status != RegionException::OK) { std::vector region_ids; region_ids.push_back(info.region_id); - throw RegionException(std::move(region_ids), - current_region ? RegionException::RegionReadStatus::NOT_FOUND : RegionException::RegionReadStatus::VERSION_ERROR); + throw RegionException(std::move(region_ids), region_read_status); } if (!checkKeyRanges(dag.getKeyRanges(), table_id, storage->pkIsUInt64(), current_region->getRange())) throw Exception("Cop request only support full range scan for given region", ErrorCodes::COP_BAD_DAG_REQUEST); diff --git a/dbms/src/Flash/Coprocessor/InterpreterDAG.h b/dbms/src/Flash/Coprocessor/InterpreterDAG.h index 2491d243847..ae52395dcd4 100644 --- a/dbms/src/Flash/Coprocessor/InterpreterDAG.h +++ b/dbms/src/Flash/Coprocessor/InterpreterDAG.h @@ -13,7 +13,9 @@ #include #include #include +#include #include +#include #include namespace DB @@ -84,6 +86,7 @@ class InterpreterDAG : public IInterpreter AnalysisResult analyzeExpressions(); void recordProfileStreams(Pipeline & pipeline, Int32 index); bool addTimeZoneCastAfterTS(std::vector & is_ts_column, Pipeline & pipeline); + RegionException::RegionReadStatus getRegionReadStatus(RegionPtr current_region); private: Context & context; From 194abdf0f37770e79b524f30257ffeb1bf76ac52 Mon Sep 17 00:00:00 2001 From: xufei Date: Wed, 4 Dec 2019 10:04:27 +0800 Subject: [PATCH 3/7] address comments --- dbms/src/Debug/dbgFuncCoprocessor.cpp | 8 ++++---- tests/mutable-test/txn_dag/table_scan.test | 20 ++++++++++---------- 2 files changed, 14 insertions(+), 14 deletions(-) diff --git a/dbms/src/Debug/dbgFuncCoprocessor.cpp b/dbms/src/Debug/dbgFuncCoprocessor.cpp index 95b71548c7e..f8c3dd93d08 100644 --- a/dbms/src/Debug/dbgFuncCoprocessor.cpp +++ b/dbms/src/Debug/dbgFuncCoprocessor.cpp @@ -145,7 +145,7 @@ BlockInputStreamPtr dbgFuncMockDAG(Context & context, const ASTs & args) return outputDAGResponse(context, schema, dag_response); } -const String VOID_COL_NAME = "_void"; +const String ROWID_COL_NAME = "_tidb_rowid"; struct ExecutorCtx { @@ -392,14 +392,14 @@ std::tuple compileQuery(Context & context, { if (ASTIdentifier * identifier = typeid_cast(expr.get())) { - if (identifier->getColumnName() == VOID_COL_NAME) + if (identifier->getColumnName() == ROWID_COL_NAME) { ColumnInfo ci; ci.tp = TiDB::TypeLongLong; ci.setPriKeyFlag(); ci.setNotNullFlag(); hijackTiDBTypeForMockTest(ci); - ts_output.emplace_back(std::make_pair(VOID_COL_NAME, std::move(ci))); + ts_output.emplace_back(std::make_pair(ROWID_COL_NAME, std::move(ci))); } } } @@ -465,7 +465,7 @@ std::tuple compileQuery(Context & context, for (const auto & info : executor_ctx.output) { tipb::ColumnInfo * ci = ts->add_columns(); - if (info.first == VOID_COL_NAME) + if (info.first == ROWID_COL_NAME) ci->set_column_id(-1); else ci->set_column_id(table_info.getColumnID(info.first)); diff --git a/tests/mutable-test/txn_dag/table_scan.test b/tests/mutable-test/txn_dag/table_scan.test index 3fa30419965..4a78a04b33b 100644 --- a/tests/mutable-test/txn_dag/table_scan.test +++ b/tests/mutable-test/txn_dag/table_scan.test @@ -30,16 +30,16 @@ │ test1 │ └───────┘ -# TiDB may push down table scan with -1 column, use keyword _void testing this case. -=> DBGInvoke dag('select _void from default.test') " --dag_planner="optree -┌─_void─┐ -│ 50 │ -└───────┘ - -=> DBGInvoke dag('select _void, col_1 from default.test') " --dag_planner="optree -┌─_void─┬─col_1─┐ -│ 50 │ test1 │ -└───────┴───────┘ +# select TiDB rowid +=> DBGInvoke dag('select _tidb_rowid from default.test') " --dag_planner="optree +┌─_tidb_rowid─┐ +│ 50 │ +└─────────────┘ + +=> DBGInvoke dag('select _tidb_rowid, col_1 from default.test') " --dag_planner="optree +┌─_tidb_rowid─┬─col_1─┐ +│ 50 │ test1 │ +└─────────────┴───────┘ # Clean up. => DBGInvoke __drop_tidb_table(default, test) From 65d315cbf8af84dc2b459f7b4facae3e7cd962ff Mon Sep 17 00:00:00 2001 From: xufei Date: Wed, 4 Dec 2019 15:34:31 +0800 Subject: [PATCH 4/7] address comments --- dbms/src/Debug/dbgFuncCoprocessor.cpp | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/dbms/src/Debug/dbgFuncCoprocessor.cpp b/dbms/src/Debug/dbgFuncCoprocessor.cpp index f8c3dd93d08..dacd339041a 100644 --- a/dbms/src/Debug/dbgFuncCoprocessor.cpp +++ b/dbms/src/Debug/dbgFuncCoprocessor.cpp @@ -16,6 +16,7 @@ #include #include #include +#include #include #include #include @@ -145,8 +146,6 @@ BlockInputStreamPtr dbgFuncMockDAG(Context & context, const ASTs & args) return outputDAGResponse(context, schema, dag_response); } -const String ROWID_COL_NAME = "_tidb_rowid"; - struct ExecutorCtx { tipb::Executor * input; @@ -392,14 +391,14 @@ std::tuple compileQuery(Context & context, { if (ASTIdentifier * identifier = typeid_cast(expr.get())) { - if (identifier->getColumnName() == ROWID_COL_NAME) + if (identifier->getColumnName() == MutableSupport::tidb_pk_column_name) { ColumnInfo ci; ci.tp = TiDB::TypeLongLong; ci.setPriKeyFlag(); ci.setNotNullFlag(); hijackTiDBTypeForMockTest(ci); - ts_output.emplace_back(std::make_pair(ROWID_COL_NAME, std::move(ci))); + ts_output.emplace_back(std::make_pair(MutableSupport::tidb_pk_column_name, std::move(ci))); } } } @@ -465,7 +464,7 @@ std::tuple compileQuery(Context & context, for (const auto & info : executor_ctx.output) { tipb::ColumnInfo * ci = ts->add_columns(); - if (info.first == ROWID_COL_NAME) + if (info.first == MutableSupport::tidb_pk_column_name) ci->set_column_id(-1); else ci->set_column_id(table_info.getColumnID(info.first)); From 4f73a18bbf9ce3eea24d6e75c85d314c6117230c Mon Sep 17 00:00:00 2001 From: xufei Date: Wed, 11 Dec 2019 13:40:37 +0800 Subject: [PATCH 5/7] refine log --- dbms/src/Flash/Coprocessor/DAGDriver.cpp | 19 +----- .../Flash/Coprocessor/DAGStringConverter.cpp | 62 ++++++++++++++++--- .../Flash/Coprocessor/DAGStringConverter.h | 14 +---- dbms/src/Flash/Coprocessor/DAGUtils.cpp | 36 +++++++---- dbms/src/Flash/Coprocessor/InterpreterDAG.cpp | 22 ++++++- dbms/src/Flash/Coprocessor/InterpreterDAG.h | 2 +- dbms/src/Flash/CoprocessorHandler.cpp | 13 ++-- dbms/src/Flash/FlashService.cpp | 2 - dbms/src/Interpreters/Settings.h | 1 - 9 files changed, 105 insertions(+), 66 deletions(-) diff --git a/dbms/src/Flash/Coprocessor/DAGDriver.cpp b/dbms/src/Flash/Coprocessor/DAGDriver.cpp index 984f99cec57..3ba2063e548 100644 --- a/dbms/src/Flash/Coprocessor/DAGDriver.cpp +++ b/dbms/src/Flash/Coprocessor/DAGDriver.cpp @@ -43,25 +43,8 @@ try DAGContext dag_context(dag_request.executors_size()); DAGQuerySource dag(context, dag_context, region_id, region_version, region_conf_version, key_ranges, dag_request); - BlockIO streams; - - String planner = context.getSettings().dag_planner; - if (planner == "sql") - { - DAGStringConverter converter(context, dag_request); - String query = converter.buildSqlString(); - if (!query.empty()) - streams = executeQuery(query, context, internal, QueryProcessingStage::Complete); - } - else if (planner == "optree") - { - streams = executeQuery(dag, context, internal, QueryProcessingStage::Complete); - } - else - { - throw Exception("Unknown DAG planner type " + planner, ErrorCodes::LOGICAL_ERROR); - } + BlockIO streams = executeQuery(dag, context, internal, QueryProcessingStage::Complete); if (!streams.in || streams.out) // Only query is allowed, so streams.in must not be null and streams.out must be null throw Exception("DAG is not query.", ErrorCodes::LOGICAL_ERROR); diff --git a/dbms/src/Flash/Coprocessor/DAGStringConverter.cpp b/dbms/src/Flash/Coprocessor/DAGStringConverter.cpp index ef5efcc08f7..47a546a2c28 100644 --- a/dbms/src/Flash/Coprocessor/DAGStringConverter.cpp +++ b/dbms/src/Flash/Coprocessor/DAGStringConverter.cpp @@ -2,10 +2,12 @@ #include #include +#include #include #include #include #include +#include #include namespace DB @@ -55,12 +57,19 @@ void DAGStringConverter::buildTSString(const tipb::TableScan & ts, std::stringst for (const tipb::ColumnInfo & ci : ts.columns()) { ColumnID cid = ci.column_id(); - if (cid <= 0 || cid > (ColumnID)columns_from_ts.size()) + if (cid == -1) { - throw Exception("column id out of bound", ErrorCodes::COP_BAD_DAG_REQUEST); + // Column ID -1 returns the handle column + auto pk_handle_col = storage->getTableInfo().getPKHandleColumn(); + pk_handle_col.has_value(); + auto pair = storage->getColumns().getPhysical( + pk_handle_col.has_value() ? pk_handle_col->get().name : MutableSupport::tidb_pk_column_name); + columns_from_ts.push_back(pair); + continue; } - String name = merge_tree->getTableInfo().columns[cid - 1].name; - output_from_ts.push_back(std::move(name)); + auto name = storage->getTableInfo().getColumnName(cid); + auto pair = storage->getColumns().getPhysical(name); + columns_from_ts.push_back(pair); } ss << "FROM " << storage->getDatabaseName() << "." << storage->getTableName() << " "; } @@ -86,6 +95,43 @@ void DAGStringConverter::buildSelString(const tipb::Selection & sel, std::string void DAGStringConverter::buildLimitString(const tipb::Limit & limit, std::stringstream & ss) { ss << "LIMIT " << limit.limit() << " "; } +void DAGStringConverter::buildAggString(const tipb::Aggregation & agg, std::stringstream & ss) +{ + if (agg.group_by_size() != 0) + { + ss << "GROUP BY "; + bool first = true; + for (auto & group_by : agg.group_by()) + { + if (first) + first = false; + else + ss << ", "; + ss << exprToString(group_by, getCurrentColumns()); + } + } + for (auto & agg_func : agg.agg_func()) + { + columns_from_agg.emplace_back(exprToString(agg_func, getCurrentColumns()), getDataTypeByFieldType(agg_func.field_type())); + } + afterAgg = true; +} +void DAGStringConverter::buildTopNString(const tipb::TopN & topN, std::stringstream & ss) +{ + ss << "ORDER BY "; + bool first = true; + for (auto & order_by_item : topN.order_by()) + { + if (first) + first = false; + else + ss << ", "; + ss << exprToString(order_by_item.expr(), getCurrentColumns()) << " "; + ss << (order_by_item.desc() ? "DESC" : "ASC"); + } + ss << " LIMIT " << topN.limit() << " "; +} + //todo return the error message void DAGStringConverter::buildString(const tipb::Executor & executor, std::stringstream & ss) { @@ -101,11 +147,9 @@ void DAGStringConverter::buildString(const tipb::Executor & executor, std::strin case tipb::ExecType::TypeAggregation: // stream agg is not supported, treated as normal agg case tipb::ExecType::TypeStreamAgg: - //todo support agg - throw Exception("Aggregation is not supported", ErrorCodes::NOT_IMPLEMENTED); + return buildAggString(executor.aggregation(), ss); case tipb::ExecType::TypeTopN: - // todo support top n - throw Exception("TopN is not supported", ErrorCodes::NOT_IMPLEMENTED); + return buildTopNString(executor.topn(), ss); case tipb::ExecType::TypeLimit: return buildLimitString(executor.limit(), ss); } @@ -145,7 +189,7 @@ String DAGStringConverter::buildSqlString() { project << ", "; } - project << getCurrentOutputColumns()[index]; + project << getCurrentColumns()[index].name; } project << " "; } diff --git a/dbms/src/Flash/Coprocessor/DAGStringConverter.h b/dbms/src/Flash/Coprocessor/DAGStringConverter.h index f90396be281..9b1492dc89d 100644 --- a/dbms/src/Flash/Coprocessor/DAGStringConverter.h +++ b/dbms/src/Flash/Coprocessor/DAGStringConverter.h @@ -31,19 +31,12 @@ class DAGStringConverter return columns_from_ts; } - const Names & getCurrentOutputColumns() - { - if (afterAgg) - { - return output_from_agg; - } - return output_from_ts; - } - protected: void buildTSString(const tipb::TableScan & ts, std::stringstream & ss); void buildSelString(const tipb::Selection & sel, std::stringstream & ss); void buildLimitString(const tipb::Limit & limit, std::stringstream & ss); + void buildAggString(const tipb::Aggregation & agg, std::stringstream & ss); + void buildTopNString(const tipb::TopN & topN, std::stringstream & ss); void buildString(const tipb::Executor & executor, std::stringstream & ss); protected: @@ -52,9 +45,6 @@ class DAGStringConverter // used by columnRef, which starts with 1, and refs column index in the original ts/agg output std::vector columns_from_ts; std::vector columns_from_agg; - // used by output_offset, which starts with 0, and refs the index in the selected output of ts/agg operater - Names output_from_ts; - Names output_from_agg; bool afterAgg; }; diff --git a/dbms/src/Flash/Coprocessor/DAGUtils.cpp b/dbms/src/Flash/Coprocessor/DAGUtils.cpp index 5211691a258..ac85a9a59bc 100644 --- a/dbms/src/Flash/Coprocessor/DAGUtils.cpp +++ b/dbms/src/Flash/Coprocessor/DAGUtils.cpp @@ -122,24 +122,34 @@ String exprToString(const tipb::Expr & expr, const std::vector if (isInOrGlobalInOperator(func_name)) { // for in, we could not represent the function expr using func_name(param1, param2, ...) - throw Exception("Function " + func_name + " not supported", ErrorCodes::UNSUPPORTED_METHOD); - } - ss << func_name << "("; - bool first = true; - for (const tipb::Expr & child : expr.children()) - { - String s = exprToString(child, input_col); - if (first) + ss << exprToString(expr.children(0), input_col) << " " << func_name << " ("; + bool first = true; + for (int i = 1; i < expr.children_size(); i++) { - first = false; + String s = exprToString(expr.children(i), input_col); + if (first) + first = false; + else + ss << ", "; + ss << s; } - else + ss << ")"; + } + else + { + ss << func_name << "("; + bool first = true; + for (const tipb::Expr & child : expr.children()) { - ss << ", "; + String s = exprToString(child, input_col); + if (first) + first = false; + else + ss << ", "; + ss << s; } - ss << s; + ss << ")"; } - ss << ") "; return ss.str(); } diff --git a/dbms/src/Flash/Coprocessor/InterpreterDAG.cpp b/dbms/src/Flash/Coprocessor/InterpreterDAG.cpp index 556ce8e7417..34865b05db1 100644 --- a/dbms/src/Flash/Coprocessor/InterpreterDAG.cpp +++ b/dbms/src/Flash/Coprocessor/InterpreterDAG.cpp @@ -14,6 +14,7 @@ #include #include #include +#include #include #include #include @@ -140,7 +141,7 @@ bool checkKeyRanges(const std::vector> return isAllValueCoveredByRanges(handle_ranges, region_handle_ranges); } -RegionException::RegionReadStatus InterpreterDAG::getRegionReadStatus(RegionPtr current_region) +RegionException::RegionReadStatus InterpreterDAG::getRegionReadStatus(const RegionPtr & current_region) { if (!current_region) return RegionException::NOT_FOUND; @@ -270,6 +271,7 @@ void InterpreterDAG::executeTS(const tipb::TableScan & ts, Pipeline & pipeline) { std::vector region_ids; region_ids.push_back(info.region_id); + LOG_WARNING(log, __PRETTY_FUNCTION__ << " Meet region exception for region " << info.region_id); throw RegionException(std::move(region_ids), region_read_status); } if (!checkKeyRanges(dag.getKeyRanges(), table_id, storage->pkIsUInt64(), current_region->getRange())) @@ -374,7 +376,7 @@ InterpreterDAG::AnalysisResult InterpreterDAG::analyzeExpressions() // add cast if type is not match analyzer->appendAggSelect(chain, dag.getAggregation(), dag.getDAGRequest(), keep_session_timezone_info); //todo use output_offset to reconstruct the final project columns - for (auto element : analyzer->getCurrentInputColumns()) + for (auto & element : analyzer->getCurrentInputColumns()) { final_project.emplace_back(element.name, ""); } @@ -654,7 +656,7 @@ void InterpreterDAG::executeFinalProject(Pipeline & pipeline) { auto columns = pipeline.firstStream()->getHeader(); NamesAndTypesList input_column; - for (auto column : columns.getColumnsWithTypeAndName()) + for (auto & column : columns.getColumnsWithTypeAndName()) { input_column.emplace_back(column.name, column.type); } @@ -686,6 +688,20 @@ BlockIO InterpreterDAG::execute() LOG_DEBUG( log, __PRETTY_FUNCTION__ << " Convert DAG request to BlockIO, adding " << analyzer->getImplicitCastCount() << " implicit cast"); + if (log->debug()) + { + try + { + DAGStringConverter converter(context, dag.getDAGRequest()); + auto sql_text = converter.buildSqlString(); + LOG_DEBUG(log, __PRETTY_FUNCTION__ << " SQL in DAG request is " << sql_text); + } + catch (...) + { + // catch all the exceptions so the convert error will not affect the query execution + LOG_DEBUG(log, __PRETTY_FUNCTION__ << " Failed to convert DAG request to sql text"); + } + } return res; } } // namespace DB diff --git a/dbms/src/Flash/Coprocessor/InterpreterDAG.h b/dbms/src/Flash/Coprocessor/InterpreterDAG.h index ae52395dcd4..8463940a85b 100644 --- a/dbms/src/Flash/Coprocessor/InterpreterDAG.h +++ b/dbms/src/Flash/Coprocessor/InterpreterDAG.h @@ -86,7 +86,7 @@ class InterpreterDAG : public IInterpreter AnalysisResult analyzeExpressions(); void recordProfileStreams(Pipeline & pipeline, Int32 index); bool addTimeZoneCastAfterTS(std::vector & is_ts_column, Pipeline & pipeline); - RegionException::RegionReadStatus getRegionReadStatus(RegionPtr current_region); + RegionException::RegionReadStatus getRegionReadStatus(const RegionPtr & current_region); private: Context & context; diff --git a/dbms/src/Flash/CoprocessorHandler.cpp b/dbms/src/Flash/CoprocessorHandler.cpp index 315f12df519..1205bac46e8 100644 --- a/dbms/src/Flash/CoprocessorHandler.cpp +++ b/dbms/src/Flash/CoprocessorHandler.cpp @@ -22,7 +22,8 @@ CoprocessorHandler::CoprocessorHandler( : cop_context(cop_context_), cop_request(cop_request_), cop_response(cop_response_), log(&Logger::get("CoprocessorHandler")) {} -grpc::Status CoprocessorHandler::execute() try +grpc::Status CoprocessorHandler::execute() +try { switch (cop_request->tp()) { @@ -59,9 +60,8 @@ grpc::Status CoprocessorHandler::execute() try } catch (const LockException & e) { - LOG_ERROR(log, - __PRETTY_FUNCTION__ << ": LockException: region " << cop_request->context().region_id() << "\n" - << e.getStackTrace().toString()); + LOG_WARNING( + log, __PRETTY_FUNCTION__ << ": LockException: region " << cop_request->context().region_id() << ", message: " << e.message()); cop_response->Clear(); kvrpcpb::LockInfo * lock_info = cop_response->mutable_locked(); lock_info->set_key(e.lock_infos[0]->key); @@ -73,9 +73,8 @@ catch (const LockException & e) } catch (const RegionException & e) { - LOG_ERROR(log, - __PRETTY_FUNCTION__ << ": RegionException: region " << cop_request->context().region_id() << "\n" - << e.getStackTrace().toString()); + LOG_WARNING( + log, __PRETTY_FUNCTION__ << ": RegionException: region " << cop_request->context().region_id() << ", message: " << e.message()); cop_response->Clear(); errorpb::Error * region_err; switch (e.status) diff --git a/dbms/src/Flash/FlashService.cpp b/dbms/src/Flash/FlashService.cpp index 5a404b0d5f2..a997a7e47ef 100644 --- a/dbms/src/Flash/FlashService.cpp +++ b/dbms/src/Flash/FlashService.cpp @@ -112,8 +112,6 @@ std::tuple FlashService::createDBContext(const grpc::Serv { context.setSetting("dag_records_per_chunk", dag_records_per_chunk_str); } - std::string planner = getClientMetaVarWithDefault(grpc_context, "dag_planner", "optree"); - context.setSetting("dag_planner", planner); std::string expr_field_type_check = getClientMetaVarWithDefault(grpc_context, "dag_expr_field_type_strict_check", "1"); context.setSetting("dag_expr_field_type_strict_check", expr_field_type_check); diff --git a/dbms/src/Interpreters/Settings.h b/dbms/src/Interpreters/Settings.h index 83c55d51089..efeed1f7148 100644 --- a/dbms/src/Interpreters/Settings.h +++ b/dbms/src/Interpreters/Settings.h @@ -30,7 +30,6 @@ struct Settings M(SettingBool, resolve_locks, false, "tmt resolve locks.") \ M(SettingUInt64, read_tso, DEFAULT_MAX_READ_TSO, "tmt read tso.") \ M(SettingInt64, dag_records_per_chunk, DEFAULT_DAG_RECORDS_PER_CHUNK, "default chunk size of a DAG response.") \ - M(SettingString, dag_planner, "optree", "planner for DAG query, sql builds the SQL string, optree builds the internal operator(stream) tree.") \ M(SettingBool, dag_expr_field_type_strict_check, true, "when set to true, every expr in the dag request must provide field type, otherwise only the result expr will be checked.") \ M(SettingInt64, schema_version, DEFAULT_UNSPECIFIED_SCHEMA_VERSION, "tmt schema version.") \ M(SettingUInt64, batch_commands_threads, 0, "Number of threads to use for handling batch commands concurrently. 0 means - same as 'max_threads'.") \ From 9e2ce8942dacc03f13a46a00c0934217bdcad555 Mon Sep 17 00:00:00 2001 From: xufei Date: Wed, 11 Dec 2019 13:57:02 +0800 Subject: [PATCH 6/7] remove useless comment --- dbms/src/Flash/Coprocessor/DAGStringConverter.h | 1 - 1 file changed, 1 deletion(-) diff --git a/dbms/src/Flash/Coprocessor/DAGStringConverter.h b/dbms/src/Flash/Coprocessor/DAGStringConverter.h index 9b1492dc89d..7e8ae66aa0d 100644 --- a/dbms/src/Flash/Coprocessor/DAGStringConverter.h +++ b/dbms/src/Flash/Coprocessor/DAGStringConverter.h @@ -42,7 +42,6 @@ class DAGStringConverter protected: Context & context; const tipb::DAGRequest & dag_request; - // used by columnRef, which starts with 1, and refs column index in the original ts/agg output std::vector columns_from_ts; std::vector columns_from_agg; bool afterAgg; From 45f413e28fa5a4374d981d74c7703ef159fef3af Mon Sep 17 00:00:00 2001 From: xufei Date: Wed, 11 Dec 2019 15:01:41 +0800 Subject: [PATCH 7/7] update tests --- dbms/src/Debug/dbgFuncCoprocessor.cpp | 1 - tests/mutable-test/txn_dag/arrow_encode.test | 4 +-- .../txn_dag/data_type_number.test | 26 +++++++++---------- .../txn_dag/data_type_others.test | 10 +++---- .../mutable-test/txn_dag/data_type_time.test | 6 ++--- .../txn_dag/data_type_time_bit.test | 4 +-- tests/mutable-test/txn_dag/project.test | 8 +++--- tests/mutable-test/txn_dag/table_scan.test | 10 +++---- tests/mutable-test/txn_dag/time_zone.test | 22 ++++++++-------- 9 files changed, 45 insertions(+), 46 deletions(-) diff --git a/dbms/src/Debug/dbgFuncCoprocessor.cpp b/dbms/src/Debug/dbgFuncCoprocessor.cpp index dacd339041a..f9b90e63f4c 100644 --- a/dbms/src/Debug/dbgFuncCoprocessor.cpp +++ b/dbms/src/Debug/dbgFuncCoprocessor.cpp @@ -626,7 +626,6 @@ tipb::SelectResponse executeDAGRequest(Context & context, const tipb::DAGRequest { static Logger * log = &Logger::get("MockDAG"); LOG_DEBUG(log, __PRETTY_FUNCTION__ << ": Handling DAG request: " << dag_request.DebugString()); - context.setSetting("dag_planner", "optree"); tipb::SelectResponse dag_response; DAGDriver driver(context, dag_request, region_id, region_version, region_conf_version, std::move(key_ranges), dag_response, true); driver.execute(); diff --git a/tests/mutable-test/txn_dag/arrow_encode.test b/tests/mutable-test/txn_dag/arrow_encode.test index b806993e856..55edb912e1b 100644 --- a/tests/mutable-test/txn_dag/arrow_encode.test +++ b/tests/mutable-test/txn_dag/arrow_encode.test @@ -18,7 +18,7 @@ => DBGInvoke __raft_insert_row(default, test, 4, 55, -128, 255, -32768, 65535, -2147483648, 4294967295, -9223372036854775808, 18446744073709551615, null, 1234567.890123, '2010-01-01', '2010-01-01 11:11:11', 'arrow encode test',11.11,100,null,9572888,1) => DBGInvoke __raft_insert_row(default, test, 4, 56, -128, 255, -32768, null, -2147483648, 4294967295, -9223372036854775808, 18446744073709551615, 12345.6789, 1234567.890123, '2010-01-01', '2010-01-01 11:11:11', 'arrow encode test',22.22,100,0,9572888,2) -=> DBGInvoke dag('select * from default.test',4,'arrow') " --dag_planner="optree +=> DBGInvoke dag('select * from default.test',4,'arrow') ┌─col_1─┬─col_2─┬──col_3─┬─col_4─┬───────col_5─┬──────col_6─┬────────────────col_7─┬────────────────col_8─┬─────col_9─┬─────────col_10─┬─────col_11─┬──────────────col_12─┬─col_13────────────┬─col_14───┬─col_15─┬─col_16─┬──col_17─┬─col_18─┐ │ -128 │ 255 │ -32768 │ 65535 │ -2147483648 │ 4294967295 │ -9223372036854775808 │ 18446744073709551615 │ 12345.679 │ 1234567.890123 │ 2010-01-01 │ 2010-01-01 11:11:11 │ arrow encode test │ 12.12 │ 100 │ 1 │ 9572888 │ a │ │ -128 │ 255 │ -32768 │ 65535 │ -2147483648 │ 4294967295 │ -9223372036854775808 │ 18446744073709551615 │ 12345.679 │ 1234567.890123 │ 2010-01-01 │ 2010-01-01 11:11:11 │ arrow encode │ 123.23 │ 100 │ 0 │ \N │ b │ @@ -29,7 +29,7 @@ │ -128 │ 255 │ -32768 │ \N │ -2147483648 │ 4294967295 │ -9223372036854775808 │ 18446744073709551615 │ 12345.679 │ 1234567.890123 │ 2010-01-01 │ 2010-01-01 11:11:11 │ arrow encode test │ 22.22 │ 100 │ 0 │ 9572888 │ b │ └───────┴───────┴────────┴───────┴─────────────┴────────────┴──────────────────────┴──────────────────────┴───────────┴────────────────┴────────────┴─────────────────────┴───────────────────┴──────────┴────────┴────────┴─────────┴────────┘ -=> DBGInvoke mock_dag('select * from default.test',4,0,'arrow') " --dag_planner="optree +=> DBGInvoke mock_dag('select * from default.test',4,0,'arrow') ┌─col_1─┬─col_2─┬──col_3─┬─col_4─┬───────col_5─┬──────col_6─┬────────────────col_7─┬────────────────col_8─┬─────col_9─┬─────────col_10─┬─────col_11─┬──────────────col_12─┬─col_13────────────┬─col_14───┬─col_15─┬─col_16─┬──col_17─┬─col_18─┐ │ -128 │ 255 │ -32768 │ 65535 │ -2147483648 │ 4294967295 │ -9223372036854775808 │ 18446744073709551615 │ 12345.679 │ 1234567.890123 │ 2010-01-01 │ 2010-01-01 11:11:11 │ arrow encode test │ 12.12 │ 100 │ 1 │ 9572888 │ a │ │ -128 │ 255 │ -32768 │ 65535 │ -2147483648 │ 4294967295 │ -9223372036854775808 │ 18446744073709551615 │ 12345.679 │ 1234567.890123 │ 2010-01-01 │ 2010-01-01 11:11:11 │ arrow encode │ 123.23 │ 100 │ 0 │ \N │ b │ diff --git a/tests/mutable-test/txn_dag/data_type_number.test b/tests/mutable-test/txn_dag/data_type_number.test index 4231d8d955f..69c44863e5c 100644 --- a/tests/mutable-test/txn_dag/data_type_number.test +++ b/tests/mutable-test/txn_dag/data_type_number.test @@ -15,77 +15,77 @@ => DBGInvoke __raft_insert_row(default, test, 4, 50, -128, 255, -32768, 65535, -2147483648, 4294967295, -9223372036854775808, 18446744073709551615, 12345.6789, 1234567.890123) # DAG read full table scan -=> DBGInvoke dag('select * from default.test') " --dag_planner="optree +=> DBGInvoke dag('select * from default.test') ┌─col_1─┬─col_2─┬──col_3─┬─col_4─┬───────col_5─┬──────col_6─┬────────────────col_7─┬────────────────col_8─┬─────col_9─┬─────────col_10─┐ │ -128 │ 255 │ -32768 │ 65535 │ -2147483648 │ 4294967295 │ -9223372036854775808 │ 18446744073709551615 │ 12345.679 │ 1234567.890123 │ └───────┴───────┴────────┴───────┴─────────────┴────────────┴──────────────────────┴──────────────────────┴───────────┴────────────────┘ -=> DBGInvoke dag('select * from default.test',4,'arrow') " --dag_planner="optree +=> DBGInvoke dag('select * from default.test',4,'arrow') ┌─col_1─┬─col_2─┬──col_3─┬─col_4─┬───────col_5─┬──────col_6─┬────────────────col_7─┬────────────────col_8─┬─────col_9─┬─────────col_10─┐ │ -128 │ 255 │ -32768 │ 65535 │ -2147483648 │ 4294967295 │ -9223372036854775808 │ 18446744073709551615 │ 12345.679 │ 1234567.890123 │ └───────┴───────┴────────┴───────┴─────────────┴────────────┴──────────────────────┴──────────────────────┴───────────┴────────────────┘ # DAG read filter by Int8 column -=> DBGInvoke dag('select * from default.test where col_1 = -128') " --dag_planner="optree +=> DBGInvoke dag('select * from default.test where col_1 = -128') ┌─col_1─┬─col_2─┬──col_3─┬─col_4─┬───────col_5─┬──────col_6─┬────────────────col_7─┬────────────────col_8─┬─────col_9─┬─────────col_10─┐ │ -128 │ 255 │ -32768 │ 65535 │ -2147483648 │ 4294967295 │ -9223372036854775808 │ 18446744073709551615 │ 12345.679 │ 1234567.890123 │ └───────┴───────┴────────┴───────┴─────────────┴────────────┴──────────────────────┴──────────────────────┴───────────┴────────────────┘ # DAG read filter by UInt8 column -=> DBGInvoke dag('select * from default.test where col_2 = 255') " --dag_planner="optree +=> DBGInvoke dag('select * from default.test where col_2 = 255') ┌─col_1─┬─col_2─┬──col_3─┬─col_4─┬───────col_5─┬──────col_6─┬────────────────col_7─┬────────────────col_8─┬─────col_9─┬─────────col_10─┐ │ -128 │ 255 │ -32768 │ 65535 │ -2147483648 │ 4294967295 │ -9223372036854775808 │ 18446744073709551615 │ 12345.679 │ 1234567.890123 │ └───────┴───────┴────────┴───────┴─────────────┴────────────┴──────────────────────┴──────────────────────┴───────────┴────────────────┘ # DAG read filter by Int16 column -=> DBGInvoke dag('select * from default.test where col_3 = -32768') " --dag_planner="optree +=> DBGInvoke dag('select * from default.test where col_3 = -32768') ┌─col_1─┬─col_2─┬──col_3─┬─col_4─┬───────col_5─┬──────col_6─┬────────────────col_7─┬────────────────col_8─┬─────col_9─┬─────────col_10─┐ │ -128 │ 255 │ -32768 │ 65535 │ -2147483648 │ 4294967295 │ -9223372036854775808 │ 18446744073709551615 │ 12345.679 │ 1234567.890123 │ └───────┴───────┴────────┴───────┴─────────────┴────────────┴──────────────────────┴──────────────────────┴───────────┴────────────────┘ # DAG read filter by UInt16 column -=> DBGInvoke dag('select * from default.test where col_4 = 65535') " --dag_planner="optree +=> DBGInvoke dag('select * from default.test where col_4 = 65535') ┌─col_1─┬─col_2─┬──col_3─┬─col_4─┬───────col_5─┬──────col_6─┬────────────────col_7─┬────────────────col_8─┬─────col_9─┬─────────col_10─┐ │ -128 │ 255 │ -32768 │ 65535 │ -2147483648 │ 4294967295 │ -9223372036854775808 │ 18446744073709551615 │ 12345.679 │ 1234567.890123 │ └───────┴───────┴────────┴───────┴─────────────┴────────────┴──────────────────────┴──────────────────────┴───────────┴────────────────┘ # DAG read filter by Int32 column -=> DBGInvoke dag('select * from default.test where col_5 = -2147483648') " --dag_planner="optree +=> DBGInvoke dag('select * from default.test where col_5 = -2147483648') ┌─col_1─┬─col_2─┬──col_3─┬─col_4─┬───────col_5─┬──────col_6─┬────────────────col_7─┬────────────────col_8─┬─────col_9─┬─────────col_10─┐ │ -128 │ 255 │ -32768 │ 65535 │ -2147483648 │ 4294967295 │ -9223372036854775808 │ 18446744073709551615 │ 12345.679 │ 1234567.890123 │ └───────┴───────┴────────┴───────┴─────────────┴────────────┴──────────────────────┴──────────────────────┴───────────┴────────────────┘ # DAG read filter by UInt32 column -=> DBGInvoke dag('select * from default.test where col_6 = 4294967295') " --dag_planner="optree +=> DBGInvoke dag('select * from default.test where col_6 = 4294967295') ┌─col_1─┬─col_2─┬──col_3─┬─col_4─┬───────col_5─┬──────col_6─┬────────────────col_7─┬────────────────col_8─┬─────col_9─┬─────────col_10─┐ │ -128 │ 255 │ -32768 │ 65535 │ -2147483648 │ 4294967295 │ -9223372036854775808 │ 18446744073709551615 │ 12345.679 │ 1234567.890123 │ └───────┴───────┴────────┴───────┴─────────────┴────────────┴──────────────────────┴──────────────────────┴───────────┴────────────────┘ # DAG read filter by Int64 column -=> DBGInvoke dag('select * from default.test where col_7 = -9223372036854775808') " --dag_planner="optree +=> DBGInvoke dag('select * from default.test where col_7 = -9223372036854775808') ┌─col_1─┬─col_2─┬──col_3─┬─col_4─┬───────col_5─┬──────col_6─┬────────────────col_7─┬────────────────col_8─┬─────col_9─┬─────────col_10─┐ │ -128 │ 255 │ -32768 │ 65535 │ -2147483648 │ 4294967295 │ -9223372036854775808 │ 18446744073709551615 │ 12345.679 │ 1234567.890123 │ └───────┴───────┴────────┴───────┴─────────────┴────────────┴──────────────────────┴──────────────────────┴───────────┴────────────────┘ # DAG read filter by UInt64 column -=> DBGInvoke dag('select * from default.test where col_8 = 18446744073709551615') " --dag_planner="optree +=> DBGInvoke dag('select * from default.test where col_8 = 18446744073709551615') ┌─col_1─┬─col_2─┬──col_3─┬─col_4─┬───────col_5─┬──────col_6─┬────────────────col_7─┬────────────────col_8─┬─────col_9─┬─────────col_10─┐ │ -128 │ 255 │ -32768 │ 65535 │ -2147483648 │ 4294967295 │ -9223372036854775808 │ 18446744073709551615 │ 12345.679 │ 1234567.890123 │ └───────┴───────┴────────┴───────┴─────────────┴────────────┴──────────────────────┴──────────────────────┴───────────┴────────────────┘ # DAG read filter by Float32 column -#=> DBGInvoke dag('select * from default.test where col_9 = 12345.679') " --dag_planner="optree +#=> DBGInvoke dag('select * from default.test where col_9 = 12345.679') #┌─col_1─┬─col_2─┬──col_3─┬─col_4─┬───────col_5─┬──────col_6─┬────────────────col_7─┬────────────────col_8─┬─────col_9─┬─────────col_10─┐ #│ -128 │ 255 │ -32768 │ 65535 │ -2147483648 │ 4294967295 │ -9223372036854775808 │ 18446744073709551615 │ 12345.679 │ 1234567.890123 │ #└───────┴───────┴────────┴───────┴─────────────┴────────────┴──────────────────────┴──────────────────────┴───────────┴────────────────┘ # DAG read filter by Float64 column -#=> DBGInvoke dag('select * from default.test where col_10 = 1234567.890123') " --dag_planner="optree +#=> DBGInvoke dag('select * from default.test where col_10 = 1234567.890123') #┌─col_1─┬─col_2─┬──col_3─┬─col_4─┬───────col_5─┬──────col_6─┬────────────────col_7─┬────────────────col_8─┬─────col_9─┬─────────col_10─┐ #│ -128 │ 255 │ -32768 │ 65535 │ -2147483648 │ 4294967295 │ -9223372036854775808 │ 18446744073709551615 │ 12345.679 │ 1234567.890123 │ #└───────┴───────┴────────┴───────┴─────────────┴────────────┴──────────────────────┴──────────────────────┴───────────┴────────────────┘ # DAG read filter by Decimal column -#=> DBGInvoke dag('select * from default.test where col_11 = 666.88') " --dag_planner="optree +#=> DBGInvoke dag('select * from default.test where col_11 = 666.88') #┌─col_1─┬─col_2─┬──col_3─┬─col_4─┬───────col_5─┬──────col_6─┬────────────────col_7─┬────────────────col_8─┬─────col_9─┬─────────col_10─┬─col_11─┐ #│ -128 │ 255 │ -32768 │ 65535 │ -2147483648 │ 4294967295 │ -9223372036854775808 │ 18446744073709551615 │ 12345.679 │ 1234567.890123 │ 666.88 │ #└───────┴───────┴────────┴───────┴─────────────┴────────────┴──────────────────────┴──────────────────────┴───────────┴────────────────┴────────┘ diff --git a/tests/mutable-test/txn_dag/data_type_others.test b/tests/mutable-test/txn_dag/data_type_others.test index 36975114399..699e47a6e0a 100644 --- a/tests/mutable-test/txn_dag/data_type_others.test +++ b/tests/mutable-test/txn_dag/data_type_others.test @@ -13,30 +13,30 @@ => DBGInvoke __raft_insert_row(default, test, 4, 50, 'data type test', 2) # DAG read full table scan -=> DBGInvoke dag('select * from default.test') " --dag_planner="optree +=> DBGInvoke dag('select * from default.test') ┌─col_1──────────┬─col_2──┐ │ data type test │ female │ └────────────────┴────────┘ -=> DBGInvoke dag('select * from default.test', 4, 'arrow') " --dag_planner="optree +=> DBGInvoke dag('select * from default.test', 4, 'arrow') ┌─col_1──────────┬─col_2──┐ │ data type test │ female │ └────────────────┴────────┘ # DAG read filter string column -=> DBGInvoke dag('select * from default.test where col_1 = \'data type test\'') " --dag_planner="optree +=> DBGInvoke dag('select * from default.test where col_1 = \'data type test\'') ┌─col_1──────────┬─col_2──┐ │ data type test │ female │ └────────────────┴────────┘ # DAG read filter enum column -=> DBGInvoke dag('select * from default.test where col_2 = \'female\'') " --dag_planner="optree +=> DBGInvoke dag('select * from default.test where col_2 = \'female\'') ┌─col_1──────────┬─col_2──┐ │ data type test │ female │ └────────────────┴────────┘ # DAG read filter enum column -=> DBGInvoke dag('select * from default.test where col_2 = 2') " --dag_planner="optree +=> DBGInvoke dag('select * from default.test where col_2 = 2') ┌─col_1──────────┬─col_2──┐ │ data type test │ female │ └────────────────┴────────┘ diff --git a/tests/mutable-test/txn_dag/data_type_time.test b/tests/mutable-test/txn_dag/data_type_time.test index 4ec7efb05cb..53ad4e6fe45 100644 --- a/tests/mutable-test/txn_dag/data_type_time.test +++ b/tests/mutable-test/txn_dag/data_type_time.test @@ -12,17 +12,17 @@ => DBGInvoke __put_region(4, 0, 100, default, test) => DBGInvoke __raft_insert_row(default, test, 4, 50, '2019-06-10', '2019-06-10 09:00:00') -=> DBGInvoke dag('select * from default.test') " --dag_planner="optree +=> DBGInvoke dag('select * from default.test') ┌──────col_1─┬───────────────col_2─┐ │ 2019-06-10 │ 2019-06-10 09:00:00 │ └────────────┴─────────────────────┘ -=> DBGInvoke dag('select * from default.test',4,'arrow') " --dag_planner="optree +=> DBGInvoke dag('select * from default.test',4,'arrow') ┌──────col_1─┬───────────────col_2─┐ │ 2019-06-10 │ 2019-06-10 09:00:00 │ └────────────┴─────────────────────┘ # Mock DAG doesn't support date/datetime comparison with string, may need type inference and do implicit conversion to literal. -# => DBGInvoke dag('select * from default.test where col_1 = \'2019-06-06\' and col_2 = \'2019-06-10 09:00:00\'') " --dag_planner="optree +# => DBGInvoke dag('select * from default.test where col_1 = \'2019-06-06\' and col_2 = \'2019-06-10 09:00:00\'') # ┌──────col_1─┬───────────────col_2─┐ # │ 2019-06-10 │ 2019-06-10 09:00:00 │ # └────────────┴─────────────────────┘ diff --git a/tests/mutable-test/txn_dag/data_type_time_bit.test b/tests/mutable-test/txn_dag/data_type_time_bit.test index 522da374629..bde44d9c11f 100644 --- a/tests/mutable-test/txn_dag/data_type_time_bit.test +++ b/tests/mutable-test/txn_dag/data_type_time_bit.test @@ -13,12 +13,12 @@ => DBGInvoke __put_region(4, 0, 100, default, test) => DBGInvoke __raft_insert_row(default, test, 4, 50, 100, 98, 9572888) -=> DBGInvoke dag('select * from default.test',4,'arrow') " --dag_planner="optree +=> DBGInvoke dag('select * from default.test',4,'arrow') ┌─col_1─┬─col_2─┬───col_3─┐ │ 100 │ 98 │ 9572888 │ └───────┴───────┴─────────┘ -=> DBGInvoke dag('select * from default.test',4) " --dag_planner="optree +=> DBGInvoke dag('select * from default.test',4) ┌─col_1─┬─col_2─┬───col_3─┐ │ 100 │ 98 │ 9572888 │ └───────┴───────┴─────────┘ diff --git a/tests/mutable-test/txn_dag/project.test b/tests/mutable-test/txn_dag/project.test index 8b29b4a7a08..ed5266a2f71 100644 --- a/tests/mutable-test/txn_dag/project.test +++ b/tests/mutable-test/txn_dag/project.test @@ -13,25 +13,25 @@ => DBGInvoke __raft_insert_row(default, test, 4, 50, 'test1', 666) # DAG read by not specifying region id, select *. -=> DBGInvoke dag('select * from default.test') " --dag_planner="optree +=> DBGInvoke dag('select * from default.test') ┌─col_1─┬─col_2─┐ │ test1 │ 666 │ └───────┴───────┘ # DAG read by not specifying region id, select col_1. -=> DBGInvoke dag('select col_1 from default.test') " --dag_planner="optree +=> DBGInvoke dag('select col_1 from default.test') ┌─col_1─┐ │ test1 │ └───────┘ # DAG read by explicitly specifying region id, select col_2. -=> DBGInvoke dag('select col_2 from default.test', 4) " --dag_planner="optree +=> DBGInvoke dag('select col_2 from default.test', 4) ┌─col_2─┐ │ 666 │ └───────┘ # Mock DAG read, select col_2, col_1, col_2. -=> DBGInvoke mock_dag('select col_2, col_1, col_2 from default.test', 4) " --dag_planner="optree +=> DBGInvoke mock_dag('select col_2, col_1, col_2 from default.test', 4) ┌─col_2─┬─col_1─┬─col_2─┐ │ 666 │ test1 │ 666 │ └───────┴───────┴───────┘ diff --git a/tests/mutable-test/txn_dag/table_scan.test b/tests/mutable-test/txn_dag/table_scan.test index 4a78a04b33b..d5730f00ced 100644 --- a/tests/mutable-test/txn_dag/table_scan.test +++ b/tests/mutable-test/txn_dag/table_scan.test @@ -13,30 +13,30 @@ => DBGInvoke __raft_insert_row(default, test, 4, 50, 'test1') # DAG read by not specifying region id. -=> DBGInvoke dag('select * from default.test') " --dag_planner="optree +=> DBGInvoke dag('select * from default.test') ┌─col_1─┐ │ test1 │ └───────┘ # DAG read by explicitly specifying region id. -=> DBGInvoke dag('select * from default.test', 4) " --dag_planner="optree +=> DBGInvoke dag('select * from default.test', 4) ┌─col_1─┐ │ test1 │ └───────┘ # Mock DAG read. -=> DBGInvoke mock_dag('select * from default.test', 4) " --dag_planner="optree +=> DBGInvoke mock_dag('select * from default.test', 4) ┌─col_1─┐ │ test1 │ └───────┘ # select TiDB rowid -=> DBGInvoke dag('select _tidb_rowid from default.test') " --dag_planner="optree +=> DBGInvoke dag('select _tidb_rowid from default.test') ┌─_tidb_rowid─┐ │ 50 │ └─────────────┘ -=> DBGInvoke dag('select _tidb_rowid, col_1 from default.test') " --dag_planner="optree +=> DBGInvoke dag('select _tidb_rowid, col_1 from default.test') ┌─_tidb_rowid─┬─col_1─┐ │ 50 │ test1 │ └─────────────┴───────┘ diff --git a/tests/mutable-test/txn_dag/time_zone.test b/tests/mutable-test/txn_dag/time_zone.test index fbe8a58aa9c..3facf948a29 100644 --- a/tests/mutable-test/txn_dag/time_zone.test +++ b/tests/mutable-test/txn_dag/time_zone.test @@ -16,7 +16,7 @@ => DBGInvoke __raft_insert_row(default, test, 4, 53, '2019-06-12', '2019-06-11 08:00:00', '2019-06-11 09:00:00') => DBGInvoke __raft_insert_row(default, test, 4, 54, '0000-00-00', '0000-00-00 00:00:00', '0000-00-00 00:00:00') -=> DBGInvoke dag('select * from default.test') " --dag_planner="optree +=> DBGInvoke dag('select * from default.test') ┌──────col_1─┬─────────────────────col_2─┬───────────────col_3─┐ │ 2019-06-10 │ 2019-06-10 09:00:00.00000 │ 2019-06-10 09:00:00 │ │ 2019-06-11 │ 2019-06-11 07:00:00.00000 │ 2019-06-11 09:00:00 │ @@ -26,7 +26,7 @@ └────────────┴───────────────────────────┴─────────────────────┘ # use tz_offset, result is the same since cop will convert the timestamp value to utc timestamp when returing to tidb -=> DBGInvoke dag('select * from default.test',4,'default',28800) " --dag_planner="optree +=> DBGInvoke dag('select * from default.test',4,'default',28800) ┌──────col_1─┬─────────────────────col_2─┬───────────────col_3─┐ │ 2019-06-10 │ 2019-06-10 09:00:00.00000 │ 2019-06-10 09:00:00 │ │ 2019-06-11 │ 2019-06-11 07:00:00.00000 │ 2019-06-11 09:00:00 │ @@ -36,7 +36,7 @@ └────────────┴───────────────────────────┴─────────────────────┘ # test arrow encode -=> DBGInvoke dag('select * from default.test',4,'arrow',28800) " --dag_planner="optree +=> DBGInvoke dag('select * from default.test',4,'arrow',28800) ┌──────col_1─┬─────────────────────col_2─┬───────────────col_3─┐ │ 2019-06-10 │ 2019-06-10 17:00:00.00000 │ 2019-06-10 09:00:00 │ │ 2019-06-11 │ 2019-06-11 15:00:00.00000 │ 2019-06-11 09:00:00 │ @@ -45,9 +45,9 @@ │ 0000-00-00 │ 0000-00-00 00:00:00.00000 │ 0000-00-00 00:00:00 │ └────────────┴───────────────────────────┴─────────────────────┘ -=> DBGInvoke dag('select * from default.test where col_2 > col_3') " --dag_planner="optree +=> DBGInvoke dag('select * from default.test where col_2 > col_3') -=> DBGInvoke dag('select * from default.test where col_2 > col_3',4,'default',28800) " --dag_planner="optree +=> DBGInvoke dag('select * from default.test where col_2 > col_3',4,'default',28800) ┌──────col_1─┬─────────────────────col_2─┬───────────────col_3─┐ │ 2019-06-10 │ 2019-06-10 09:00:00.00000 │ 2019-06-10 09:00:00 │ │ 2019-06-11 │ 2019-06-11 07:00:00.00000 │ 2019-06-11 09:00:00 │ @@ -55,30 +55,30 @@ │ 2019-06-12 │ 2019-06-11 08:00:00.00000 │ 2019-06-11 09:00:00 │ └────────────┴───────────────────────────┴─────────────────────┘ -=> DBGInvoke dag('select * from default.test where col_2 = col_3',4,'default',3600) " --dag_planner="optree +=> DBGInvoke dag('select * from default.test where col_2 = col_3',4,'default',3600) ┌──────col_1─┬─────────────────────col_2─┬───────────────col_3─┐ │ 2019-06-11 │ 2019-06-11 08:00:00.00000 │ 2019-06-11 09:00:00 │ │ 2019-06-12 │ 2019-06-11 08:00:00.00000 │ 2019-06-11 09:00:00 │ │ 0000-00-00 │ 0000-00-00 00:00:00.00000 │ 0000-00-00 00:00:00 │ └────────────┴───────────────────────────┴─────────────────────┘ -=> DBGInvoke dag('select * from default.test where col_2 = col_3',4,'default',7200) " --dag_planner="optree +=> DBGInvoke dag('select * from default.test where col_2 = col_3',4,'default',7200) ┌──────col_1─┬─────────────────────col_2─┬───────────────col_3─┐ │ 2019-06-11 │ 2019-06-11 07:00:00.00000 │ 2019-06-11 09:00:00 │ │ 0000-00-00 │ 0000-00-00 00:00:00.00000 │ 0000-00-00 00:00:00 │ └────────────┴───────────────────────────┴─────────────────────┘ # tz_name overwrite tz_offset -=> DBGInvoke dag('select * from default.test where col_2 > col_3',4,'default',28800,'UTC') " --dag_planner="optree +=> DBGInvoke dag('select * from default.test where col_2 > col_3',4,'default',28800,'UTC') # ts_col in group by clause -=> DBGInvoke dag('select count(1) from default.test where col_2 > \'2019-06-11 15:00:00\' group by col_2',4,'default',28800) " --dag_planner="optree +=> DBGInvoke dag('select count(1) from default.test where col_2 > \'2019-06-11 15:00:00\' group by col_2',4,'default',28800) ┌─count(1)─┬─────────────────────col_2─┐ │ 2 │ 2019-06-11 08:00:00.00000 │ └──────────┴───────────────────────────┘ # ts_col in agg clause -=> DBGInvoke dag('select max(col_2) from default.test group by col_1',4,'default',28800) " --dag_planner="optree +=> DBGInvoke dag('select max(col_2) from default.test group by col_1',4,'default',28800) ┌──────────max(col_2)─┬──────col_1─┐ │ 0000-00-00 00:00:00 │ 0000-00-00 │ │ 2019-06-11 08:00:00 │ 2019-06-12 │ @@ -87,7 +87,7 @@ └─────────────────────┴────────────┘ # ts_col in agg clause for arrow encode -=> DBGInvoke dag('select max(col_2) from default.test group by col_1',4,'arrow',28800) " --dag_planner="optree +=> DBGInvoke dag('select max(col_2) from default.test group by col_1',4,'arrow',28800) ┌──────────max(col_2)─┬──────col_1─┐ │ 0000-00-00 00:00:00 │ 0000-00-00 │ │ 2019-06-11 16:00:00 │ 2019-06-12 │