From 9d0cde03fc419e0822a59c0cef46037c31fdc844 Mon Sep 17 00:00:00 2001 From: HappenLee Date: Tue, 1 Oct 2024 10:54:26 +0800 Subject: [PATCH] disable mem save code in shuffle --- be/src/pipeline/exec/hashjoin_build_sink.cpp | 3 +-- be/src/vec/core/block.cpp | 4 +++- be/src/vec/runtime/shared_hash_table_controller.cpp | 6 +----- be/src/vec/runtime/shared_hash_table_controller.h | 1 - 4 files changed, 5 insertions(+), 9 deletions(-) diff --git a/be/src/pipeline/exec/hashjoin_build_sink.cpp b/be/src/pipeline/exec/hashjoin_build_sink.cpp index bd618d245984a2..b6cd342d3988be 100644 --- a/be/src/pipeline/exec/hashjoin_build_sink.cpp +++ b/be/src/pipeline/exec/hashjoin_build_sink.cpp @@ -123,7 +123,7 @@ Status HashJoinBuildSinkLocalState::close(RuntimeState* state, Status exec_statu if (_shared_state->build_block) { // release the memory of unused column in probe stage _shared_state->build_block->clear_column_mem_not_keep( - p._should_keep_column_flags, p._shared_hash_table_context.get() != nullptr); + p._should_keep_column_flags, bool(p._shared_hashtable_controller)); } } @@ -596,7 +596,6 @@ Status HashJoinBuildSinkOperatorX::sink(RuntimeState* state, vectorized::Block* _shared_hash_table_context->build_indexes_null = local_state._shared_state->build_indexes_null; local_state._runtime_filter_slots->copy_to_shared_context(_shared_hash_table_context); - _shared_hashtable_controller->signal(node_id()); } } else if (!local_state._should_build_hash_table) { DCHECK(_shared_hashtable_controller != nullptr); diff --git a/be/src/vec/core/block.cpp b/be/src/vec/core/block.cpp index 6f07a8e9c6f324..32c47d42e57e65 100644 --- a/be/src/vec/core/block.cpp +++ b/be/src/vec/core/block.cpp @@ -759,7 +759,9 @@ void Block::clear_column_mem_not_keep(const std::vector& column_keep_flags } } - if (need_keep_first && data[0].column->size() != origin_rows) { + if (need_keep_first && data.size() == column_keep_flags.size() && + std::all_of(column_keep_flags.begin(), column_keep_flags.end(), + [](bool v) { return !v; })) { auto first_column = data[0].column->clone_empty(); first_column->resize(origin_rows); data[0].column = std::move(first_column); diff --git a/be/src/vec/runtime/shared_hash_table_controller.cpp b/be/src/vec/runtime/shared_hash_table_controller.cpp index 4b77b1ed8a34bd..e5a28bed6ce50a 100644 --- a/be/src/vec/runtime/shared_hash_table_controller.cpp +++ b/be/src/vec/runtime/shared_hash_table_controller.cpp @@ -42,7 +42,7 @@ SharedHashTableContextPtr SharedHashTableController::get_context(int my_node_id) return _shared_contexts[my_node_id]; } -void SharedHashTableController::signal(int my_node_id) { +void SharedHashTableController::signal_finish(int my_node_id) { std::lock_guard lock(_mutex); auto it = _shared_contexts.find(my_node_id); if (it != _shared_contexts.cend()) { @@ -52,10 +52,6 @@ void SharedHashTableController::signal(int my_node_id) { for (auto& dep : _dependencies[my_node_id]) { dep->set_ready(); } -} - -void SharedHashTableController::signal_finish(int my_node_id) { - std::lock_guard lock(_mutex); for (auto& dep : _finish_dependencies[my_node_id]) { dep->set_ready(); } diff --git a/be/src/vec/runtime/shared_hash_table_controller.h b/be/src/vec/runtime/shared_hash_table_controller.h index b04b1cdba064b9..173f9d46e890c8 100644 --- a/be/src/vec/runtime/shared_hash_table_controller.h +++ b/be/src/vec/runtime/shared_hash_table_controller.h @@ -76,7 +76,6 @@ class SharedHashTableController { void set_builder_and_consumers(TUniqueId builder, int node_id); TUniqueId get_builder_fragment_instance_id(int my_node_id); SharedHashTableContextPtr get_context(int my_node_id); - void signal(int my_node_id); void signal_finish(int my_node_id); void append_dependency(int node_id, std::shared_ptr dep, std::shared_ptr finish_dep) {