Skip to content

Commit

Permalink
7
Browse files Browse the repository at this point in the history
  • Loading branch information
@Yukang-Lian
Yukang-Lian committed Dec 25, 2023
1 parent 8b6d3f4 commit dcdc080
Show file tree
Hide file tree
Showing 5 changed files with 46 additions and 21 deletions.
6 changes: 6 additions & 0 deletions be/src/olap/wal_dirs_info.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -124,6 +124,12 @@ Status WalDirsInfo::add(const std::string& wal_dir, size_t limit, size_t used,
return Status::OK();
}

Status WalDirsInfo::clear() {
std::unique_lock wlock(_lock);
_wal_dirs_info_vec.clear();
return Status::OK();
}

std::string WalDirsInfo::get_available_random_wal_dir() {
if (_wal_dirs_info_vec.size() == 1) {
return (*_wal_dirs_info_vec.begin())->get_wal_dir();
Expand Down
1 change: 1 addition & 0 deletions be/src/olap/wal_dirs_info.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -65,6 +65,7 @@ class WalDirsInfo {
WalDirsInfo() = default;
~WalDirsInfo() = default;
Status add(const std::string& wal_dir, size_t limit, size_t used, size_t pre_allocated);
Status clear();
std::string get_available_random_wal_dir();
size_t get_max_available_size();
Status update_wal_dir_limit(std::string wal_dir, size_t limit = -1);
Expand Down
9 changes: 8 additions & 1 deletion be/src/runtime/group_commit_mgr.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -512,9 +512,16 @@ bool LoadBlockQueue::has_enough_wal_disk_space(
}
if (pre_allocated < available_bytes) {
st = wal_mgr->update_wal_dir_limit(wal_base_path);
if (!st.ok()) {
LOG(WARNING) << "update wal dir limit failed, reason: " << st.to_string();
}
st = wal_mgr->update_wal_dir_used(wal_base_path);
if (!st.ok()) {
LOG(WARNING) << "update wal disk info map failed, reason: " << st.to_string();
LOG(WARNING) << "update wal dir used failed, reason: " << st.to_string();
}
st = wal_mgr->update_wal_dir_pre_allocated(wal_base_path, pre_allocated, true);
if (!st.ok()) {
LOG(WARNING) << "update wal dir pre_allocated failed, reason: " << st.to_string();
}
block_queue_pre_allocated.fetch_add(pre_allocated);
return true;
Expand Down
9 changes: 3 additions & 6 deletions be/test/http/stream_load_test.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -55,12 +55,9 @@ void http_request_done_cb(struct evhttp_request* req, void* arg) {
TEST_F(StreamLoadTest, TestHeader) {
// 1G
auto wal_mgr = WalManager::create_shared(ExecEnv::GetInstance(), config::group_commit_wal_path);
wal_mgr->_wal_disk_info_map.insert(
std::make_pair("test_path_1", std::make_shared<WalManager::WalDiskInfo>(1000, 0, 0)));
wal_mgr->_wal_disk_info_map.insert(
std::make_pair("test_path_2", std::make_shared<WalManager::WalDiskInfo>(10000, 0, 0)));
wal_mgr->_wal_disk_info_map.insert(
std::make_pair("test_path_3", std::make_shared<WalManager::WalDiskInfo>(100000, 0, 0)));
static_cast<void>(wal_mgr->_wal_dirs_info->add("test_path_1", 1000, 0, 0));
static_cast<void>(wal_mgr->_wal_dirs_info->add("test_path_2", 10000, 0, 0));
static_cast<void>(wal_mgr->_wal_dirs_info->add("test_path_3", 100000, 0, 0));
ExecEnv::GetInstance()->set_wal_mgr(wal_mgr);
// 1. empty info
{
Expand Down
42 changes: 28 additions & 14 deletions be/test/olap/wal_manager_test.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -135,81 +135,95 @@ TEST_F(WalManagerTest, TestDynamicWalSpaceLimt) {

_env->wal_mgr()->wal_limit_test_bytes = available_bytes;
config::group_commit_wal_max_disk_limit = "0%";
EXPECT_EQ(_env->wal_mgr()->_init_wal_disk_info(), Status::OK());
EXPECT_EQ(_env->wal_mgr()->_init_wal_dirs_info(), Status::OK());
EXPECT_EQ(_env->wal_mgr()->wal_limit_test_bytes, 0);
EXPECT_EQ(_env->wal_mgr()->_wal_dirs_info->clear(), Status::OK());

_env->wal_mgr()->wal_limit_test_bytes = available_bytes;
config::group_commit_wal_max_disk_limit = "5%";
EXPECT_EQ(_env->wal_mgr()->_init_wal_disk_info(), Status::OK());
EXPECT_EQ(_env->wal_mgr()->_init_wal_dirs_info(), Status::OK());
wal_limit_bytes = available_bytes * 0.05;
EXPECT_EQ(_env->wal_mgr()->wal_limit_test_bytes, wal_limit_bytes);
EXPECT_EQ(_env->wal_mgr()->_wal_dirs_info->clear(), Status::OK());

_env->wal_mgr()->wal_limit_test_bytes = available_bytes;
config::group_commit_wal_max_disk_limit = "50%";
EXPECT_EQ(_env->wal_mgr()->_init_wal_disk_info(), Status::OK());
EXPECT_EQ(_env->wal_mgr()->_init_wal_dirs_info(), Status::OK());
wal_limit_bytes = available_bytes * 0.5;
EXPECT_EQ(_env->wal_mgr()->wal_limit_test_bytes, wal_limit_bytes);
EXPECT_EQ(_env->wal_mgr()->_wal_dirs_info->clear(), Status::OK());

_env->wal_mgr()->wal_limit_test_bytes = available_bytes;
config::group_commit_wal_max_disk_limit = "200%";
EXPECT_EQ(_env->wal_mgr()->_init_wal_disk_info(), Status::OK());
EXPECT_EQ(_env->wal_mgr()->_init_wal_dirs_info(), Status::OK());
wal_limit_bytes = available_bytes * 2;
EXPECT_EQ(_env->wal_mgr()->wal_limit_test_bytes, wal_limit_bytes);
EXPECT_EQ(_env->wal_mgr()->_wal_dirs_info->clear(), Status::OK());

_env->wal_mgr()->wal_limit_test_bytes = available_bytes;
config::group_commit_wal_max_disk_limit = "-10%";
EXPECT_EQ(_env->wal_mgr()->_init_wal_disk_info(), Status::InternalError(""));
EXPECT_EQ(_env->wal_mgr()->_init_wal_dirs_info(), Status::InternalError(""));
EXPECT_EQ(_env->wal_mgr()->wal_limit_test_bytes, available_bytes);
EXPECT_EQ(_env->wal_mgr()->_wal_dirs_info->clear(), Status::OK());

_env->wal_mgr()->wal_limit_test_bytes = available_bytes;
config::group_commit_wal_max_disk_limit = "0";
EXPECT_EQ(_env->wal_mgr()->_init_wal_disk_info(), Status::OK());
EXPECT_EQ(_env->wal_mgr()->_init_wal_dirs_info(), Status::OK());
EXPECT_EQ(_env->wal_mgr()->wal_limit_test_bytes, 0);
EXPECT_EQ(_env->wal_mgr()->_wal_dirs_info->clear(), Status::OK());

// 1M
_env->wal_mgr()->wal_limit_test_bytes = available_bytes;
config::group_commit_wal_max_disk_limit = "1048576";
EXPECT_EQ(_env->wal_mgr()->_init_wal_disk_info(), Status::OK());
EXPECT_EQ(_env->wal_mgr()->_init_wal_dirs_info(), Status::OK());
EXPECT_EQ(_env->wal_mgr()->wal_limit_test_bytes, 1048576);
EXPECT_EQ(_env->wal_mgr()->_wal_dirs_info->clear(), Status::OK());

// 1G
_env->wal_mgr()->wal_limit_test_bytes = available_bytes;
config::group_commit_wal_max_disk_limit = "1073741824";
EXPECT_EQ(_env->wal_mgr()->_init_wal_disk_info(), Status::OK());
EXPECT_EQ(_env->wal_mgr()->_init_wal_dirs_info(), Status::OK());
EXPECT_EQ(_env->wal_mgr()->wal_limit_test_bytes, 1073741824);
EXPECT_EQ(_env->wal_mgr()->_wal_dirs_info->clear(), Status::OK());

// 100G
_env->wal_mgr()->wal_limit_test_bytes = available_bytes;
config::group_commit_wal_max_disk_limit = "107374182400";
EXPECT_EQ(_env->wal_mgr()->_init_wal_disk_info(), Status::OK());
EXPECT_EQ(_env->wal_mgr()->_init_wal_dirs_info(), Status::OK());
EXPECT_EQ(_env->wal_mgr()->wal_limit_test_bytes, 107374182400);
EXPECT_EQ(_env->wal_mgr()->_wal_dirs_info->clear(), Status::OK());

// 1M
_env->wal_mgr()->wal_limit_test_bytes = available_bytes;
config::group_commit_wal_max_disk_limit = "1M";
EXPECT_EQ(_env->wal_mgr()->_init_wal_disk_info(), Status::OK());
EXPECT_EQ(_env->wal_mgr()->_init_wal_dirs_info(), Status::OK());
EXPECT_EQ(_env->wal_mgr()->wal_limit_test_bytes, 1048576);
EXPECT_EQ(_env->wal_mgr()->_wal_dirs_info->clear(), Status::OK());

// 1G
_env->wal_mgr()->wal_limit_test_bytes = available_bytes;
config::group_commit_wal_max_disk_limit = "1G";
EXPECT_EQ(_env->wal_mgr()->_init_wal_disk_info(), Status::OK());
EXPECT_EQ(_env->wal_mgr()->_init_wal_dirs_info(), Status::OK());
EXPECT_EQ(_env->wal_mgr()->wal_limit_test_bytes, 1073741824);
EXPECT_EQ(_env->wal_mgr()->_wal_dirs_info->clear(), Status::OK());

// 100G
_env->wal_mgr()->wal_limit_test_bytes = available_bytes;
config::group_commit_wal_max_disk_limit = "100G";
EXPECT_EQ(_env->wal_mgr()->_init_wal_disk_info(), Status::OK());
EXPECT_EQ(_env->wal_mgr()->_init_wal_dirs_info(), Status::OK());
EXPECT_EQ(_env->wal_mgr()->wal_limit_test_bytes, 107374182400);
EXPECT_EQ(_env->wal_mgr()->_wal_dirs_info->clear(), Status::OK());

_env->wal_mgr()->wal_limit_test_bytes = available_bytes;
config::group_commit_wal_max_disk_limit = "-1024";
EXPECT_EQ(_env->wal_mgr()->_init_wal_disk_info(), Status::InternalError(""));
EXPECT_EQ(_env->wal_mgr()->_init_wal_dirs_info(), Status::InternalError(""));
EXPECT_EQ(_env->wal_mgr()->wal_limit_test_bytes, available_bytes);
EXPECT_EQ(_env->wal_mgr()->_wal_dirs_info->clear(), Status::OK());

_env->wal_mgr()->wal_limit_test_bytes = available_bytes;
config::group_commit_wal_max_disk_limit = "-1M";
EXPECT_EQ(_env->wal_mgr()->_init_wal_disk_info(), Status::InternalError(""));
EXPECT_EQ(_env->wal_mgr()->_init_wal_dirs_info(), Status::InternalError(""));
EXPECT_EQ(_env->wal_mgr()->wal_limit_test_bytes, available_bytes);
EXPECT_EQ(_env->wal_mgr()->_wal_dirs_info->clear(), Status::OK());
}
} // namespace doris

0 comments on commit dcdc080

Please sign in to comment.