[improve](load) reduce memory reserved in memtable limiter (#37511)

be/src/common/config.cpp

@@ -120,6 +120,8 @@ DEFINE_mInt32(double_resize_threshold, "23");
 // Turn down max. will use as much memory as possible.
 DEFINE_Int64(max_sys_mem_available_low_water_mark_bytes, "1717986918");
 
+DEFINE_Int64(memtable_limiter_reserved_memory_bytes, "838860800");
+
 // The size of the memory that gc wants to release each time, as a percentage of the mem limit.
 DEFINE_mString(process_minor_gc_size, "10%");
 DEFINE_mString(process_full_gc_size, "20%");

be/src/common/config.h

@@ -162,6 +162,9 @@ DECLARE_mInt32(double_resize_threshold);
 // Turn down max. will use as much memory as possible.
 DECLARE_Int64(max_sys_mem_available_low_water_mark_bytes);
 
+// reserve a small amount of memory so we do not trigger MinorGC
+DECLARE_Int64(memtable_limiter_reserved_memory_bytes);
+
 // The size of the memory that gc wants to release each time, as a percentage of the mem limit.
 DECLARE_mString(process_minor_gc_size);
 DECLARE_mString(process_full_gc_size);

be/src/olap/memtable_memory_limiter.cpp

@@ -77,29 +77,27 @@ void MemTableMemoryLimiter::register_writer(std::weak_ptr<MemTableWriter> writer
     _writers.push_back(writer);
 }
 
-int64_t MemTableMemoryLimiter::_avail_mem_lack() {
+bool MemTableMemoryLimiter::_sys_avail_mem_less_than_warning_water_mark() {
     // reserve a small amount of memory so we do not trigger MinorGC
-    auto reserved_mem = doris::MemInfo::sys_mem_available_low_water_mark();
-    auto avail_mem_lack = doris::MemInfo::sys_mem_available_warning_water_mark() -
-                          doris::GlobalMemoryArbitrator::sys_mem_available();
-    return avail_mem_lack + reserved_mem;
+    return doris::GlobalMemoryArbitrator::sys_mem_available() <
+           doris::MemInfo::sys_mem_available_warning_water_mark() +
+                   config::memtable_limiter_reserved_memory_bytes;
 }
 
-int64_t MemTableMemoryLimiter::_proc_mem_extra() {
+bool MemTableMemoryLimiter::_process_used_mem_more_than_soft_mem_limit() {
     // reserve a small amount of memory so we do not trigger MinorGC
-    auto reserved_mem = doris::MemInfo::sys_mem_available_low_water_mark();
-    auto proc_mem_extra =
-            GlobalMemoryArbitrator::process_memory_usage() - MemInfo::soft_mem_limit();
-    return proc_mem_extra + reserved_mem;
+    return GlobalMemoryArbitrator::process_memory_usage() >
+           MemInfo::soft_mem_limit() - config::memtable_limiter_reserved_memory_bytes;
 }
 
 bool MemTableMemoryLimiter::_soft_limit_reached() {
     return _mem_tracker->consumption() >= _load_soft_mem_limit || _hard_limit_reached();
 }
 
 bool MemTableMemoryLimiter::_hard_limit_reached() {
-    return _mem_tracker->consumption() >= _load_hard_mem_limit || _avail_mem_lack() >= 0 ||
-           _proc_mem_extra() >= 0;
+    return _mem_tracker->consumption() >= _load_hard_mem_limit ||
+           _sys_avail_mem_less_than_warning_water_mark() ||
+           _process_used_mem_more_than_soft_mem_limit();
 }
 
 bool MemTableMemoryLimiter::_load_usage_low() {

be/src/olap/memtable_memory_limiter.h

@@ -51,8 +51,8 @@ class MemTableMemoryLimiter {
     int64_t mem_usage() const { return _mem_usage; }
 
 private:
-    static int64_t _avail_mem_lack();
-    static int64_t _proc_mem_extra();
+    static inline bool _sys_avail_mem_less_than_warning_water_mark();
+    static inline bool _process_used_mem_more_than_soft_mem_limit();
 
     bool _soft_limit_reached();
     bool _hard_limit_reached();