Merge pull request redpanda-data#13094 from dotnwat/space-management-…

…concurrency-config storage: add configuration to control space management concurrency
vbotbuildovich · Aug 30, 2023 · 66eef6f · 66eef6f
2 parents 46be53a + 36179ab
commit 66eef6f
Show file tree

Hide file tree

Showing 4 changed files with 26 additions and 3 deletions.
diff --git a/src/v/config/configuration.cc b/src/v/config/configuration.cc
@@ -1748,6 +1748,24 @@ configuration::configuration()
       25.0,
       {.min = 0.0, .max = 100.0},
       legacy_default<double>(0.0, legacy_version{9}))
+  , space_management_max_log_concurrency(
+      *this,
+      "space_management_max_log_concurrency",
+      "Maximum parallel logs inspected during space management process.",
+      {.needs_restart = needs_restart::no,
+       .example = "20",
+       .visibility = visibility::tunable},
+      20,
+      {.min = 1})
+  , space_management_max_segment_concurrency(
+      *this,
+      "space_management_max_segment_concurrency",
+      "Maximum parallel segments inspected during space management process.",
+      {.needs_restart = needs_restart::no,
+       .example = "10",
+       .visibility = visibility::tunable},
+      10,
+      {.min = 1})
   , cloud_storage_cache_size(
       *this,
       "cloud_storage_cache_size",

diff --git a/src/v/config/configuration.h b/src/v/config/configuration.h
@@ -337,6 +337,8 @@ struct configuration final : public config_store {
     property<double> retention_local_trim_overage_coeff;
     property<bool> space_management_enable;
     bounded_property<double, numeric_bounds> disk_reservation_percent;
+    bounded_property<uint16_t> space_management_max_log_concurrency;
+    bounded_property<uint16_t> space_management_max_segment_concurrency;
 
     // Archival cache
     property<uint64_t> cloud_storage_cache_size;

diff --git a/src/v/storage/disk_log_impl.cc b/src/v/storage/disk_log_impl.cc
@@ -2079,7 +2079,8 @@ disk_log_impl::disk_usage_and_reclaimable_space(gc_config input_cfg) {
         }
     }
 
-    ss::semaphore limit(10);
+    ss::semaphore limit(std::max<size_t>(
+      1, config::shard_local_cfg().space_management_max_segment_concurrency()));
 
     auto [retention, available, remaining, lcl] = co_await ss::when_all_succeed(
       // reduce segment subject to retention policy
@@ -2307,7 +2308,8 @@ disk_log_impl::disk_usage_target_time_retention(gc_config cfg) {
         co_return std::nullopt;
     }
 
-    ss::semaphore limit(10);
+    ss::semaphore limit(std::max<size_t>(
+      1, config::shard_local_cfg().space_management_max_segment_concurrency()));
 
     // roll up the amount of disk space taken by these segments
     auto usage = co_await ss::map_reduce(

diff --git a/src/v/storage/log_manager.cc b/src/v/storage/log_manager.cc
@@ -757,7 +757,8 @@ ss::future<usage_report> log_manager::disk_usage() {
         logs.push_back(it.second->handle);
     }
 
-    ss::semaphore limit(20);
+    ss::semaphore limit(std::max<size_t>(
+      1, config::shard_local_cfg().space_management_max_log_concurrency()));
 
     co_return co_await ss::map_reduce(
       logs.begin(),