Ydb stable 23-2-12

x-stable-origin-commit: bdfbd91e7d23d931fe44daf4085f7c5573a0c951

ydb/apps/dstool/lib/commands.py

@@ -17,6 +17,7 @@
 import ydb.apps.dstool.lib.dstool_cmd_group_state as group_state
 import ydb.apps.dstool.lib.dstool_cmd_group_take_snapshot as group_take_snapshot
 import ydb.apps.dstool.lib.dstool_cmd_group_virtual_create as group_virtual_create
+import ydb.apps.dstool.lib.dstool_cmd_group_virtual_cancel as group_virtual_cancel
 
 import ydb.apps.dstool.lib.dstool_cmd_pool_create_virtual as pool_create_virtual
 import ydb.apps.dstool.lib.dstool_cmd_pool_list as pool_list
@@ -41,15 +42,15 @@
     node_list,
     box_list,
     pool_list, pool_create_virtual,
-    group_check, group_decommit, group_show_blob_info, group_show_usage_by_tablets, group_state, group_take_snapshot, group_add, group_list, group_virtual_create,
+    group_check, group_decommit, group_show_blob_info, group_show_usage_by_tablets, group_state, group_take_snapshot, group_add, group_list, group_virtual_create, group_virtual_cancel,
     pdisk_add_by_serial, pdisk_remove_by_serial, pdisk_set, pdisk_list,
     vdisk_remove_donor, vdisk_evict, vdisk_list, vdisk_wipe,
 ]
 
 default_structure = [
     ('pdisk', ['add-by-serial', 'remove-by-serial', 'set', 'list']),
     ('vdisk', ['evict', 'remove-donor', 'wipe', 'list']),
-    ('group', ['add', 'check', 'decommit', ('show', ['blob-info', 'usage-by-tablets']), 'state', 'take-snapshot', 'list', ('virtual', ['create'])]),
+    ('group', ['add', 'check', 'decommit', ('show', ['blob-info', 'usage-by-tablets']), 'state', 'take-snapshot', 'list', ('virtual', ['create', 'cancel'])]),
     ('pool', ['list', ('create', ['virtual'])]),
     ('box', ['list']),
     ('node', ['list']),

ydb/apps/dstool/lib/dstool_cmd_group_decommit.py

@@ -7,7 +7,9 @@
 
 def add_options(p):
     common.add_group_ids_option(p, required=True)
-    p.add_argument('--hive-id', type=int, required=True, help='tablet id of containing hive')
+    g = p.add_mutually_exclusive_group(required=True)
+    g.add_argument('--hive-id', type=int, help='tablet id of containing hive')
+    g.add_argument('--database', type=str, help='database path of containing hive')
     p.add_argument('--log-channel-sp', type=str, metavar='POOL_NAME', help='channel 0 specifier')
     p.add_argument('--snapshot-channel-sp', type=str, metavar='POOL_NAME', help='channel 1 specifier (defaults to channel 0)')
     p.add_argument('--data-channel-sp', type=str, metavar='POOL_NAME[*COUNT]', nargs='*', help='data channel specifier')
@@ -17,7 +19,10 @@ def do(args):
     request = common.create_bsc_request(args)
     cmd = request.Command.add().DecommitGroups
     cmd.GroupIds.extend(args.group_ids)
-    cmd.HiveId = args.hive_id
+    if args.hive_id is not None:
+        cmd.HiveId = args.hive_id
+    if args.database is not None:
+        cmd.Database = args.database
 
     if args.log_channel_sp or args.snapshot_channel_sp or args.data_channel_sp:
         if args.log_channel_sp is None:

ydb/apps/dstool/lib/dstool_cmd_group_virtual_cancel.py

@@ -0,0 +1,15 @@
+import ydb.apps.dstool.lib.common as common
+
+description = 'Cancel virtual group creation/decommission'
+
+
+def add_options(p):
+    p.add_argument('--group-id', type=int, required=True, help='group id to cancel')
+
+
+def do(args):
+    request = common.create_bsc_request(args)
+    cmd = request.Command.add().CancelVirtualGroup
+    cmd.GroupId = args.group_id
+    response = common.invoke_bsc_request(request)
+    common.print_request_result(args, request, response)

ydb/apps/dstool/lib/dstool_cmd_group_virtual_create.py

@@ -8,7 +8,9 @@
 
 def add_options(p):
     p.add_argument('--name', type=str, required=True, nargs='+', help='cluster-unique name(s) of newly created virtual groups')
-    p.add_argument('--hive-id', type=int, required=True, help='tablet id of containing hive')
+    g = p.add_mutually_exclusive_group(required=True)
+    g.add_argument('--hive-id', type=int, help='tablet id of containing hive')
+    g.add_argument('--database', type=str, help='database path of containing hive')
     g = p.add_mutually_exclusive_group(required=True)
     g.add_argument('--storage-pool-name', type=str, metavar='POOL_NAME', help='name of the containing storage pool')
     g.add_argument('--storage-pool-id', type=str, metavar='BOX:POOL', help='id of the cotaining storage pool')
@@ -24,7 +26,10 @@ def do(args):
         cmd = request.Command.add().AllocateVirtualGroup
 
         cmd.Name = name
-        cmd.HiveId = args.hive_id
+        if args.hive_id is not None:
+            cmd.HiveId = args.hive_id
+        if args.database is not None:
+            cmd.Database = args.database
 
         if args.storage_pool_name is not None:
             cmd.StoragePoolName = args.storage_pool_name

ydb/core/base/appdata.h

@@ -181,6 +181,8 @@ struct TAppData {
 
     TString ClusterName;
 
+    bool YamlConfigEnabled = false;
+
     TAppData(
             ui32 sysPoolId, ui32 userPoolId, ui32 ioPoolId, ui32 batchPoolId,
             TMap<TString, ui32> servicePools,

ydb/core/blobstorage/backpressure/queue_backpressure_client.cpp

@@ -701,7 +701,7 @@ class TVDiskBackpressureClientActor : public TActorBootstrapped<TVDiskBackpressu
             ctx.Send(sender, ev->Release().Release(), 0, cookie, std::move(ev->TraceId));
             AssimilateRequests.erase(it);
         } else {
-            const TString& message = TStringBuilder() << "unexpected TEvVAssimilateResult received"
+            const TString message = TStringBuilder() << "unexpected TEvVAssimilateResult received"
                 << " Cookie# " << ev->Cookie
                 << " Sender# " << ev->Sender
                 << " Msg# " << ev->Get()->ToString()

ydb/core/blobstorage/groupinfo/blobstorage_groupinfo.cpp

@@ -359,7 +359,7 @@ bool TBlobStorageGroupInfo::TTopology::IsValidId(const TVDiskID& vdisk) const {
     if (vdisk.FailRealm >= FailRealms.size()) {
         return false;
     }
-    if (vdisk.FailDomain >= TotalFailDomains) {
+    if (vdisk.FailDomain >= GetNumFailDomainsPerFailRealm()) {
         return false;
     }
     if (vdisk.VDisk >= GetNumVDisksPerFailDomain()) {
@@ -372,7 +372,7 @@ bool TBlobStorageGroupInfo::TTopology::IsValidId(const TVDiskIdShort& vdisk) con
     if (vdisk.FailRealm >= FailRealms.size()) {
         return false;
     }
-    if (vdisk.FailDomain >= TotalFailDomains) {
+    if (vdisk.FailDomain >= GetNumFailDomainsPerFailRealm()) {
         return false;
     }
     if (vdisk.VDisk >= GetNumVDisksPerFailDomain()) {

ydb/core/blobstorage/vdisk/common/vdisk_config.cpp

@@ -49,7 +49,7 @@ namespace NKikimr {
         RecoveryLogCutterRegularDuration = TDuration::Seconds(30);
         AdvanceEntryPointTimeout = TDuration::Seconds(10);          // 10 seconds (FIXME: use feedback from PDisk)
         SyncTimeInterval = TDuration::Seconds(3);                   // 3 seconds
-        SyncJobTimeout = TDuration::Minutes(30);                    // 30 minutes
+        SyncJobTimeout = TDuration::Max();                          // disabled
         SyncerRLDRetryTimeout = TDuration::Seconds(1);
         AnubisTimeout = TDuration::Minutes(60);
         RunSyncer = true;

ydb/core/blobstorage/vdisk/scrub/scrub_actor.cpp

@@ -37,6 +37,7 @@ namespace NKikimr {
             hFunc(TEvNonrestoredCorruptedBlobNotify, Handle);
             hFunc(NPDisk::TEvLogResult, Handle);
             hFunc(NPDisk::TEvCutLog, Handle);
+            hFunc(TEvTakeHullSnapshotResult, Handle);
 
             default:
                 Y_FAIL("unexpected event Type# 0x%08" PRIx32, type);

ydb/core/blobstorage/vdisk/syncer/syncer_job_actor.cpp

@@ -125,7 +125,9 @@ namespace NKikimr {
 
         void Bootstrap(const TActorContext &ctx) {
             // don't run sync job for too long
-            ctx.Schedule(SyncerCtx->Config->SyncJobTimeout, new TEvents::TEvWakeup());
+            if (const auto timeout = SyncerCtx->Config->SyncJobTimeout; timeout != TDuration::Max()) {
+                ctx.Schedule(timeout, new TEvents::TEvWakeup());
+            }
 
             // initiate requests
             TSjOutcome outcome = Task->NextRequest();

ydb/core/cms/cms_ut_common.cpp

@@ -1,7 +1,7 @@
 #include "cms_impl.h"
 #include "cms_ut_common.h"
-#include "ut_helpers.h"
 #include "sentinel.h"
+#include "ut_helpers.h"
 
 #include <ydb/core/base/tabletid.h>
 #include <ydb/core/blobstorage/crypto/default.h>
@@ -56,7 +56,7 @@ void TFakeNodeWhiteboardService::Handle(TEvBlobStorage::TEvControllerConfigReque
 {
     TGuard<TMutex> guard(Mutex);
     auto &rec = ev->Get()->Record;
-    auto *resp = new TEvBlobStorage::TEvControllerConfigResponse;
+    auto resp = MakeHolder<TEvBlobStorage::TEvControllerConfigResponse>();
     if (rec.GetRequest().CommandSize() && rec.GetRequest().GetCommand(0).HasQueryBaseConfig()) {
         resp->Record.CopyFrom(Config);
     } else if (rec.GetRequest().CommandSize() && rec.GetRequest().GetCommand(0).HasReadDriveStatus()) {
@@ -69,27 +69,27 @@ void TFakeNodeWhiteboardService::Handle(TEvBlobStorage::TEvControllerConfigReque
         driveStatus.MutableHostKey()->SetIcPort(drive.GetHostKey().GetIcPort());
         driveStatus.SetPath(drive.GetPath());
         driveStatus.SetStatus(NKikimrBlobStorage::ACTIVE);
-    } else if (rec.GetRequest().CommandSize() && rec.GetRequest().GetCommand(0).HasUpdateDriveStatus()) { // assume that all commands are UpdateDriveStatus
+    } else if (rec.GetRequest().CommandSize() && rec.GetRequest().GetCommand(0).HasUpdateDriveStatus()) {
+        // assume that all commands are UpdateDriveStatus
         if (NoisyBSCPipe && ++NoisyBSCPipeCounter % 3) {
             ctx.Send(ev->Sender, new TEvSentinel::TEvBSCPipeDisconnected, 0);
-            delete resp;
             return;
         }
         bool success = true;
         for (ui32 i = 0; i < rec.GetRequest().CommandSize(); ++i) {
-            auto cmd = rec.GetRequest().GetCommand(i).GetUpdateDriveStatus();
-            auto id = NCms::TPDiskID(cmd.GetHostKey().GetNodeId(), cmd.GetPDiskId());
-            if (auto& pattern = BSControllerResponsePatterns[id]; !pattern.empty() && !pattern[0]) {
-                success = false;
+            const auto &cmd = rec.GetRequest().GetCommand(i).GetUpdateDriveStatus();
+            const auto id = NCms::TPDiskID(cmd.GetHostKey().GetNodeId(), cmd.GetPDiskId());
+            if (auto& pattern = BSControllerResponsePatterns[id]; !pattern.empty()) {
+                success = success && pattern[0];
+                resp->Record.MutableResponse()->AddStatus()->SetSuccess(pattern[0]);
                 pattern.erase(pattern.begin());
-                resp->Record.MutableResponse()->AddStatus()->SetSuccess(false);
             } else {
                 resp->Record.MutableResponse()->AddStatus()->SetSuccess(true);
             }
         }
         resp->Record.MutableResponse()->SetSuccess(success);
     }
-    ctx.Send(ev->Sender, resp, 0, ev->Cookie);
+    ctx.Send(ev->Sender, std::move(resp), 0, ev->Cookie);
 }
 
 void TFakeNodeWhiteboardService::Handle(TEvWhiteboard::TEvTabletStateRequest::TPtr &ev,
@@ -1177,10 +1177,10 @@ NKikimrCms::TGetLogTailResponse TCmsTestEnv::GetLogTail(ui32 type,
     return rec;
 }
 
-void TCmsTestEnv::AddBSCFailures(const NCms::TPDiskID& id, TVector<bool> failuresPattern) {
+void TCmsTestEnv::AddBSCFailures(const NCms::TPDiskID& id, TVector<bool> &&failuresPattern) {
     TGuard<TMutex> guard(TFakeNodeWhiteboardService::Mutex);
     auto& vec = TFakeNodeWhiteboardService::BSControllerResponsePatterns[id];
-    vec.insert(vec.end(), failuresPattern.begin(), failuresPattern.end());
+    std::move(failuresPattern.begin(), failuresPattern.end(), std::back_inserter(vec));
 }
 
 void TCmsTestEnv::EnableNoisyBSCPipe() {

ydb/core/cms/cms_ut_common.h

@@ -387,7 +387,7 @@ class TCmsTestEnv : public TTestBasicRuntime {
                                                ui32 limit = 100,
                                                ui32 offset = 0);
 
-    void AddBSCFailures(const NCms::TPDiskID& id, TVector<bool> failuresPattern);
+    void AddBSCFailures(const NCms::TPDiskID &id, TVector<bool> &&failuresPattern);
 
     void EnableNoisyBSCPipe();
 

ydb/core/cms/config.h

@@ -23,8 +23,6 @@ struct TCmsSentinelConfig {
     TDuration RetryChangeStatus;
     ui32 ChangeStatusRetries;
 
-    TDuration BSCBatchTimeout;
-
     ui32 DefaultStateLimit;
     TMap<EPDiskState, ui32> StateLimits;
 

ydb/core/cms/console/configs_dispatcher.cpp

@@ -665,6 +665,8 @@ void TConfigsDispatcher::Handle(TEvConsole::TEvConfigSubscriptionNotification::T
         newYamlProtoConfig = YamlProtoConfig;
     }
 
+    AppData()->YamlConfigEnabled = YamlConfigEnabled;
+
     std::swap(YamlProtoConfig, newYamlProtoConfig);
 
     THashSet<ui32> affectedKinds;

ydb/core/cms/http.cpp

@@ -211,6 +211,15 @@ class TCmsHttp : public TActorBootstrapped<TCmsHttp> {
             return;
         }
 
+        if (msg->Request.GetPathInfo() == "/yaml-config-enabled") {
+
+            ctx.Send(ev->Sender, new NMon::TEvHttpInfoRes(TString(NMonitoring::HTTPOKJSON)
+                                                          + R"({"enabled":)" + (AppData()->YamlConfigEnabled ? "true" : "false") + "}",
+                                                          0,
+                                                          NMon::IEvHttpInfoRes::EContentType::Custom));
+            return;
+        }
+
         ReplyWithFile(ev, ctx, TString{msg->Request.GetPathInfo()});
     }