From 32deac1db61ff0831736c484e0d244e0b8cf969d Mon Sep 17 00:00:00 2001 From: Gil Bregman Date: Tue, 10 Sep 2024 13:03:31 +0300 Subject: [PATCH] Revert change lb group to use namespace delete and add. Signed-off-by: Gil Bregman --- control/grpc.py | 251 ++++++++++++++++++++++++------------ control/proto/gateway.proto | 1 + control/server.py | 4 - control/state.py | 91 ------------- tests/test_cli_change_lb.py | 3 - 5 files changed, 167 insertions(+), 183 deletions(-) diff --git a/control/grpc.py b/control/grpc.py index 8add6a96..12ba7bf4 100644 --- a/control/grpc.py +++ b/control/grpc.py @@ -229,7 +229,6 @@ def __init__(self, config: GatewayConfig, gateway_state: GatewayStateHandler, rp self.ana_map = defaultdict(dict) self.cluster_nonce = {} self.bdev_cluster = {} - self.bdev_params = {} self.subsystem_nsid_bdev_and_uuid = NamespacesLocalList() self.subsystem_listeners = defaultdict(set) self._init_cluster_context() @@ -395,7 +394,7 @@ def execute_grpc_function(self, func, request, context): """ return self.omap_lock.execute_omap_locking_function(self._grpc_function_with_lock, func, request, context) - def create_bdev(self, anagrp: int, name, uuid, rbd_pool_name, rbd_image_name, block_size, create_image, rbd_image_size, context, peer_msg = ""): + def create_bdev(self, anagrp: int, name, uuid, rbd_pool_name, rbd_image_name, block_size, create_image, rbd_image_size, can_reuse_bdevs, context, peer_msg = ""): """Creates a bdev from an RBD image.""" if create_image: @@ -403,9 +402,13 @@ def create_bdev(self, anagrp: int, name, uuid, rbd_pool_name, rbd_image_name, bl else: cr_img_msg = "will not create image if doesn't exist" + reuse_msg = "" + if can_reuse_bdevs: + reuse_msg = ", will re-use bdev if already exists" + self.logger.info(f"Received request to create bdev {name} from" f" {rbd_pool_name}/{rbd_image_name} (size {rbd_image_size} bytes)" - f" with block size {block_size}, {cr_img_msg}, context={context}{peer_msg}") + f" with block size {block_size}, {cr_img_msg}{reuse_msg}, context={context}{peer_msg}") if block_size == 0: return BdevStatus(status=errno.EINVAL, @@ -444,6 +447,8 @@ def create_bdev(self, anagrp: int, name, uuid, rbd_pool_name, rbd_image_name, bl self.logger.exception(errmsg) return BdevStatus(status=errcode, error_message=f"Failure creating bdev {name}: {errmsg}") + status = 0 + errmsg = "" try: cluster_name=self._get_cluster(anagrp) bdev_name = rpc_bdev.bdev_rbd_create( @@ -455,32 +460,40 @@ def create_bdev(self, anagrp: int, name, uuid, rbd_pool_name, rbd_image_name, bl block_size=block_size, uuid=uuid, ) - with self.shared_state_lock: - self.bdev_cluster[name] = cluster_name - self.bdev_params[name] = {'uuid':uuid, 'pool_name':rbd_pool_name, 'image_name':rbd_image_name, 'image_size':rbd_image_size, 'block_size': block_size} self.logger.debug(f"bdev_rbd_create: {bdev_name}, cluster_name {cluster_name}") except Exception as ex: - self._put_cluster(cluster_name) - errmsg = f"bdev_rbd_create {name} failed" - self.logger.exception(errmsg) - errmsg = f"{errmsg} with:\n{ex}" - resp = self.parse_json_exeption(ex) + errmsg = f"Failure creating bdev {name}:\n{ex}" status = errno.ENODEV - if resp: - status = resp["code"] - errmsg = f"Failure creating bdev {name}: {resp['message']}" - return BdevStatus(status=status, error_message=errmsg) + if cluster_name: + resp = self.parse_json_exeption(ex) + if resp: + status = resp["code"] + if status < 0: + status = -status + if can_reuse_bdevs and status == errno.EEXIST: + self.logger.warning(f"Bdev {name} already exists") + bdev_name = name + if status != errno.EEXIST or not can_reuse_bdevs: + errmsg = f"bdev_rbd_create {name} failed" + self.logger.exception(errmsg) - # Just in case SPDK failed with no exception - if not bdev_name: - errmsg = f"Can't create bdev {name}" + if bdev_name: + assert name == bdev_name, f"Created bdev name {bdev_name} differs from requested name {name}" + assert cluster_name, f"Failed to get cluster name for bdev {name}" + if status and status != errno.EEXIST: + self.logger.warning(f"Bdev {bdev_name} was created but status is {status}") + with self.shared_state_lock: + self.bdev_cluster[name] = cluster_name + else: + if cluster_name: + self._put_cluster(cluster_name) + if not errmsg: + errmsg = f"Can't create bdev {name}" self.logger.error(errmsg) - return BdevStatus(status=errno.ENODEV, error_message=errmsg) - - assert name == bdev_name, f"Created bdev name {bdev_name} differs from requested name {name}" + return BdevStatus(status=status, error_message=errmsg) - return BdevStatus(status=0, error_message=os.strerror(0), bdev_name=name) + return BdevStatus(status=status, error_message=os.strerror(0), bdev_name=name) def resize_bdev(self, bdev_name, new_size, peer_msg = ""): """Resizes a bdev.""" @@ -548,12 +561,15 @@ def delete_bdev(self, bdev_name, recycling_mode=False, peer_msg=""): self.spdk_rpc_client, bdev_name, ) - if not recycling_mode: - del self.bdev_params[bdev_name] with self.shared_state_lock: cluster = self.bdev_cluster[bdev_name] self.logger.debug(f"to delete_bdev {bdev_name} cluster {cluster} ") self._put_cluster(cluster) + with self.shared_state_lock: + try: + del self.bdev_cluster[bdev_name] + except KeyError: + pass self.logger.debug(f"delete_bdev {bdev_name}: {ret}") except Exception as ex: errmsg = f"Failure deleting bdev {bdev_name}" @@ -1125,8 +1141,9 @@ def namespace_add_safe(self, request, context): anagrp = request.anagrpid ret_bdev = self.create_bdev(anagrp, bdev_name, request.uuid, request.rbd_pool_name, - request.rbd_image_name, request.block_size, create_image, request.size, context, peer_msg) - if ret_bdev.status != 0: + request.rbd_image_name, request.block_size, create_image, + request.size, request.can_reuse_bdevs, context, peer_msg) + if ret_bdev.status != 0 and (ret_bdev.status != errno.EEXIST or not request.can_reuse_bdevs): errmsg = f"Failure adding namespace {nsid_msg}to {request.subsystem_nqn}: {ret_bdev.error_message}" self.logger.error(errmsg) # Delete the bdev unless there was one already there, just to be on the safe side @@ -1192,6 +1209,7 @@ def namespace_change_load_balancing_group_safe(self, request, context): peer_msg = self.get_peer_message(context) change_lb_group_failure_prefix = f"Failure changing load balancing group for namespace with NSID {request.nsid} in {request.subsystem_nqn}" self.logger.info(f"Received request to change load balancing group for namespace with NSID {request.nsid} in {request.subsystem_nqn} to {request.anagrpid}, context: {context}{peer_msg}") + assert context, "Can't have a change load balancing group request in an update" if not request.subsystem_nqn: errmsg = f"Failure changing load balancing group for namespace, missing subsystem NQN" @@ -1213,64 +1231,127 @@ def namespace_change_load_balancing_group_safe(self, request, context): omap_lock = self.omap_lock.get_omap_lock_to_use(context) with omap_lock: ns_entry = None - if context: - # notice that the local state might not be up to date in case we're in the middle of update() but as the - # context is not None, we are not in an update(), the omap lock made sure that we got here with an updated local state - state = self.gateway_state.local.get_state() - ns_key = GatewayState.build_namespace_key(request.subsystem_nqn, request.nsid) - try: - state_ns = state[ns_key] - ns_entry = json.loads(state_ns) - except Exception as ex: - errmsg = f"{change_lb_group_failure_prefix}: Can't find entry for namespace {request.nsid} in {request.subsystem_nqn}" - self.logger.error(errmsg) - return pb2.req_status(status=errno.ENOENT, error_message=errmsg) - + # notice that the local state might not be up to date in case we're in the middle of update() but as the + # context is not None, we are not in an update(), the omap lock made sure that we got here with an updated local state + state = self.gateway_state.local.get_state() + ns_key = GatewayState.build_namespace_key(request.subsystem_nqn, request.nsid) try: - ret = rpc_nvmf.nvmf_subsystem_set_ns_ana_group( - self.spdk_rpc_client, - nqn=request.subsystem_nqn, - nsid=request.nsid, - anagrpid=request.anagrpid, - transit_anagrpid=0 - ) - self.logger.debug(f"nvmf_subsystem_set_ns_ana_group: {ret}") + state_ns = state[ns_key] + ns_entry = json.loads(state_ns) + assert request.nsid == ns_entry["nsid"], f'Got a namespace with NSID {ns_entry["nsid"]} which is different than the requested one {request.nsid}' + assert request.subsystem_nqn == ns_entry["subsystem_nqn"], f'Got a namespace from subsystem {ns_entry["subsystem_nqn"]} which is different than the requested one {request.subsystem_nqn}' except Exception as ex: - errmsg = f"{change_lb_group_failure_prefix}:\n{ex}" - resp = self.parse_json_exeption(ex) - status = errno.EINVAL - if resp: - status = resp["code"] - errmsg = f"{change_lb_group_failure_prefix}: {resp['message']}" - return pb2.req_status(status=status, error_message=errmsg) + errmsg = f"{change_lb_group_failure_prefix}: Can't find entry for namespace {request.nsid} in {request.subsystem_nqn}\n{ex}" + self.logger.error(errmsg) + return pb2.req_status(status=errno.ENOENT, error_message=errmsg) - # Just in case SPDK failed with no exception - if not ret: - self.logger.error(change_lb_group_failure_prefix) - return pb2.req_status(status=errno.EINVAL, error_message=change_lb_group_failure_prefix) + if not ns_entry: + errmsg = f"Failure changing load balancing group for namespace {request.nsid} in {request.subsystem_nqn}. Can't get namespace entry from local state" + self.logger.error(errmsg) + return pb2.req_status(status=errno.ENOENT, error_message=errmsg) - if context: - assert ns_entry, "Namespace entry is None for non-update call" - # Update gateway state + create_image = False + ns_size = None + force = False + try: + force = ns_entry["force"] + create_image = ns_entry["create_image"] + ns_size = ns_entry["size"] + except Exception: + self.logger.warning(f"Can't get all the attributes for namespace {request.nsid} in {request.subsystem_nqn}.") + + ns_qos_entry = None + ns_qos_key = GatewayState.build_namespace_qos_key(request.subsystem_nqn, request.nsid) + try: + state_ns_qos = state[ns_qos_key] + ns_qos_entry = json.loads(state_ns_qos) + except Exception: + self.logger.debug(f"No QOS limits found for namespace {request.nsid} in {request.subsystem_nqn}") + + if ns_qos_entry: try: - add_req = pb2.namespace_add_req(rbd_pool_name=ns_entry["rbd_pool_name"], - rbd_image_name=ns_entry["rbd_image_name"], - subsystem_nqn=ns_entry["subsystem_nqn"], - nsid=ns_entry["nsid"], - block_size=ns_entry["block_size"], - uuid=ns_entry["uuid"], - anagrpid=request.anagrpid, - create_image=ns_entry["create_image"], - size=int(ns_entry["size"]), - force=ns_entry["force"]) - json_req = json_format.MessageToJson( - add_req, preserving_proto_field_name=True, including_default_value_fields=True) - self.gateway_state.add_namespace(request.subsystem_nqn, request.nsid, json_req) - except Exception as ex: - errmsg = f"Error persisting namespace load balancing group for namespace with NSID {request.nsid} in {request.subsystem_nqn}" - self.logger.exception(errmsg) - errmsg = f"{errmsg}:\n{ex}" - return pb2.req_status(status=errno.EINVAL, error_message=errmsg) + if ns_qos_entry["subsystem_nqn"] != request.subsystem_nqn: + errmsg = f'Got subsystem {ns_qos_entry["subsystem_nqn"]} for QOS limits instead of {request.subsystem_nqn}, will not set QOS' + self.logger.error(errmsg) + ns_qos_entry = None + elif ns_qos_entry["nsid"] != request.nsid: + errmsg = f'Got NSID {ns_qos_entry["nsid"]} for QOS limits instead of {request.nsid}, will not set QOS' + self.logger.error(errmsg) + ns_qos_entry = None + except Exception: + self.logger.error(f"Error processing QOS limits, will not set QOS\n{ex}") + ns_qos_entry = None + + set_qos_req = None + if ns_qos_entry: + set_qos_req = pb2.namespace_set_qos_req() + set_qos_req.subsystem_nqn = request.subsystem_nqn + set_qos_req.nsid = request.nsid + try: + set_qos_req.rw_ios_per_second = int(ns_qos_entry["rw_ios_per_second"]) + except Exception: + self.logger.warning(f"Couldn't get QOS attribute rw_ios_per_second") + try: + set_qos_req.rw_mbytes_per_second = int(ns_qos_entry["rw_mbytes_per_second"]) + except Exception: + self.logger.warning(f"Couldn't get QOS attribute rw_mbytes_per_second") + try: + set_qos_req.r_mbytes_per_second = int(ns_qos_entry["r_mbytes_per_second"]) + except Exception: + self.logger.warning(f"Couldn't get QOS attribute r_mbytes_per_second") + try: + set_qos_req.w_mbytes_per_second = int(ns_qos_entry["w_mbytes_per_second"]) + except Exception: + self.logger.warning(f"Couldn't get QOS attribute w_mbytes_per_second") + + namespace_add_req = pb2.namespace_add_req() + namespace_add_req.subsystem_nqn = request.subsystem_nqn + namespace_add_req.nsid = request.nsid + namespace_add_req.anagrpid = request.anagrpid + namespace_add_req.create_image = False + namespace_add_req.create_image = create_image + if ns_size: + namespace_add_req.size = int(ns_size) + namespace_add_req.force = force + namespace_add_req.can_reuse_bdevs = True + errmsg = None + try: + if ns_entry["uuid"]: + namespace_add_req.uuid=ns_entry["uuid"] + except KeyError: + errmsg = f"Failure changing load balancing group for namespace {request.nsid} in {request.subsystem_nqn}: Can't get UUID" + try: + namespace_add_req.rbd_pool_name=ns_entry["rbd_pool_name"] + except KeyError: + errmsg = f"Failure changing load balancing group for namespace {request.nsid} in {request.subsystem_nqn}: Can't find RBD pool name" + try: + namespace_add_req.rbd_image_name=ns_entry["rbd_image_name"] + except KeyError: + errmsg = f"Failure changing load balancing group for namespace {request.nsid} in {request.subsystem_nqn}: Can't find RBD image name" + try: + namespace_add_req.block_size=ns_entry["block_size"] + except KeyError: + errmsg = f"Failure changing load balancing group for namespace {request.nsid} in {request.subsystem_nqn}: Can't find associated block device block size" + + if errmsg: + self.logger.error(errmsg) + return pb2.req_status(status=errno.ENODEV, error_message=errmsg) + + del_req = pb2.namespace_delete_req(subsystem_nqn=request.subsystem_nqn, nsid=request.nsid) + ret_del = self.namespace_delete_extended(del_req, False, context) + if ret_del.status != 0: + errmsg = f"Failure changing load balancing group for namespace {request.nsid} in {request.subsystem_nqn}. Can't delete namespace: {ret_del.error_message}" + self.logger.error(errmsg) + return pb2.req_status(status=ret_del.status, error_message=errmsg) + + ret_ns = self.namespace_add_safe(namespace_add_req, context) + if ret_ns.status != 0: + errmsg = f"Failure changing load balancing group for namespace {request.nsid} in {request.subsystem_nqn}:{ret_ns.error_message}" + self.logger.error(errmsg) + return pb2.req_status(status=ret_ns.status, error_message=errmsg) + + if set_qos_req: + self.namespace_set_qos_limits_safe(set_qos_req, context) return pb2.req_status(status=0, error_message=os.strerror(0)) @@ -1290,10 +1371,6 @@ def remove_namespace_from_state(self, nqn, nsid, context): self.gateway_state.remove_namespace_qos(nqn, str(nsid)) except Exception as ex: pass - try: - self.gateway_state.remove_namespace_lb_group(nqn, str(nsid)) - except Exception as ex: - pass try: self.gateway_state.remove_namespace(nqn, str(nsid)) except Exception as ex: @@ -1724,6 +1801,10 @@ def namespace_resize(self, request, context=None): def namespace_delete_safe(self, request, context): """Delete a namespace.""" + return self.namespace_delete_extended(request, True, context) + + def namespace_delete_extended(self, request, delete_bdev, context): + """Delete a namespace with or without its bdev.""" if not request.nsid: errmsg = f"Failure deleting namespace, missing NSID" @@ -1736,7 +1817,7 @@ def namespace_delete_safe(self, request, context): return pb2.req_status(status = errno.EINVAL, error_message = errmsg) peer_msg = self.get_peer_message(context) - self.logger.info(f"Received request to delete namespace {request.nsid} from {request.subsystem_nqn}, context: {context}{peer_msg}") + self.logger.info(f"Received request to delete namespace {request.nsid} from {request.subsystem_nqn}, delete also bdev: {delete_bdev}, context: {context}{peer_msg}") find_ret = self.subsystem_nsid_bdev_and_uuid.find_namespace(request.subsystem_nqn, request.nsid) if not find_ret.uuid: @@ -1755,7 +1836,7 @@ def namespace_delete_safe(self, request, context): self.remove_namespace_from_state(request.subsystem_nqn, request.nsid, context) self.subsystem_nsid_bdev_and_uuid.remove_namespace(request.subsystem_nqn, request.nsid) - if bdev_name: + if delete_bdev and bdev_name: ret_del = self.delete_bdev(bdev_name, peer_msg = peer_msg) if ret_del.status != 0: errmsg = f"Failure deleting namespace {request.nsid} from {request.subsystem_nqn}: {ret_del.error_message}" diff --git a/control/proto/gateway.proto b/control/proto/gateway.proto index f57cd305..64ebe31b 100644 --- a/control/proto/gateway.proto +++ b/control/proto/gateway.proto @@ -122,6 +122,7 @@ message namespace_add_req { optional bool create_image = 8; optional uint64 size = 9; optional bool force = 10; + optional bool can_reuse_bdevs = 11; } message namespace_resize_req { diff --git a/control/server.py b/control/server.py index e7b8df29..01924a38 100644 --- a/control/server.py +++ b/control/server.py @@ -615,7 +615,3 @@ def gateway_rpc_caller(self, requests, is_add_req): else: req = json_format.Parse(val, pb2.delete_listener_req(), ignore_unknown_fields=True) self.gateway_rpc.delete_listener(req) - elif key.startswith(GatewayState.NAMESPACE_LB_GROUP_PREFIX): - if is_add_req: - req = json_format.Parse(val, pb2.namespace_change_load_balancing_group_req(), ignore_unknown_fields=True) - self.gateway_rpc.namespace_change_load_balancing_group(req) diff --git a/control/state.py b/control/state.py index 2cbe73de..8aac6a80 100644 --- a/control/state.py +++ b/control/state.py @@ -33,7 +33,6 @@ class GatewayState(ABC): HOST_PREFIX = "host" + OMAP_KEY_DELIMITER LISTENER_PREFIX = "listener" + OMAP_KEY_DELIMITER NAMESPACE_QOS_PREFIX = "qos" + OMAP_KEY_DELIMITER - NAMESPACE_LB_GROUP_PREFIX = "lbgroup" + OMAP_KEY_DELIMITER def is_key_element_valid(s: str) -> bool: if type(s) != str: @@ -48,12 +47,6 @@ def build_namespace_key(subsystem_nqn: str, nsid) -> str: key += GatewayState.OMAP_KEY_DELIMITER + str(nsid) return key - def build_namespace_lbgroup_key(subsystem_nqn: str, nsid) -> str: - key = GatewayState.NAMESPACE_LB_GROUP_PREFIX + subsystem_nqn - if nsid is not None: - key += GatewayState.OMAP_KEY_DELIMITER + str(nsid) - return key - def build_namespace_qos_key(subsystem_nqn: str, nsid) -> str: key = GatewayState.NAMESPACE_QOS_PREFIX + subsystem_nqn if nsid is not None: @@ -690,50 +683,6 @@ def _update_caller(self, notify_event): notify_event.wait(max(update_time - time.time(), 0)) notify_event.clear() - def namespace_only_lb_group_id_changed(self, old_val, new_val): - old_req = None - new_req = None - try: - old_req = json_format.Parse(old_val, pb2.namespace_add_req(), ignore_unknown_fields=True) - except Exception as ex: - self.logger.exception(f"Got exception parsing {old_val}") - return (False, None) - try: - new_req = json_format.Parse(new_val, pb2.namespace_add_req(), ignore_unknown_fields=True) - except Exception as ex: - self.logger.exeption(f"Got exception parsing {new_val}") - return (False, None) - if not old_req or not new_req: - self.logger.debug(f"Failed to parse requests, old: {old_val} -> {old_req}, new: {new_val} -> {new_req}") - return (False, None) - assert old_req != new_req, f"Something was wrong we shouldn't get identical old and new values ({old_req})" - old_req.anagrpid = new_req.anagrpid - if old_req != new_req: - # Something besides the group id is different - return (False, None) - return (True, new_req.anagrpid) - - def break_namespace_key(self, ns_key: str): - if not ns_key.startswith(GatewayState.NAMESPACE_PREFIX): - self.logger.warning(f"Invalid namespace key \"{ns_key}\", can't find key parts") - return (None, None) - key_end = ns_key[len(GatewayState.NAMESPACE_PREFIX) : ] - key_parts = key_end.split(GatewayState.OMAP_KEY_DELIMITER) - if len(key_parts) != 2: - self.logger.warning(f"Invalid namespace key \"{ns_key}\", can't find key parts") - return (None, None) - if not GatewayUtils.is_valid_nqn(key_parts[0]): - self.logger.warning(f"Invalid NQN \"{key_parts[0]}\" found for namespace key \"{ns_key}\", can't find key parts") - return (None, None) - nqn = key_parts[0] - try: - nsid = int(key_parts[1]) - except Exception as ex: - self.logger.warning(f"Invalid NSID \"{key_parts[1]}\" found for namespace key \"{ns_key}\", can't find key parts") - return (None, None) - - return (nqn, nsid) - def get_str_from_bytes(val): val_str = val.decode() if type(val) == type(b'') else val return val_str @@ -787,46 +736,6 @@ def update(self) -> bool: } grouped_changed = self._group_by_prefix(changed, prefix_list) - # Handle namespace changes in which only the load balancing group id was changed - only_lb_group_changed = [] - ns_prefix = GatewayState.build_namespace_key("nqn", None) - for key in changed.keys(): - if not key.startswith(ns_prefix): - continue - try: - (should_process, new_lb_grp_id) = self.namespace_only_lb_group_id_changed(local_state_dict[key], - omap_state_dict[key]) - if should_process: - assert new_lb_grp_id, "Shouldn't get here with en empty lb group id" - self.logger.debug(f"Found {key} where only the load balancing group id has changed. The new group id is {new_lb_grp_id}") - only_lb_group_changed.insert(0, (key, new_lb_grp_id)) - except Exception as ex: - self.logger.warning("Got exception checking namespace for load balancing group id change") - - for ns_key, new_lb_grp in only_lb_group_changed: - ns_nqn = None - ns_nsid = None - try: - changed.pop(ns_key) - (ns_nqn, ns_nsid) = self.break_namespace_key(ns_key) - except Exception as ex: - self.logger.error(f"Exception removing {ns_key} from {changed}:\n{ex}") - if ns_nqn and ns_nsid: - try: - lbgroup_key = GatewayState.build_namespace_lbgroup_key(ns_nqn, ns_nsid) - req = pb2.namespace_change_load_balancing_group_req(subsystem_nqn=ns_nqn, nsid=ns_nsid, - anagrpid=new_lb_grp) - json_req = json_format.MessageToJson(req, preserving_proto_field_name=True, - including_default_value_fields=True) - added[lbgroup_key] = json_req - except Exception as ex: - self.logger.error(f"Exception formatting change namespace load balancing group request:\n{ex}") - - if len(only_lb_group_changed) > 0: - grouped_changed = self._group_by_prefix(changed, prefix_list) - prefix_list += [GatewayState.NAMESPACE_LB_GROUP_PREFIX] - grouped_added = self._group_by_prefix(added, prefix_list) - # Find OMAP removals removed_keys = local_state_keys - omap_state_keys removed = {key: local_state_dict[key] for key in removed_keys} diff --git a/tests/test_cli_change_lb.py b/tests/test_cli_change_lb.py index 5d2f67a6..eb41ff4d 100644 --- a/tests/test_cli_change_lb.py +++ b/tests/test_cli_change_lb.py @@ -113,9 +113,6 @@ def change_one_namespace_lb_group(caplog, subsys, nsid_to_change, new_group): time.sleep(10) assert f"Changing load balancing group of namespace {nsid_to_change} in {subsys} to {new_group}: Successful" in caplog.text assert f"Received request to change load balancing group for namespace with NSID {nsid_to_change} in {subsys} to {new_group}, context: