From 07e39dc6a8f5d92e889ac76af7c805f489c76675 Mon Sep 17 00:00:00 2001 From: ghooo Date: Mon, 18 Oct 2021 12:35:13 -0700 Subject: [PATCH 1/2] [generic-config-updater] More logging --- generic_config_updater/generic_updater.py | 61 ++++++++++++++++++++++- 1 file changed, 60 insertions(+), 1 deletion(-) diff --git a/generic_config_updater/generic_updater.py b/generic_config_updater/generic_updater.py index 8205a20b27..cd18f87783 100644 --- a/generic_config_updater/generic_updater.py +++ b/generic_config_updater/generic_updater.py @@ -86,64 +86,123 @@ def apply(self, patch): class ConfigReplacer: def __init__(self, patch_applier=None, config_wrapper=None, patch_wrapper=None): + self.logger = genericUpdaterLogging.get_logger(title="Config Replacer") self.patch_applier = patch_applier if patch_applier is not None else PatchApplier() self.config_wrapper = config_wrapper if config_wrapper is not None else ConfigWrapper() self.patch_wrapper = patch_wrapper if patch_wrapper is not None else PatchWrapper() def replace(self, target_config): + print_to_console=True + self.logger.log_notice("Config replacement starting.", print_to_console) + self.logger.log_notice(f"Target config length: {len(json.dumps(target_config))}.", print_to_console) + + self.logger.log_notice("Validating target config according to YANG models.", print_to_console) if not(self.config_wrapper.validate_config_db_config(target_config)): raise ValueError(f"The given target config is not valid") + self.logger.log_notice("Getting current config db.", print_to_console) old_config = self.config_wrapper.get_config_db_as_json() + + self.logger.log_notice("Generating patch between target config and current config db.", print_to_console) patch = self.patch_wrapper.generate_patch(old_config, target_config) + self.logger.log_debug(f"Generated patch: {patch}.", print_to_console) # debug since the patch will printed again in 'patch_applier.apply' + self.logger.log_notice("Applying patch using 'Patch Applier'.", print_to_console) self.patch_applier.apply(patch) + self.logger.log_notice("Verifying config replacement is reflected on ConfigDB.", print_to_console) new_config = self.config_wrapper.get_config_db_as_json() if not(self.patch_wrapper.verify_same_json(target_config, new_config)): raise GenericConfigUpdaterError(f"After replacing config, there is still some parts not updated") + self.logger.log_notice("Config replacement completed.", print_to_console) + class FileSystemConfigRollbacker: def __init__(self, checkpoints_dir=CHECKPOINTS_DIR, config_replacer=None, config_wrapper=None): + self.logger = genericUpdaterLogging.get_logger(title="Config Rollbacker") self.checkpoints_dir = checkpoints_dir self.config_replacer = config_replacer if config_replacer is not None else ConfigReplacer() self.config_wrapper = config_wrapper if config_wrapper is not None else ConfigWrapper() def rollback(self, checkpoint_name): + print_to_console=True + self.logger.log_notice("Config rollbacking starting.", print_to_console) + self.logger.log_notice(f"Checkpoint name: {checkpoint_name}.", print_to_console) + + self.logger.log_notice(f"Verifying '{checkpoint_name}' exists.", print_to_console) if not self._check_checkpoint_exists(checkpoint_name): raise ValueError(f"Checkpoint '{checkpoint_name}' does not exist") + self.logger.log_notice(f"Loading checkpoint into memory.", print_to_console) target_config = self._get_checkpoint_content(checkpoint_name) + self.logger.log_notice(f"Replacing config using 'Config Replacer'.", print_to_console) self.config_replacer.replace(target_config) + self.logger.log_notice("Config rollbacking completed.", print_to_console) + def checkpoint(self, checkpoint_name): + print_to_console=True + self.logger.log_notice("Config checkpoint starting.", print_to_console) + self.logger.log_notice(f"Checkpoint name: {checkpoint_name}.", print_to_console) + + self.logger.log_notice("Getting current config db.", print_to_console) json_content = self.config_wrapper.get_config_db_as_json() + # if current config are not valid, we might not be able to rollback to it. So fail early by not taking checkpoint at all. + self.logger.log_notice("Validating current config according to YANG models.", print_to_console) if not self.config_wrapper.validate_config_db_config(json_content): raise ValueError(f"Running configs on the device are not valid.") + self.logger.log_notice("Getting checkpoint full-path.", print_to_console) path = self._get_checkpoint_full_path(checkpoint_name) + self.logger.log_notice("Ensuring checkpoint directory exist.", print_to_console) self._ensure_checkpoints_dir_exists() + self.logger.log_notice(f"Saving config db content to {path}.", print_to_console) self._save_json_file(path, json_content) + self.logger.log_notice("Config checkpoint completed.", print_to_console) + def list_checkpoints(self): + print_to_console=True + self.logger.log_info("Listing checkpoints starting.", print_to_console) + + self.logger.log_info(f"Verifying checkpoints directory '{self.checkpoints_dir}' exists.", print_to_console) if not self._checkpoints_dir_exist(): + self.logger.log_info("Checkpoints directory is empty, returning empty checkpoints list.", print_to_console) return [] - return self._get_checkpoint_names() + self.logger.log_info("Getting checkpoints in checkpoints directory.", print_to_console) + checkpoint_names = self._get_checkpoint_names() + + checkpoints_len = len(checkpoint_names) + self.logger.log_info(f"Found {checkpoints_len} checkpoint{'s' if checkpoints_len != 1 else ''}{':' if checkpoints_len > 0 else '.'}", print_to_console) + for checkpoint_name in checkpoint_names: + self.logger.log_info(f" * {checkpoint_name}", print_to_console) + + self.logger.log_info("Listing checkpoints completed.", print_to_console) + + return checkpoint_names def delete_checkpoint(self, checkpoint_name): + print_to_console=True + self.logger.log_notice("Deleting checkpoint starting.", print_to_console) + self.logger.log_notice(f"Checkpoint name: {checkpoint_name}.", print_to_console) + + self.logger.log_notice(f"Checking checkpoint exists.", print_to_console) if not self._check_checkpoint_exists(checkpoint_name): raise ValueError(f"Checkpoint '{checkpoint_name}' does not exist") + self.logger.log_notice(f"Deleting checkpoint.", print_to_console) self._delete_checkpoint(checkpoint_name) + self.logger.log_notice("Deleting checkpoint completed.", print_to_console) + def _ensure_checkpoints_dir_exists(self): os.makedirs(self.checkpoints_dir, exist_ok=True) From 94e4e919285eaadfb01618ef7bd4fafac01217d1 Mon Sep 17 00:00:00 2001 From: ghooo Date: Fri, 29 Oct 2021 11:23:24 -0700 Subject: [PATCH 2/2] adding print_all_to_console to TitledLogger --- generic_config_updater/generic_updater.py | 107 ++++++++++------------ generic_config_updater/gu_common.py | 9 +- 2 files changed, 55 insertions(+), 61 deletions(-) diff --git a/generic_config_updater/generic_updater.py b/generic_config_updater/generic_updater.py index cd18f87783..9ad491c662 100644 --- a/generic_config_updater/generic_updater.py +++ b/generic_config_updater/generic_updater.py @@ -32,176 +32,169 @@ def __init__(self, changeapplier=None, config_wrapper=None, patch_wrapper=None): - self.logger = genericUpdaterLogging.get_logger(title="Patch Applier") + self.logger = genericUpdaterLogging.get_logger(title="Patch Applier", print_all_to_console=True) self.config_wrapper = config_wrapper if config_wrapper is not None else ConfigWrapper() self.patch_wrapper = patch_wrapper if patch_wrapper is not None else PatchWrapper() self.patchsorter = patchsorter if patchsorter is not None else PatchSorter(self.config_wrapper, self.patch_wrapper) self.changeapplier = changeapplier if changeapplier is not None else ChangeApplier() def apply(self, patch): - print_to_console=True - self.logger.log_notice("Patch application starting.", print_to_console) - self.logger.log_notice(f"Patch: {patch}", print_to_console) + self.logger.log_notice("Patch application starting.") + self.logger.log_notice(f"Patch: {patch}") # validate patch is only updating tables with yang models - self.logger.log_notice("Validating patch is not making changes to tables without YANG models.", print_to_console) + self.logger.log_notice("Validating patch is not making changes to tables without YANG models.") if not(self.patch_wrapper.validate_config_db_patch_has_yang_models(patch)): raise ValueError(f"Given patch is not valid because it has changes to tables without YANG models") # Get old config - self.logger.log_notice("Getting current config db.", print_to_console) + self.logger.log_notice("Getting current config db.") old_config = self.config_wrapper.get_config_db_as_json() # Generate target config - self.logger.log_notice("Simulating the target full config after applying the patch.", print_to_console) + self.logger.log_notice("Simulating the target full config after applying the patch.") target_config = self.patch_wrapper.simulate_patch(patch, old_config) # Validate target config - self.logger.log_notice("Validating target config according to YANG models.", print_to_console) + self.logger.log_notice("Validating target config according to YANG models.") if not(self.config_wrapper.validate_config_db_config(target_config)): raise ValueError(f"Given patch is not valid because it will result in an invalid config") # Generate list of changes to apply - self.logger.log_notice("Sorting patch updates.", print_to_console) + self.logger.log_notice("Sorting patch updates.") changes = self.patchsorter.sort(patch) changes_len = len(changes) self.logger.log_notice(f"The patch was sorted into {changes_len} " \ - f"change{'s' if changes_len != 1 else ''}{':' if changes_len > 0 else '.'}", - print_to_console) + f"change{'s' if changes_len != 1 else ''}{':' if changes_len > 0 else '.'}") for change in changes: - self.logger.log_notice(f" * {change}", print_to_console) + self.logger.log_notice(f" * {change}") # Apply changes in order - self.logger.log_notice("Applying changes in order.", print_to_console) + self.logger.log_notice("Applying changes in order.") for change in changes: self.changeapplier.apply(change) # Validate config updated successfully - self.logger.log_notice("Verifying patch updates are reflected on ConfigDB.", print_to_console) + self.logger.log_notice("Verifying patch updates are reflected on ConfigDB.") new_config = self.config_wrapper.get_config_db_as_json() if not(self.patch_wrapper.verify_same_json(target_config, new_config)): raise GenericConfigUpdaterError(f"After applying patch to config, there are still some parts not updated") - self.logger.log_notice("Patch application completed.", print_to_console) + self.logger.log_notice("Patch application completed.") class ConfigReplacer: def __init__(self, patch_applier=None, config_wrapper=None, patch_wrapper=None): - self.logger = genericUpdaterLogging.get_logger(title="Config Replacer") + self.logger = genericUpdaterLogging.get_logger(title="Config Replacer", print_all_to_console=True) self.patch_applier = patch_applier if patch_applier is not None else PatchApplier() self.config_wrapper = config_wrapper if config_wrapper is not None else ConfigWrapper() self.patch_wrapper = patch_wrapper if patch_wrapper is not None else PatchWrapper() def replace(self, target_config): - print_to_console=True - self.logger.log_notice("Config replacement starting.", print_to_console) - self.logger.log_notice(f"Target config length: {len(json.dumps(target_config))}.", print_to_console) + self.logger.log_notice("Config replacement starting.") + self.logger.log_notice(f"Target config length: {len(json.dumps(target_config))}.") - self.logger.log_notice("Validating target config according to YANG models.", print_to_console) + self.logger.log_notice("Validating target config according to YANG models.") if not(self.config_wrapper.validate_config_db_config(target_config)): raise ValueError(f"The given target config is not valid") - self.logger.log_notice("Getting current config db.", print_to_console) + self.logger.log_notice("Getting current config db.") old_config = self.config_wrapper.get_config_db_as_json() - self.logger.log_notice("Generating patch between target config and current config db.", print_to_console) + self.logger.log_notice("Generating patch between target config and current config db.") patch = self.patch_wrapper.generate_patch(old_config, target_config) - self.logger.log_debug(f"Generated patch: {patch}.", print_to_console) # debug since the patch will printed again in 'patch_applier.apply' + self.logger.log_debug(f"Generated patch: {patch}.") # debug since the patch will printed again in 'patch_applier.apply' - self.logger.log_notice("Applying patch using 'Patch Applier'.", print_to_console) + self.logger.log_notice("Applying patch using 'Patch Applier'.") self.patch_applier.apply(patch) - self.logger.log_notice("Verifying config replacement is reflected on ConfigDB.", print_to_console) + self.logger.log_notice("Verifying config replacement is reflected on ConfigDB.") new_config = self.config_wrapper.get_config_db_as_json() if not(self.patch_wrapper.verify_same_json(target_config, new_config)): raise GenericConfigUpdaterError(f"After replacing config, there is still some parts not updated") - self.logger.log_notice("Config replacement completed.", print_to_console) + self.logger.log_notice("Config replacement completed.") class FileSystemConfigRollbacker: def __init__(self, checkpoints_dir=CHECKPOINTS_DIR, config_replacer=None, config_wrapper=None): - self.logger = genericUpdaterLogging.get_logger(title="Config Rollbacker") + self.logger = genericUpdaterLogging.get_logger(title="Config Rollbacker", print_all_to_console=True) self.checkpoints_dir = checkpoints_dir self.config_replacer = config_replacer if config_replacer is not None else ConfigReplacer() self.config_wrapper = config_wrapper if config_wrapper is not None else ConfigWrapper() def rollback(self, checkpoint_name): - print_to_console=True - self.logger.log_notice("Config rollbacking starting.", print_to_console) - self.logger.log_notice(f"Checkpoint name: {checkpoint_name}.", print_to_console) + self.logger.log_notice("Config rollbacking starting.") + self.logger.log_notice(f"Checkpoint name: {checkpoint_name}.") - self.logger.log_notice(f"Verifying '{checkpoint_name}' exists.", print_to_console) + self.logger.log_notice(f"Verifying '{checkpoint_name}' exists.") if not self._check_checkpoint_exists(checkpoint_name): raise ValueError(f"Checkpoint '{checkpoint_name}' does not exist") - self.logger.log_notice(f"Loading checkpoint into memory.", print_to_console) + self.logger.log_notice(f"Loading checkpoint into memory.") target_config = self._get_checkpoint_content(checkpoint_name) - self.logger.log_notice(f"Replacing config using 'Config Replacer'.", print_to_console) + self.logger.log_notice(f"Replacing config using 'Config Replacer'.") self.config_replacer.replace(target_config) - self.logger.log_notice("Config rollbacking completed.", print_to_console) + self.logger.log_notice("Config rollbacking completed.") def checkpoint(self, checkpoint_name): - print_to_console=True - self.logger.log_notice("Config checkpoint starting.", print_to_console) - self.logger.log_notice(f"Checkpoint name: {checkpoint_name}.", print_to_console) + self.logger.log_notice("Config checkpoint starting.") + self.logger.log_notice(f"Checkpoint name: {checkpoint_name}.") - self.logger.log_notice("Getting current config db.", print_to_console) + self.logger.log_notice("Getting current config db.") json_content = self.config_wrapper.get_config_db_as_json() # if current config are not valid, we might not be able to rollback to it. So fail early by not taking checkpoint at all. - self.logger.log_notice("Validating current config according to YANG models.", print_to_console) + self.logger.log_notice("Validating current config according to YANG models.") if not self.config_wrapper.validate_config_db_config(json_content): raise ValueError(f"Running configs on the device are not valid.") - self.logger.log_notice("Getting checkpoint full-path.", print_to_console) + self.logger.log_notice("Getting checkpoint full-path.") path = self._get_checkpoint_full_path(checkpoint_name) - self.logger.log_notice("Ensuring checkpoint directory exist.", print_to_console) + self.logger.log_notice("Ensuring checkpoint directory exist.") self._ensure_checkpoints_dir_exists() - self.logger.log_notice(f"Saving config db content to {path}.", print_to_console) + self.logger.log_notice(f"Saving config db content to {path}.") self._save_json_file(path, json_content) - self.logger.log_notice("Config checkpoint completed.", print_to_console) + self.logger.log_notice("Config checkpoint completed.") def list_checkpoints(self): - print_to_console=True - self.logger.log_info("Listing checkpoints starting.", print_to_console) + self.logger.log_info("Listing checkpoints starting.") - self.logger.log_info(f"Verifying checkpoints directory '{self.checkpoints_dir}' exists.", print_to_console) + self.logger.log_info(f"Verifying checkpoints directory '{self.checkpoints_dir}' exists.") if not self._checkpoints_dir_exist(): - self.logger.log_info("Checkpoints directory is empty, returning empty checkpoints list.", print_to_console) + self.logger.log_info("Checkpoints directory is empty, returning empty checkpoints list.") return [] - self.logger.log_info("Getting checkpoints in checkpoints directory.", print_to_console) + self.logger.log_info("Getting checkpoints in checkpoints directory.") checkpoint_names = self._get_checkpoint_names() checkpoints_len = len(checkpoint_names) - self.logger.log_info(f"Found {checkpoints_len} checkpoint{'s' if checkpoints_len != 1 else ''}{':' if checkpoints_len > 0 else '.'}", print_to_console) + self.logger.log_info(f"Found {checkpoints_len} checkpoint{'s' if checkpoints_len != 1 else ''}{':' if checkpoints_len > 0 else '.'}") for checkpoint_name in checkpoint_names: - self.logger.log_info(f" * {checkpoint_name}", print_to_console) + self.logger.log_info(f" * {checkpoint_name}") - self.logger.log_info("Listing checkpoints completed.", print_to_console) + self.logger.log_info("Listing checkpoints completed.") return checkpoint_names def delete_checkpoint(self, checkpoint_name): - print_to_console=True - self.logger.log_notice("Deleting checkpoint starting.", print_to_console) - self.logger.log_notice(f"Checkpoint name: {checkpoint_name}.", print_to_console) + self.logger.log_notice("Deleting checkpoint starting.") + self.logger.log_notice(f"Checkpoint name: {checkpoint_name}.") - self.logger.log_notice(f"Checking checkpoint exists.", print_to_console) + self.logger.log_notice(f"Checking checkpoint exists.") if not self._check_checkpoint_exists(checkpoint_name): raise ValueError(f"Checkpoint '{checkpoint_name}' does not exist") - self.logger.log_notice(f"Deleting checkpoint.", print_to_console) + self.logger.log_notice(f"Deleting checkpoint.") self._delete_checkpoint(checkpoint_name) - self.logger.log_notice("Deleting checkpoint completed.", print_to_console) + self.logger.log_notice("Deleting checkpoint completed.") def _ensure_checkpoints_dir_exists(self): os.makedirs(self.checkpoints_dir, exist_ok=True) diff --git a/generic_config_updater/gu_common.py b/generic_config_updater/gu_common.py index be1f6e5db7..1f213e4371 100644 --- a/generic_config_updater/gu_common.py +++ b/generic_config_updater/gu_common.py @@ -695,15 +695,16 @@ def _get_model(self, model, name): return None class TitledLogger(logger.Logger): - def __init__(self, syslog_identifier, title, verbose): + def __init__(self, syslog_identifier, title, verbose, print_all_to_console): super().__init__(syslog_identifier) self._title = title if verbose: self.set_min_log_priority_debug() + self.print_all_to_console = print_all_to_console def log(self, priority, msg, also_print_to_console=False): combined_msg = f"{self._title}: {msg}" - super().log(priority, combined_msg, also_print_to_console) + super().log(priority, combined_msg, self.print_all_to_console or also_print_to_console) class GenericUpdaterLogging: def __init__(self): @@ -712,7 +713,7 @@ def __init__(self): def set_verbose(self, verbose): self._verbose = verbose - def get_logger(self, title): - return TitledLogger(SYSLOG_IDENTIFIER, title, self._verbose) + def get_logger(self, title, print_all_to_console=False): + return TitledLogger(SYSLOG_IDENTIFIER, title, self._verbose, print_all_to_console) genericUpdaterLogging = GenericUpdaterLogging()