From 90084499e49fc4718aecfe0f251c1b39489ef0c8 Mon Sep 17 00:00:00 2001 From: Maksym Hedeon Date: Fri, 27 Jan 2023 14:55:27 +0200 Subject: [PATCH] sonic-cfggen POC (#15) Don't write to the config DB keys of disabled features Signed-off-by: Maksym Hedeon --- .../data/flags_to_keys_map.yaml | 59 +++++++++ src/sonic-config-engine/setup.py | 3 +- src/sonic-config-engine/sonic-cfggen | 14 +- .../sonic_py_common/cleanup_common.py | 125 ++++++++++++++++++ 4 files changed, 196 insertions(+), 5 deletions(-) create mode 100644 src/sonic-config-engine/data/flags_to_keys_map.yaml create mode 100644 src/sonic-py-common/sonic_py_common/cleanup_common.py diff --git a/src/sonic-config-engine/data/flags_to_keys_map.yaml b/src/sonic-config-engine/data/flags_to_keys_map.yaml new file mode 100644 index 000000000000..5ecc26489663 --- /dev/null +++ b/src/sonic-config-engine/data/flags_to_keys_map.yaml @@ -0,0 +1,59 @@ +INCLUDE_DATABASE: + - '' +INCLUDE_DHCP_RELAY: + - '' +INCLUDE_FRR_BFD: + - '' +INCLUDE_FRR_BGP: + - BGP_NEIGHBOR + - DEVICE_METADATA.localhost.bgp_asn +INCLUDE_FRR_OSPF: + - '' +INCLUDE_FRR_PBR: + - '' +INCLUDE_FRR_VRRP: + - '' +INCLUDE_ICCPD: + - '' +INCLUDE_KUBERNETES: + - '' +INCLUDE_LLDP: + - '' +INCLUDE_MACSEC: + - '' +INCLUDE_MGMT_FRAMEWORK: + - '' +INCLUDE_MUX: + - '' +INCLUDE_NAT: + - '' +INCLUDE_NTP: + - '' +INCLUDE_P4RT: + - '' +INCLUDE_PMON: + - '' +INCLUDE_RADIUS: + - '' +INCLUDE_RESTAPI: + - '' +INCLUDE_ROUTER_ADVERTISER: + - '' +INCLUDE_ROUTING_STACK: + - '' +INCLUDE_SFLOW: + - '' +INCLUDE_SNMP: + - '' +INCLUDE_SSH: + - '' +INCLUDE_SWSS: + - '' +INCLUDE_SYNCD: + - '' +INCLUDE_SYSLOG: + - '' +INCLUDE_SYSTEM_TELEMETRY: + - '' +INCLUDE_TEAMD: + - '' \ No newline at end of file diff --git a/src/sonic-config-engine/setup.py b/src/sonic-config-engine/setup.py index c43c47ef3a6a..47c5f152b0bd 100644 --- a/src/sonic-config-engine/setup.py +++ b/src/sonic-config-engine/setup.py @@ -63,7 +63,8 @@ ], install_requires = dependencies, data_files = [ - ('/usr/share/sonic/templates', glob.glob('data/*')), + ('/usr/share/sonic/templates', glob.glob('data/*.j2')), + ('/etc/sonic', glob.glob('data/flags_to_keys_map.yaml')) ], setup_requires= [ 'pytest-runner', diff --git a/src/sonic-config-engine/sonic-cfggen b/src/sonic-config-engine/sonic-cfggen index 0e53e6309325..f087d03cc12f 100755 --- a/src/sonic-config-engine/sonic-cfggen +++ b/src/sonic-config-engine/sonic-cfggen @@ -32,10 +32,9 @@ from functools import partial from minigraph import minigraph_encoder, parse_xml, parse_device_desc_xml, parse_asic_sub_role, parse_asic_switch_type from portconfig import get_port_config, get_breakout_mode from sonic_py_common.multi_asic import get_asic_id_from_name, get_asic_device_id -from sonic_py_common import device_info +from sonic_py_common import device_info, cleanup_common from swsscommon.swsscommon import ConfigDBConnector, SonicDBConfig, ConfigDBPipeConnector - PY3x = sys.version_info >= (3, 0) # TODO: Remove STR_TYPE, FILE_TYPE once SONiC moves to Python 3.x @@ -187,8 +186,7 @@ TODO(taoyl): Current version of config db only supports BGP admin states. data[table][new_key] = data[table].pop(key) return data - -def deep_update(dst, src): +def deep_update(dst, src, cleanup = False): """ Deep update of dst dict with contest of src dict""" pending_nodes = [(dst, src)] while len(pending_nodes) > 0: @@ -199,6 +197,12 @@ def deep_update(dst, src): pending_nodes.append((node, value)) else: d[key] = value + + if cleanup: + disabled_flags = list(cleanup_common.read_metadata_conifg(key_pattern="INCLUDE_", value_pattern="^n$")) + keys_list = cleanup_common.read_map(disabled_flags) + cleanup_common.clean_cfggen_dict(dst, keys_list) + return dst # sort_data is required as it is being imported by config/config_mgmt module in sonic_utilities @@ -428,6 +432,7 @@ def main(): SonicDBConfig.load_sonic_global_db_config(namespace=args.namespace) configdb = ConfigDBPipeConnector(use_unix_socket_path=True, namespace=args.namespace, **db_kwargs) + deep_update(data, data, cleanup=True) configdb.connect(False) configdb.mod_config(FormatConverter.output_to_db(data)) @@ -436,6 +441,7 @@ def main(): if args.preset is not None: data = generate_sample_config(data, args.preset) + deep_update(data, data, cleanup=True) print(json.dumps(FormatConverter.to_serialized(data), indent=4, cls=minigraph_encoder)) diff --git a/src/sonic-py-common/sonic_py_common/cleanup_common.py b/src/sonic-py-common/sonic_py_common/cleanup_common.py new file mode 100644 index 000000000000..6a30e3ad5dab --- /dev/null +++ b/src/sonic-py-common/sonic_py_common/cleanup_common.py @@ -0,0 +1,125 @@ +import yaml +import re +import os + +PATH_BUILD_METADATA = "/etc/sonic/build_metadata.yaml" +PATH_CONFIG = "/sonic/rules/config" +PATH_CONFIG_USER = PATH_CONFIG + ".user" +PATH_FLAG_TO_KEY_MAP = "/etc/sonic/flags_to_keys_map.yaml" +PATH_DIST_FLAG_TO_KEY_MAP = os.path.dirname(os.path.abspath(__file__)) + "/.." + PATH_FLAG_TO_KEY_MAP +PATH_CFGGEN_KEYS_CLEANER = "/tmp/keys_to_clean" + +def read_rules_config(path = None): + ''' + Read config file (e.g. rules/config) and + return in dictionary format + ''' + + if path is None: + path = PATH_CONFIG + + if os.path.exists(path) is False: + return {} + + with open(path) as file: + file = file.read().split("\n") + + dict = {} + + for line in file: + # non empty line and not comment + if re.search("^$", line) is None and \ + re.search("^#", line) is None: + line = line.replace("?=", "=", 1) + # clear from spaces + line = line.replace(" ", "") + key_value = line.split("=") + dict[key_value[0]] = key_value[1] + + return dict + +def read_metadata_conifg(key_pattern = "", value_pattern = "", path = None): + ''' + pattern: default is any string. "^$" - empty string + "word" - substring, "^word$" - definite string + + return value: dict from metadata + ''' + + if path is None: + path = PATH_BUILD_METADATA + + # Check if path exists + if os.path.exists(path) is True: + with open(path) as file: + config = yaml.safe_load(file) + config = config['Configuration'] + # probably this is build unit test + # read rules/config from sonic-buildimage + else: + config = {} + config = read_rules_config(PATH_CONFIG) + config_user = read_rules_config(PATH_CONFIG_USER) + if config_user is not None: + config.update(config_user) + + for key in list(config): + if re.search(key_pattern, key) is None or \ + (re.search(key_pattern, key) is not None and \ + re.search(value_pattern, config[key]) is None): + del config[key] + + return config + +def read_map(keys, path = None): + ''' + Get list of values from key list + ''' + + if path is None: + path = PATH_DIST_FLAG_TO_KEY_MAP + + if keys is None or os.path.exists(path) is False: + return [] + + with open(path) as file: + map = yaml.safe_load(file) + + values = [] + for key in keys: + values.extend(map[key]) + + # remove empty strings + values = list(filter(None, values)) + + return values + +def rm_top_nested_key(dict, key): + ''' + provide dot separated key string to go through path and remove top nested key from dictionary + e.g. DEVICE_METADATA.localhost.bgp_asn + e.g. BGP_NEIGHBOR + ''' + + if "." in key: + keys_splitted = key.split(".", 1) + if keys_splitted[0] in dict: + dict = dict[keys_splitted[0]] + key = keys_splitted[1] + rm_top_nested_key(dict, key) + else: + dict.pop(key, None) + +def clean_cfggen_dict(dict, keys): + ''' + Helper function for sonic-cfggen. Clear keys for + disabled build flags (e.g. for INCLUDE_FRR_BGP). + dict: sonic-cfggen dictionary + keys: dot separated keys (see rm_top_nested_key) + ''' + + if keys is None: + return + + for key in keys: + rm_top_nested_key(dict, key)