Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Run yang validation in db migrator #3102

Merged
merged 11 commits into from
Nov 13, 2024
Merged

Run yang validation in db migrator #3102

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
11 commits
Select commit Hold shift + click to select a range
8269002
Run yang validation in db migrator
ganglyu Dec 27, 2023
cb600ce
Merge branch 'sonic-net:master' into yang_validation
ganglyu Mar 15, 2024
4196bb4
Fix ut error
ganglyu Mar 15, 2024
92fd583
Improve validation logic
ganglyu Mar 21, 2024
7fd1d2c
Replace mark file
ganglyu Mar 25, 2024
8f7cb02
Check missing yang models
ganglyu Apr 2, 2024
7fef5da
Merge branch 'master' into yang_validation
ganglyu Apr 2, 2024
70ab939
Merge branch 'sonic-net:master' into yang_validation
ganglyu Oct 11, 2024
22e52cd
Fix commit error
ganglyu Oct 11, 2024
d758e85
Merge branch 'sonic-net:master' into yang_validation
ganglyu Nov 1, 2024
b5708bf
Merge branch 'sonic-net:master' into yang_validation
ganglyu Nov 12, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
44 changes: 44 additions & 0 deletions scripts/config_validator.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,44 @@
#!/usr/bin/env python3
import json
import argparse
import sonic_yang

from sonic_py_common import logger

YANG_MODELS_DIR = "/usr/local/yang-models"
SYSLOG_IDENTIFIER = 'config_validator'

# Global logger instance
log = logger.Logger(SYSLOG_IDENTIFIER)


def main():
parser = argparse.ArgumentParser()
parser.add_argument('-c',
dest='config',
metavar='config file',
type=str,
required=True,
help='the config file to be validated',
default=None)

args = parser.parse_args()
config_file = args.config
with open(config_file) as fp:
config = json.load(fp)
# Run yang validation
yang_parser = sonic_yang.SonicYang(YANG_MODELS_DIR)
yang_parser.loadYangModel()
try:
yang_parser.loadData(configdbJson=config)
yang_parser.validate_data_tree()
except sonic_yang.SonicYangException as e:
log.log_error("Yang validation failed: " + str(e))
raise
if len(yang_parser.tablesWithOutYang):
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

We could make it 100% in our StarLab DUT.
However, I suspect if we could cover it 100% in PROD devices. It will break the PROD workflow if not fully coverred in PROD.
I prefer we make the check as log_warning and not raise exception right now.

log.log_error("Tables without yang models: " + str(yang_parser.tablesWithOutYang))
raise Exception("Tables without yang models: " + str(yang_parser.tablesWithOutYang))


if __name__ == "__main__":
main()
29 changes: 29 additions & 0 deletions scripts/db_migrator.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@
import sys
import traceback
import re
import subprocess

from sonic_py_common import device_info, logger
from swsscommon.swsscommon import SonicV2Connector, ConfigDBConnector, SonicDBConfig
Expand Down Expand Up @@ -1304,6 +1305,34 @@ def migrate(self):
version = next_version
# Perform common migration ops
self.common_migration_ops()
# Perform yang validation
self.validate()

def validate(self):
config = self.configDB.get_config()
# Fix table key in tuple
for table_name, table in config.items():
new_table = {}
hit = False
for table_key, table_val in table.items():
if isinstance(table_key, tuple):
new_key = "|".join(table_key)
new_table[new_key] = table_val
hit = True
else:
new_table[table_key] = table_val
if hit:
config[table_name] = new_table
config_file = "/tmp/validate.json"
with open(config_file, 'w') as fp:
json.dump(config, fp)
process = subprocess.Popen(["config_validator.py", "-c", config_file])
# Check validation result for unit test
# Check validation result for end to end test
mark_file = "/etc/sonic/mgmt_test_mark"
if os.environ.get("UTILITIES_UNIT_TESTING", "0") == "2" or os.path.exists(mark_file):
ret = process.wait()
assert ret == 0, "Yang validation failed"

def main():
try:
Expand Down
1 change: 1 addition & 0 deletions setup.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -116,6 +116,7 @@
'scripts/buffershow',
'scripts/coredump-compress',
'scripts/configlet',
'scripts/config_validator.py',
'scripts/db_migrator.py',
'scripts/decode-syseeprom',
'scripts/dropcheck',
Expand Down
6 changes: 3 additions & 3 deletions tests/db_migrator_input/config_db/cross_branch_upgrade_to_4_0_3_expected.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,17 +3,17 @@
"VERSION": "version_4_0_3"
},
"FLEX_COUNTER_TABLE|ACL": {
"FLEX_COUNTER_STATUS": "true",
"FLEX_COUNTER_STATUS": "enable",
"FLEX_COUNTER_DELAY_STATUS": "true",
"POLL_INTERVAL": "10000"
},
"FLEX_COUNTER_TABLE|QUEUE": {
"FLEX_COUNTER_STATUS": "true",
"FLEX_COUNTER_STATUS": "enable",
"FLEX_COUNTER_DELAY_STATUS": "true",
"POLL_INTERVAL": "10000"
},
"FLEX_COUNTER_TABLE|PG_WATERMARK": {
"FLEX_COUNTER_STATUS": "false",
"FLEX_COUNTER_STATUS": "disable",
"FLEX_COUNTER_DELAY_STATUS": "true"
}
}
6 changes: 3 additions & 3 deletions tests/db_migrator_input/config_db/cross_branch_upgrade_to_4_0_3_input.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,16 +3,16 @@
"VERSION": "version_1_0_1"
},
"FLEX_COUNTER_TABLE|ACL": {
"FLEX_COUNTER_STATUS": "true",
"FLEX_COUNTER_STATUS": "enable",
"FLEX_COUNTER_DELAY_STATUS": "true",
"POLL_INTERVAL": "10000"
},
"FLEX_COUNTER_TABLE|QUEUE": {
"FLEX_COUNTER_STATUS": "true",
"FLEX_COUNTER_STATUS": "enable",
"FLEX_COUNTER_DELAY_STATUS": "false",
"POLL_INTERVAL": "10000"
},
"FLEX_COUNTER_TABLE|PG_WATERMARK": {
"FLEX_COUNTER_STATUS": "false"
"FLEX_COUNTER_STATUS": "disable"
}
}
4 changes: 0 additions & 4 deletions tests/db_migrator_input/config_db/portchannel-expected.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,28 +1,24 @@
{
"PORTCHANNEL|PortChannel0": {
"admin_status": "up",
"members@": "Ethernet0,Ethernet4",
"min_links": "2",
"mtu": "9100",
"lacp_key": "auto"
},
"PORTCHANNEL|PortChannel1": {
"admin_status": "up",
"members@": "Ethernet8,Ethernet12",
"min_links": "2",
"mtu": "9100",
"lacp_key": "auto"
},
"PORTCHANNEL|PortChannel0123": {
"admin_status": "up",
"members@": "Ethernet16",
"min_links": "1",
"mtu": "9100",
"lacp_key": "auto"
},
"PORTCHANNEL|PortChannel0011": {
"admin_status": "up",
"members@": "Ethernet20,Ethernet24",
"min_links": "2",
"mtu": "9100",
"lacp_key": "auto"
Expand Down
4 changes: 0 additions & 4 deletions tests/db_migrator_input/config_db/portchannel-input.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,25 +1,21 @@
{
"PORTCHANNEL|PortChannel0": {
"admin_status": "up",
"members@": "Ethernet0,Ethernet4",
"min_links": "2",
"mtu": "9100"
},
"PORTCHANNEL|PortChannel1": {
"admin_status": "up",
"members@": "Ethernet8,Ethernet12",
"min_links": "2",
"mtu": "9100"
},
"PORTCHANNEL|PortChannel0123": {
"admin_status": "up",
"members@": "Ethernet16",
"min_links": "1",
"mtu": "9100"
},
"PORTCHANNEL|PortChannel0011": {
"admin_status": "up",
"members@": "Ethernet20,Ethernet24",
"min_links": "2",
"mtu": "9100"
},
Expand Down
10 changes: 9 additions & 1 deletion tests/db_migrator_input/config_db/qos_map_table_expected.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,14 @@
"pfc_to_queue_map": "AZURE",
"tc_to_pg_map": "AZURE",
"tc_to_queue_map": "AZURE"
}
},
"TC_TO_QUEUE_MAP|AZURE": {"0": "0"},
"TC_TO_PRIORITY_GROUP_MAP|AZURE": {"0": "0"},
"MAP_PFC_PRIORITY_TO_QUEUE|AZURE": {"0": "0"},
"DSCP_TO_TC_MAP|AZURE": {"0": "0"},
"PORT|Ethernet0": {"lanes": "0", "speed": "1000"},
"PORT|Ethernet92": {"lanes": "92", "speed": "1000"},
"PORT|Ethernet96": {"lanes": "96", "speed": "1000"},
"PORT|Ethernet100": {"lanes": "100", "speed": "1000"}
}

10 changes: 9 additions & 1 deletion tests/db_migrator_input/config_db/qos_map_table_input.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,5 +27,13 @@
"pfc_to_queue_map": "AZURE",
"tc_to_pg_map": "AZURE",
"tc_to_queue_map": "AZURE"
}
},
"TC_TO_QUEUE_MAP|AZURE": {"0": "0"},
"TC_TO_PRIORITY_GROUP_MAP|AZURE": {"0": "0"},
"MAP_PFC_PRIORITY_TO_QUEUE|AZURE": {"0": "0"},
"DSCP_TO_TC_MAP|AZURE": {"0": "0"},
"PORT|Ethernet0": {"lanes": "0", "speed": "1000"},
"PORT|Ethernet92": {"lanes": "92", "speed": "1000"},
"PORT|Ethernet96": {"lanes": "96", "speed": "1000"},
"PORT|Ethernet100": {"lanes": "100", "speed": "1000"}
}
12 changes: 11 additions & 1 deletion tests/db_migrator_input/config_db/reclaiming-buffer-dynamic-double-pools-expected.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@
"profile": "NULL"
},
"BUFFER_PG|Ethernet8|3-4": {
"profile": "customized_lossless_profile"
"profile": "customized_ingress_lossless_profile"
},
"BUFFER_PG|Ethernet12|0": {
"profile": "ingress_lossy_profile"
Expand Down Expand Up @@ -103,6 +103,11 @@
"BUFFER_PORT_INGRESS_PROFILE_LIST|Ethernet24": {
"profile_list": "ingress_lossless_profile,ingress_lossy_profile"
},
"BUFFER_PROFILE|customized_egress_lossless_profile": {
"dynamic_th": "7",
"pool": "egress_lossless_pool",
"size": "0"
},
"BUFFER_PROFILE|egress_lossless_profile": {
"dynamic_th": "7",
"pool": "egress_lossless_pool",
Expand All @@ -118,6 +123,11 @@
"pool": "ingress_lossless_pool",
"size": "0"
},
"BUFFER_PROFILE|customized_ingress_lossless_profile": {
"dynamic_th": "7",
"pool": "ingress_lossless_pool",
"size": "0"
},
"BUFFER_PROFILE|ingress_lossy_profile": {
"dynamic_th": "3",
"pool": "ingress_lossy_pool",
Expand Down
12 changes: 11 additions & 1 deletion tests/db_migrator_input/config_db/reclaiming-buffer-dynamic-double-pools-input.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
"profile": "NULL"
},
"BUFFER_PG|Ethernet8|3-4": {
"profile": "customized_lossless_profile"
"profile": "customized_ingress_lossless_profile"
},
"BUFFER_PG|Ethernet12|0": {
"profile": "ingress_lossy_profile"
Expand Down Expand Up @@ -55,6 +55,11 @@
"BUFFER_PORT_INGRESS_PROFILE_LIST|Ethernet24": {
"profile_list": "ingress_lossless_profile,ingress_lossy_profile"
},
"BUFFER_PROFILE|customized_egress_lossless_profile": {
"dynamic_th": "7",
"pool": "egress_lossless_pool",
"size": "0"
},
"BUFFER_PROFILE|egress_lossless_profile": {
"dynamic_th": "7",
"pool": "egress_lossless_pool",
Expand All @@ -65,6 +70,11 @@
"pool": "egress_lossy_pool",
"size": "9216"
},
"BUFFER_PROFILE|customized_ingress_lossless_profile": {
"dynamic_th": "7",
"pool": "ingress_lossless_pool",
"size": "0"
},
"BUFFER_PROFILE|ingress_lossless_profile": {
"dynamic_th": "7",
"pool": "ingress_lossless_pool",
Expand Down
12 changes: 11 additions & 1 deletion tests/db_migrator_input/config_db/reclaiming-buffer-dynamic-single-pool-expected.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@
"profile": "NULL"
},
"BUFFER_PG|Ethernet8|3-4": {
"profile": "customized_lossless_profile"
"profile": "customized_ingress_lossless_profile"
},
"BUFFER_PG|Ethernet12|0": {
"profile": "ingress_lossy_profile"
Expand Down Expand Up @@ -99,6 +99,11 @@
"BUFFER_PORT_INGRESS_PROFILE_LIST|Ethernet24": {
"profile_list": "ingress_lossless_profile"
},
"BUFFER_PROFILE|customized_egress_lossless_profile": {
"dynamic_th": "7",
"pool": "egress_lossless_pool",
"size": "0"
},
"BUFFER_PROFILE|egress_lossless_profile": {
"dynamic_th": "7",
"pool": "egress_lossless_pool",
Expand All @@ -109,6 +114,11 @@
"pool": "egress_lossy_pool",
"size": "9216"
},
"BUFFER_PROFILE|customized_ingress_lossless_profile": {
"dynamic_th": "7",
"pool": "ingress_lossless_pool",
"size": "0"
},
"BUFFER_PROFILE|ingress_lossless_profile": {
"dynamic_th": "7",
"pool": "ingress_lossless_pool",
Expand Down
12 changes: 11 additions & 1 deletion tests/db_migrator_input/config_db/reclaiming-buffer-dynamic-single-pool-input.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
"profile": "NULL"
},
"BUFFER_PG|Ethernet8|3-4": {
"profile": "customized_lossless_profile"
"profile": "customized_ingress_lossless_profile"
},
"BUFFER_PG|Ethernet12|0": {
"profile": "ingress_lossy_profile"
Expand Down Expand Up @@ -51,6 +51,11 @@
"BUFFER_PORT_INGRESS_PROFILE_LIST|Ethernet24": {
"profile_list": "ingress_lossless_profile"
},
"BUFFER_PROFILE|customized_egress_lossless_profile": {
"dynamic_th": "7",
"pool": "egress_lossless_pool",
"size": "0"
},
"BUFFER_PROFILE|egress_lossless_profile": {
"dynamic_th": "7",
"pool": "egress_lossless_pool",
Expand All @@ -61,6 +66,11 @@
"pool": "egress_lossy_pool",
"size": "9216"
},
"BUFFER_PROFILE|customized_ingress_lossless_profile": {
"dynamic_th": "7",
"pool": "ingress_lossless_pool",
"size": "0"
},
"BUFFER_PROFILE|ingress_lossless_profile": {
"dynamic_th": "7",
"pool": "ingress_lossless_pool",
Expand Down
Loading