[yang-models]: Add yang extension to control traslation configDB<->ya…

…ng from YANG models.

Changes:
1.) Added yang extensions in YANG models.
2.) Changed the translation and rev translation code to depend on yang extensions.
3.) Load yang models in test code similar to dev code.
4.) Minor changes in test files.
Signed-off-by: Praveen Chaudhary pchaudhary@linkedin.com

src/sonic-yang-mgmt/_sonic_yang_ext.py

@@ -19,11 +19,15 @@
 def loadYangModel(self):
 
     try:
-        yangDir = self.yang_dir
-        self.yangFiles = glob(yangDir +"/*.yang")
+        # get all files
+        self.yangFiles = glob(self.yang_dir +"/*.yang")
+        # load yang modules
         for file in self.yangFiles:
-            if (self.load_schema_module(file) == False):
-                return False
+            m = self.load_schema_module(file)
+            if m is not None:
+                self.logInFile("module: {} is loaded successfully".format(m.name()))
+            else:
+                raise(Exception("Could not load module {}".format(file)))
 
         # keep only modules name in self.yangFiles
         self.yangFiles = [f.split('/')[-1] for f in self.yangFiles]
@@ -53,6 +57,7 @@ def loadJsonYangModel(self):
             if m is not None:
                 xml = m.print_mem(ly.LYD_JSON, ly.LYP_FORMAT)
                 self.yJson.append(parse(xml))
+                self.logInFile("Parsed Json for {}".format(m.name()))
     except Exception as e:
         print('JSON conversion for yang models failed')
         raise e
@@ -70,7 +75,8 @@ def createDBTableToModuleMap(self):
     for j in self.yJson:
         # get module name
         moduleName = j['module']['@name']
-        if "sonic-head" in moduleName or "sonic-common" in moduleName:
+        # topLevelContainer does not exist in sonic-head and sonic-extension.
+        if "sonic-head" in moduleName or "sonic-extension" in moduleName:
             continue;
         # get all top level container
         topLevelContainer = j['module']['container']
@@ -248,35 +254,25 @@ def yangConvert(val):
 """
 def xlateList(self, model, yang, config, table):
 
-    # TODO: define a keyExt dict as of now, but we should be able to extract
-    # this from YANG model extentions.
-    keyExt = {
-        "VLAN_INTERFACE_LIST": "^(Vlan[a-zA-Z0-9_-]+)$",
-        "VLAN_LIST": "^(Vlan[a-zA-Z0-9_-]+)$",
-        "VLAN_INTERFACE_IPPREFIX_LIST": "^(Vlan[a-zA-Z0-9_-]+)\|([a-fA-F0-9:./]+$)",
-        "VLAN_MEMBER_LIST": "^(Vlan[a-zA-Z0-9-_]+)\|(Ethernet[0-9]+)$",
-        "ACL_RULE_LIST": "^([a-zA-Z0-9_-]+)\|([a-zA-Z0-9_-]+)$",
-        "ACL_TABLE_LIST": "^([a-zA-Z0-9-_]+)$",
-        "INTERFACE_LIST": "^(Ethernet[0-9]+)$",
-        "INTERFACE_IPPREFIX_LIST": "^(Ethernet[0-9]+)\|([a-fA-F0-9:./]+)$",
-        "PORT_LIST": "^(Ethernet[0-9]+)$",
-        "LOOPBACK_INTERFACE_LIST": "^([a-zA-Z0-9-_]+)$",
-        "LOOPBACK_INTERFACE_IPPREFIX_LIST": "^([a-zA-Z0-9-_]+)\|([a-fA-F0-9:./]+)$",
-    }
     #create a dict to map each key under primary key with a dict yang model.
     #This is done to improve performance of mapping from values of TABLEs in
     #config DB to leaf in YANG LIST.
 
     leafDict = self.createLeafDict(model)
 
-    keyRegEx = keyExt[model['@name']]
+    # fetch regex from YANG models.
+    keyRegEx = model['ext:key-regex-configdb-to-yang']['@value']
+    # seperator `|` has special meaning in regex, so change it appropriately.
+    keyRegEx = re.sub('\|', '\\|', keyRegEx)
+
     # get keys from YANG model list itself
     listKeys = model['key']['@value']
 
     for pkey in config.keys():
         try:
             vKey = None
-            self.logInFile("xlateList Extract pkey {}".format(pkey))
+            self.logInFile("xlateList Extract pkey:{} regex:{} keyList:{}\
+                ".format(pkey, keyRegEx, listKeys))
             # Find and extracts key from each dict in config
             keyDict = self.extractKey(pkey, listKeys, keyRegEx)
             # fill rest of the values in keyDict
@@ -290,8 +286,6 @@ def xlateList(self, model, yang, config, table):
 
         except Exception as e:
             self.logInFile("xlateList Exception {}".format(e))
-            self.logInFile("Exception while Config DB --> YANG: pkey:{}, "\
-            "vKey:{}, value: {}".format(pkey, vKey, config[pkey].get(vKey)))
             # with multilist, we continue matching other keys.
             continue
 
@@ -333,6 +327,7 @@ def xlateContainer(self, model, yang, config, table):
                 del yang[modelList['@name']]
 
     if len(configC):
+        self.logInFile("Alert: Remaining keys in Config", obj=configC, json=True)
         raise(Exception("All Keys are not parsed in {}".format(table)))
 
     return
@@ -421,24 +416,9 @@ def revYangConvert(val):
 """
 def revXlateList(self, model, yang, config, table):
 
-    # TODO: define a keyExt dict as of now, but we should be able to
-    # extract this from YANG model extentions.
-    keyExt = {
-        "VLAN_INTERFACE_IPPREFIX_LIST": "<vlan_name>|<ip-prefix>",
-        "VLAN_INTERFACE_LIST": "<vlan_name>",
-        "VLAN_MEMBER_LIST": "<vlan_name>|<port>",
-        "VLAN_LIST": "<vlan_name>",
-        "ACL_RULE_LIST": "<ACL_TABLE_NAME>|<RULE_NAME>",
-        "ACL_TABLE_LIST": "<ACL_TABLE_NAME>",
-        "INTERFACE_LIST": "<port_name>",
-        "INTERFACE_IPPREFIX_LIST": "<port_name>|<ip-prefix>",
-        "LOOPBACK_INTERFACE_LIST": "<loopback_interface_name>",
-        "LOOPBACK_INTERFACE_IPPREFIX_LIST": "<loopback_interface_name>|<ip-prefix>",
-        "PORT_LIST": "<port_name>"
-
-    }
-
-    keyRegEx = keyExt[model['@name']]
+    # fetch regex from YANG models
+    keyRegEx = model['ext:key-regex-yang-to-configdb']['@value']
+
     # create a dict to map each key under primary key with a dict yang model.
     # This is done to improve performance of mapping from values of TABLEs in
     # config DB to leaf in YANG LIST.
@@ -448,7 +428,9 @@ def revXlateList(self, model, yang, config, table):
     if "_LIST" in model['@name']:
         for entry in yang:
             # create key of config DB table
+            self.logInFile("revXlateList regex:{}".format(keyRegEx))
             pkey, pkeydict = self.createKey(entry, keyRegEx)
+            self.logInFile("revXlateList pkey:{}".format(pkey))
             config[pkey]= dict()
             # fill rest of the entries
             for key in entry:
@@ -468,10 +450,12 @@ def revXlateContainer(self, model, yang, config, table):
     if isinstance(model['list'], dict):
         modelList = model['list']
         # Pass matching list from Yang Json
+        self.logInFile("revXlateContainer {}".format(modelList['@name']))
         self.revXlateList(modelList, yang[modelList['@name']], config, table)
 
     elif isinstance(model['list'], list):
         for modelList in model['list']:
+            self.logInFile("revXlateContainer {}".format(modelList['@name']))
             self.revXlateList(modelList, yang[modelList['@name']], config, table)
 
     return
@@ -494,6 +478,7 @@ def revXlateYangtoConfigDB(self, yangJ, cDbJson):
             cmap = self.confDbYangMap[table]
             cDbJson[table] = dict()
             #print(key + "--" + subkey)
+            self.logInFile("revXlateYangtoConfigDB {}".format(table))
             self.revXlateContainer(cmap['container'], yangJ[module_top][container], \
                 cDbJson[table], table)
 

src/sonic-yang-mgmt/sonic_yang.py

@@ -71,7 +71,7 @@ def logInFile(self, header="", obj=None, json=False):
     """
     def load_schema_module(self, yang_file):
         try:
-            self.ctx.parse_module_path(yang_file, ly.LYS_IN_YANG)
+            return self.ctx.parse_module_path(yang_file, ly.LYS_IN_YANG)
         except Exception as e:
             print("Failed to load yang module file: " + yang_file)
             self.fail(e)

src/sonic-yang-mgmt/tests/libyang-python-tests/test_sonic_yang.py

@@ -7,12 +7,18 @@
 import getopt
 import subprocess
 import glob
+import logging
 from ijson import items as ijson_itmes
 
 test_path = os.path.dirname(os.path.abspath(__file__))
 modules_path = os.path.dirname(test_path)
 sys.path.insert(0, modules_path)
 
+logging.basicConfig(level=logging.DEBUG)
+log = logging.getLogger("YANG-TEST")
+log.setLevel(logging.INFO)
+log.addHandler(logging.NullHandler())
+
 class Test_SonicYang(object):
     # class vars
     yang_test_file = "/sonic/src/sonic-yang-mgmt/tests/yang-model-tests/yangTest.json"
@@ -268,7 +274,7 @@ def test_xlate_rev_xlate(self):
 
         syc.get_data()
 
-        if syc.jIn == syc.revXlateJson:
+        if syc.jIn and syc.jIn == syc.revXlateJson:
             print("Xlate and Rev Xlate Passed")
         else:
             print("Xlate and Rev Xlate failed")

src/sonic-yang-mgmt/tests/yang-model-tests/yangModelTesting.py

@@ -7,6 +7,7 @@
 import ijson
 import json
 #import sonic_yang as sy
+from glob import glob
 from os import listdir
 from os.path import isfile, join, splitext
 
@@ -134,27 +135,21 @@ def __init__(self, tests, yangDir, jsonFile):
         load all YANG models before test run
     """
     def loadYangModel(self, yangDir):
+
         try:
-            # get all files
-            yangFiles = [f for f in listdir(yangDir) if isfile(join(yangDir, f))]
-            # get all yang files
-            yangFiles = [f for f in yangFiles if splitext(f)[-1].lower()==".yang"]
-            yangFiles = [f.split('.')[0] for f in yangFiles]
-            # load yang mdoules
+            # create context
             self.ctx = ly.Context(yangDir)
-            log.debug(yangFiles)
-            for f in yangFiles:
-                # load a module
-                log.debug(f)
-                m = self.ctx.get_module(f)
+            # get all files
+            yangFiles = glob(yangDir +"/*.yang")
+            # load yang modules
+            for file in yangFiles:
+                log.debug(file)
+                m = self.ctx.parse_module_path(file, ly.LYS_IN_YANG)
                 if m is not None:
-                    log.error("Could not get module: {}".format(m.name()))
+                    log.info("module: {} is loaded successfully".format(m.name()))
                 else:
-                    m = self.ctx.load_module(f)
-                    if m is not None:
-                        log.info("module: {} is loaded successfully".format(m.name()))
-                    else:
-                        return
+                    log.info("Could not load module: {}".format(file))
+
         except Exception as e:
             printExceptionDetails()
             raise e
@@ -275,7 +270,8 @@ def main():
 
         yTest = YangModelTesting(tests, yangDir, jsonFile)
         if (listTests):
-            log.info(yTest.ExceptionTests.keys())
+            for key in yTest.ExceptionTests.keys():
+                log.info(key)
             sys.exit(0)
 
         ret = yTest.run()

src/sonic-yang-mgmt/yang-models/sonic-acl.yang

@@ -18,6 +18,11 @@ module sonic-acl {
 		revision-date 2019-07-01;
 	}
 
+	import sonic-extension {
+		prefix ext;
+		revision-date 2019-07-01;
+	}
+
 	import sonic-port {
 		prefix port;
 		revision-date 2019-07-01;
@@ -48,6 +53,10 @@ module sonic-acl {
 
 				key "ACL_TABLE_NAME RULE_NAME";
 
+				ext:key-regex-configdb-to-yang "^([a-zA-Z0-9_-]+)|([a-zA-Z0-9_-]+)$";
+
+				ext:key-regex-yang-to-configdb "<ACL_TABLE_NAME>|<RULE_NAME>";
+
 				leaf ACL_TABLE_NAME {
 					type leafref {
 						path "/acl:sonic-acl/acl:ACL_TABLE/acl:ACL_TABLE_LIST/acl:ACL_TABLE_NAME";
@@ -232,6 +241,10 @@ module sonic-acl {
 
 				key "ACL_TABLE_NAME";
 
+				ext:key-regex-configdb-to-yang "^([a-zA-Z0-9-_]+)$";
+
+				ext:key-regex-yang-to-configdb "<ACL_TABLE_NAME>";
+
 				leaf ACL_TABLE_NAME {
 					type string;
 				}

src/sonic-yang-mgmt/yang-models/sonic-head.yang

@@ -1,6 +1,8 @@
 module sonic-head {
 
-    namespace "http://sonic-head";
+    yang-version 1.1;
+
+    namespace "http://github.com/Azure/sonic-head";
     prefix sonic-head;
 
     organization "Linkedin Corporation";
@@ -91,4 +93,9 @@ module sonic-head {
             enum priority_tagged;
         }
     }
+
+    extension key-regex-configdb-to-yang {
+        description "Key regex used to convert config DB keys to YANG Config";
+        argument "value";
+    }
 }

src/sonic-yang-mgmt/yang-models/sonic-interface.yang

@@ -1,5 +1,7 @@
 module sonic-interface {
 
+	yang-version 1.1;
+
 	namespace "http://github.com/Azure/sonic-interface";
 	prefix intf;
 
@@ -16,6 +18,11 @@ module sonic-interface {
 		revision-date 2019-07-01;
 	}
 
+	import sonic-extension {
+		prefix ext;
+		revision-date 2019-07-01;
+	}
+
 	import sonic-port {
 		prefix port;
 		revision-date 2019-07-01;
@@ -43,6 +50,10 @@ module sonic-interface {
 
 				key "port_name";
 
+				ext:key-regex-configdb-to-yang "^(Ethernet[0-9]+)$";
+
+				ext:key-regex-yang-to-configdb "<port_name>";
+
 				leaf port_name {
 					type leafref {
 						path /port:sonic-port/port:PORT/port:PORT_LIST/port:port_name;
@@ -64,6 +75,10 @@ module sonic-interface {
 
 				key "port_name ip-prefix";
 
+				ext:key-regex-configdb-to-yang "^(Ethernet[0-9]+)|([a-fA-F0-9:./]+)$";
+
+				ext:key-regex-yang-to-configdb "<port_name>|<ip-prefix>";
+
 				leaf port_name {
 					/* This node must be present in INTERFACE_LIST */
 					must "(current() = ../../INTERFACE_LIST[port_name=current()]/port_name)"

src/sonic-yang-mgmt/yang-models/sonic-loopback-interface.yang

@@ -12,6 +12,11 @@ module sonic-loopback-interface {
         revision-date 2019-07-01;
     }
 
+    import sonic-extension {
+        prefix ext;
+        revision-date 2019-07-01;
+    }
+
     organization "Linkedin Corporation";
 
     contact "lnos_coders@linkedin.com";
@@ -30,6 +35,10 @@ module sonic-loopback-interface {
             list LOOPBACK_INTERFACE_LIST {
                 key "loopback_interface_name";
 
+                ext:key-regex-configdb-to-yang "^([a-zA-Z0-9-_]+)$";
+
+                ext:key-regex-yang-to-configdb "<loopback_interface_name>";
+
                 leaf loopback_interface_name{
                     type string;
                 }
@@ -47,8 +56,11 @@ module sonic-loopback-interface {
 
                 key "loopback_interface_name ip-prefix";
 
-                leaf loopback_interface_name{
+                ext:key-regex-configdb-to-yang "^([a-zA-Z0-9-_]+)|([a-fA-F0-9:./]+$)";
 
+                ext:key-regex-yang-to-configdb "<loopback_interface_name>|<ip-prefix>";
+
+                leaf loopback_interface_name{
                     /* This node must be present in LOOPBACK_INTERFACE_LIST */
                     must "(current() = ../../LOOPBACK_INTERFACE_LIST[loopback_interface_name=current()]/loopback_interface_name)"
                     {