Implement RFC4363 dot1qTpFdbTable Q-BRIDGE-MIB (sonic-net#3)

* Implement Dot1qTpFdbPort OIDs

* (comment)

* Unit test pass

* Make doctest working

* FDB entries are in ASIC_DB

* Fix integration test

* Fix sub_id, add error handling

* Fix MIBEntry PREFIXLEN

* Move common function

* (update README)

* Clean up import

* Fix: GetPDU contructs search range, add test case for GetNextPDU

* Refactor MIBSubtreeEntry

* SubtreeMIBEntry supports dynamic OIDs

* Simple refinement

* Clean code

README.md

@@ -6,6 +6,7 @@ MIB implementations included:
 
 * [RFC 1213](https://www.ietf.org/rfc/rfc1213.txt) MIB-II
 * [RFC 2863](https://www.ietf.org/rfc/rfc2863.txt) Interfaces MIB
+* [RFC 4363](https://tools.ietf.org/html/rfc4363) dot1qTpFdbPort in Q-BRIDGE-MIB
 * [IEEE 802.1 AB](http://www.ieee802.org/1/files/public/MIBs/LLDP-MIB-200505060000Z.txt) LLDP-MIB
 
 To install:

src/ax_interface/__init__.py

@@ -10,4 +10,4 @@
 from . import exceptions
 from .agent import Agent
 from .constants import ValueType
-from .mib import MIBMeta, MIBUpdater, MIBEntry, ContextualMIBEntry
+from .mib import MIBMeta, MIBUpdater, MIBEntry, ContextualMIBEntry, SubtreeMIBEntry

src/ax_interface/encodings.py

@@ -52,6 +52,13 @@ def to_bytes(self, endianness):
         format_string = endianness + 'BBBB' + str(len(self.subids)) + 'L'
         return struct.pack(format_string, self.n_subid, self.prefix_, self.include, self.reserved, *self.subids)
 
+    def inc(self):
+        """
+        Returns a new object identifier with last subid increased by one
+        """
+        newsubids = self.subids[:-1] + (self.subids[-1] + 1,)
+        return self._replace(subids = newsubids)
+
     @classmethod
     def null_oid(cls):
         return cls(0, 0, 0, 0, ())

src/ax_interface/mib.py

@@ -44,26 +44,48 @@ class MIBMeta(type):
 
     def __new__(mcs, name, bases, attributes, prefix=None):
         cls = type.__new__(mcs, name, bases, attributes)
+
+        # each object-type, ie. MIBEntry will has a prefix
+        # ref: https://tools.ietf.org/html/rfc2741#section-2.1
+        prefixes = []
+
         if prefix is not None:
             if not util.is_valid_oid(prefix, dot_prefix=True):
                 raise ValueError("Invalid prefix '{}' for class '{}'".format(prefix, name))
 
             _prefix = util.oid2tuple(prefix)
+            _prefix_len = len(_prefix)
+            for me in vars(cls).values():
+                if isinstance(me, MIBEntry):
+                    setattr(me, MIBEntry.PREFIXLEN, _prefix_len + len(me.subtree))
+
+            sub_ids = {}
 
             # gather all static MIB entries.
-            sub_ids = {_prefix + v.sub_id: v for k, v in vars(cls).items() if type(v) is MIBEntry}
+            static_entries = (v for v in vars(cls).values() if type(v) is MIBEntry)
+            for me in static_entries:
+                sub_ids.update({_prefix + me.subtree: me})
+                prefixes.append(_prefix + me.subtree)
 
             # gather all contextual IDs for each MIB entry--and drop them into the sub-ID listing
             contextual_entries = (v for v in vars(cls).values() if type(v) is ContextualMIBEntry)
             for cme in contextual_entries:
+                sub_ids.update({_prefix + cme.subtree: cme})
+                prefixes.append(_prefix + cme.subtree)
                 for sub_id in cme:
-                    sub_ids.update({_prefix + sub_id: cme})
+                    sub_ids.update({_prefix + cme.subtree + sub_id: cme})
+
+
+            # gather all subtree IDs
+            # to support dynamic sub_id in the subtree, not to pour leaves into dictionary
+            subtree_entries = (v for v in vars(cls).values() if type(v) is SubtreeMIBEntry)
+            for sme in subtree_entries:
+                sub_ids.update({_prefix + sme.subtree: sme})
+                prefixes.append(_prefix + sme.subtree)
 
             # gather all updater instances
             updaters = set(v for k, v in vars(cls).items() if isinstance(v, MIBUpdater))
 
-            # inherit any MIBs from base classes (if they exist)
-            prefixes = [_prefix]
         else:
             # wrapper classes should omit the prefix.
             sub_ids = {}
@@ -92,6 +114,8 @@ def __init__(cls, name, bases, attributes, prefix=None):
 
 
 class MIBEntry:
+    PREFIXLEN = '__prefixlen__'
+
     def __init__(self, subtree, value_type, callable_, *args):
         """
         MIB Entry namespace container. Associates a particular OID subtree to a ValueType return and a callable
@@ -114,24 +138,64 @@ def __init__(self, subtree, value_type, callable_, *args):
         self._callable_args = args
         self.subtree = subtree
         self.value_type = value_type
-        self.sub_id = util.oid2tuple(subtree, dot_prefix=False)
+        self.subtree = util.oid2tuple(subtree, dot_prefix=False)
+
+    def __iter__(self):
+        yield ()
 
     def __call__(self, sub_id=None):
         return self._callable_.__call__(*self._callable_args)
 
+    def get_sub_id(self, oid_key):
+        return oid_key[getattr(self, MIBEntry.PREFIXLEN):]
+
+    def replace_sub_id(self, oid_key, sub_id):
+        return oid_key[:getattr(self, MIBEntry.PREFIXLEN)] + sub_id
+
+    def get_next(self, sub_id):
+        return None
 
 class ContextualMIBEntry(MIBEntry):
     def __init__(self, subtree, sub_ids, value_type, callable_, *args, updater=None):
         super().__init__(subtree, value_type, callable_, *args)
-        self.sub_ids = sub_ids
+        self.sub_ids = sorted(list(sub_ids))
+        self.sub_ids = [(i,) for i in self.sub_ids]
 
     def __iter__(self):
         for sub_id in self.sub_ids:
-            yield self.sub_id + (sub_id,)
+            yield sub_id
 
-    def __call__(self, sub_id=None):
+    def __call__(self, sub_id):
+        if sub_id not in self.sub_ids:
+            return None
+        return self._callable_.__call__(sub_id[0], *self._callable_args)
+
+    def get_next(self, sub_id):
+        right = bisect.bisect_right(self.sub_ids, sub_id)
+        if right == len(self.sub_ids):
+            return None
+        return self.sub_ids[right]
+
+class SubtreeMIBEntry(MIBEntry):
+    def __init__(self, subtree, iterator, value_type, callable_, *args, updater=None):
+        super().__init__(subtree, value_type, callable_, *args)
+        self.iterator = iterator
+
+    def __iter__(self):
+        sub_id = ()
+        while True:
+            sub_id = self.iterator.get_next(sub_id)
+            if sub_id is None:
+                break
+            yield self.subtree + sub_id
+
+    def __call__(self, sub_id):
+        assert isinstance(sub_id, tuple)
         return self._callable_.__call__(sub_id, *self._callable_args)
 
+    def get_next(self, sub_id):
+        return self.iterator.get_next(sub_id)
+
 
 class MIBTable(dict):
     """
@@ -166,46 +230,76 @@ def _find_parent_prefix(self, item):
         else:
             return None
 
+    def _find_parent_oid_key(self, oid_key):
+        oids = sorted(self)
+
+    def _get_value(self, mib_entry, oid_key):
+        sub_id = mib_entry.get_sub_id(oid_key)
+        oid_value = mib_entry(sub_id)
+        if oid_value is None:
+            return None
+        # OID found, call the OIDEntry
+        vr = ValueRepresentation.from_typecast(mib_entry.value_type, oid_key, oid_value)
+        return vr
+
+    def _get_nextvalue(self, mib_entry, oid_key):
+        sub_id = mib_entry.get_sub_id(oid_key)
+        key1 = mib_entry.get_next(sub_id)
+        if key1 is None:
+            return None
+        val1 = mib_entry(key1)
+        if val1 is None:
+            return None
+        oid1 = mib_entry.replace_sub_id(oid_key, key1)
+        # OID found, call the OIDEntry
+        vr = ValueRepresentation.from_typecast(mib_entry.value_type, oid1, val1)
+        return vr
+
     def get(self, sr, d=None):
         oid_key = sr.start.to_tuple()
-        mib_entry = super().get(oid_key)
-        if mib_entry is None:
-            # no exact OID found
-            prefix = self._find_parent_prefix(oid_key)
-            if prefix is not None:
-                # we found a prefix. E.g. (1,2,3) is a prefix to OID (1,2,3,1)
-                value_type = ValueType.NO_SUCH_INSTANCE
-            else:
-                # couldn't find the exact OID, or a valid prefix
-                value_type = ValueType.NO_SUCH_OBJECT
-
-            vr = ValueRepresentation(
-                value_type,
-                0,  # reserved
-                sr.start,
-                None,  # null value
-            )
+
+        # find the best match prefix, either a exact match or a parent prefix
+        prefix = self._find_parent_prefix(oid_key)
+        if prefix is not None:
+            parent_mib_entry = super().get(prefix)
+            vr = self._get_value(parent_mib_entry, oid_key)
+            if vr is not None:
+                return vr
+            # we found a prefix. E.g. (1,2,3) is a prefix to OID (1,2,3,1)
+            value_type = ValueType.NO_SUCH_INSTANCE
         else:
-            oid_value = mib_entry(oid_key[-1])
-            # OID found, call the OIDEntry
-            vr = ValueRepresentation(
-                mib_entry.value_type if oid_value is not None else ValueType.NO_SUCH_INSTANCE,
-                0,  # reserved
-                sr.start,
-                oid_value
-            )
+            # couldn't find the exact OID, or a valid prefix
+            value_type = ValueType.NO_SUCH_OBJECT
+
+        vr = ValueRepresentation(
+            value_type,
+            0,  # reserved
+            sr.start,
+            None,  # null value
+        )
         return vr
 
     def get_next(self, sr):
         start_key = sr.start.to_tuple()
         end_key = sr.end.to_tuple()
-        oid_list = sorted(self.keys())
-        if sr.start.include:
-            # return the index of an insertion point immediately preceding any duplicate value (thereby including it)
-            sorted_start_index = bisect.bisect_left(oid_list, start_key)
-        else:
-            # return the index of an insertion point immediately following any duplicate value (thereby excluding it)
-            sorted_start_index = bisect.bisect_right(oid_list, start_key)
+        oid_list = sorted(self.prefixes)
+
+        # find the best match prefix, either a exact match or a parent prefix
+        prefix = self._find_parent_prefix(start_key)
+        if prefix is not None:
+            parent_mib_entry = super().get(prefix)
+
+            if sr.start.include:
+                vr = self._get_value(parent_mib_entry, start_key)
+                if vr is not None:
+                    return vr
+
+            vr = self._get_nextvalue(parent_mib_entry, start_key)
+            if vr is not None:
+                return vr
+
+        # return the index of an insertion point immediately following any duplicate value (thereby excluding it)
+        sorted_start_index = bisect.bisect_right(oid_list, start_key)
 
         # slice our MIB by the insertion point.
         remaining_oids = oid_list[sorted_start_index:]
@@ -215,18 +309,27 @@ def get_next(self, sr):
             # is less than our end value--it's a match.
             oid_key = remaining_oids[0]
             mib_entry = self[oid_key]
-            oid_value = mib_entry(oid_key[-1])
-            if oid_value is None:
+            key1 = next(iter(mib_entry)) # get the first sub_id from the mib_etnry
+            if key1 is None:
+                # handler returned None, which implies there's no data, keep walking.
+                remaining_oids = remaining_oids[1:]
+                continue
+
+            val1 = mib_entry(key1)
+            if val1 is None:
                 # handler returned None, which implies there's no data, keep walking.
                 remaining_oids = remaining_oids[1:]
                 continue
-            else:
-                # found a concrete OID value--return it.
-                return ValueRepresentation.from_typecast(
-                    mib_entry.value_type,
-                    oid_key,
-                    oid_value
-                )
+
+            oid1 = mib_entry.replace_sub_id(oid_key, key1)
+
+            # found a concrete OID value--return it.
+            vr = ValueRepresentation.from_typecast(
+                mib_entry.value_type,
+                oid1,
+                val1
+            )
+            return vr
 
         # exhausted all remaining OID options--we're at the end of the MIB view.
         return ValueRepresentation(

src/ax_interface/pdu_implementations.py

@@ -179,7 +179,7 @@ def __init__(self, header=None, payload=None, context=None, oids=None):
         else:
             for oid in oids:
                 self.sr.append(
-                    SearchRange(start=oid, end=ObjectIdentifier.null_oid())
+                    SearchRange(start=oid, end=oid.inc())
                 )
             self.header = self.header._replace(payload_length=len(self.encode()))
 

src/ax_interface/util.py

@@ -90,3 +90,11 @@ def pad4bytes(length):
     :return:
     """
     return pad4(length) * constants.RESERVED_ZERO_BYTE
+
+def mac_decimals(mac):
+    """
+    >>> mac_decimals("52:54:00:57:59:6A")
+    (82, 84, 0, 87, 89, 106)
+    """
+    return tuple(int(h, 16) for h in mac.split(":"))
+

src/sonic_ax_impl/main.py

@@ -11,7 +11,7 @@
 import ax_interface
 from sonic_ax_impl.mibs import ieee802_1ab
 from . import logger
-from .mibs.ietf import rfc1213, rfc2863
+from .mibs.ietf import rfc1213, rfc2863, rfc4363
 
 # Background task update frequency ( in seconds )
 DEFAULT_UPDATE_FREQUENCY = 5
@@ -23,6 +23,7 @@
 class SonicMIB(
     rfc1213.InterfacesMIB,
     rfc2863.InterfaceMIBObjects,
+    rfc4363.QBridgeMIBObjects,
     ieee802_1ab.LLDPLocPortTable,
     ieee802_1ab.LLDPRemTable,
 ):

src/sonic_ax_impl/mibs/__init__.py

@@ -8,6 +8,7 @@
 COUNTERS_PORT_NAME_MAP = b'COUNTERS_PORT_NAME_MAP'
 SONIC_ETHERNET_RE_PATTERN = "^Ethernet(\d+)$"
 APPL_DB = 'APPL_DB'
+ASIC_DB = 'ASIC_DB'
 COUNTERS_DB = 'COUNTERS_DB'
 
 
@@ -49,6 +50,7 @@ def init_sync_d_interface_tables():
     db_conn.connect(COUNTERS_DB)
 
     # { if_name (SONiC) -> sai_id }
+    # ex: { "Ethernet76" : "1000000000023" }
     if_name_map = db_conn.get_all(COUNTERS_DB, COUNTERS_PORT_NAME_MAP, blocking=True)
     logger.debug("Port name map:\n" + pprint.pformat(if_name_map, indent=2))
 

src/sonic_ax_impl/mibs/ietf/__init__.py

@@ -0,0 +1 @@
+