Merge branch 'master' into reload_enhancement

acl_loader/main.py

@@ -10,8 +10,7 @@
 import pyangbind.lib.pybindJSON as pybindJSON
 from natsort import natsorted
 from sonic_py_common import device_info
-from swsssdk import ConfigDBConnector, SonicDBConfig
-from swsscommon.swsscommon import SonicV2Connector
+from swsscommon.swsscommon import SonicV2Connector, ConfigDBConnector, SonicDBConfig
 
 
 def info(msg):

azure-pipelines.yml

@@ -0,0 +1,19 @@
+# Starter pipeline
+# Start with a minimal pipeline that you can customize to build and deploy your code.
+# Add steps that build, run tests, deploy, and more:
+# https://aka.ms/yaml
+
+trigger:
+- master
+
+pool:
+  vmImage: ubuntu-20.04
+
+steps:
+- script: echo Hello, world!
+  displayName: 'Run a one-line script'
+
+- script: |
+    echo Add other tasks to build, test, and deploy your project.
+    echo See https://aka.ms/yaml
+  displayName: 'Run a multi-line script'

clear/main.py

@@ -78,8 +78,7 @@ def get_routing_stack():
         proc = subprocess.Popen(command,
                                 stdout=subprocess.PIPE,
                                 shell=True,
-                                text=True,
-                                stderr=subprocess.STDOUT)
+                                text=True)
         stdout = proc.communicate()[0]
         proc.wait()
         result = stdout.rstrip('\n')

config/aaa.py

@@ -1,5 +1,5 @@
 import click
-from swsssdk import ConfigDBConnector
+from swsscommon.swsscommon import ConfigDBConnector
 import utilities_common.cli as clicommon
 
 def add_table_kv(table, entry, key, val):

config/config_mgmt.py

@@ -14,8 +14,8 @@
 
     # SONiC specific imports
     import sonic_yang
-    from swsssdk import ConfigDBConnector, port_util
-    from swsscommon.swsscommon import SonicV2Connector
+    from swsssdk import port_util
+    from swsscommon.swsscommon import SonicV2Connector, ConfigDBConnector
 
     # Using load_source to 'import /usr/local/bin/sonic-cfggen as sonic_cfggen'
     # since /usr/local/bin/sonic-cfggen does not have .py extension.
@@ -126,7 +126,8 @@ def validateConfigData(self):
         try:
             self.sy.validate_data_tree()
         except Exception as e:
-            self.sysLog(msg='Data Validation Failed')
+            self.sysLog(doPrint=True, logLevel=syslog.LOG_ERR,
+                msg='Data Validation Failed')
             return False
 
         self.sysLog(msg='Data Validation successful', doPrint=True)
@@ -146,7 +147,8 @@ def sysLog(self, logLevel=syslog.LOG_INFO, msg=None, doPrint=False):
         # log debug only if enabled
         if self.DEBUG == False and logLevel == syslog.LOG_DEBUG:
             return
-        if flags.interactive !=0 and doPrint == True:
+        # always print < Info level msg with doPrint flag
+        if doPrint == True and (logLevel < syslog.LOG_INFO or flags.interactive != 0):
             print("{}".format(msg))
         syslog.openlog(self.SYSLOG_IDENTIFIER)
         syslog.syslog(logLevel, msg)
@@ -188,8 +190,8 @@ def readConfigDB(self):
         '''
         self.sysLog(doPrint=True, msg='Reading data from Redis configDb')
         # Read from config DB on sonic switch
-        db_kwargs = dict(); data = dict()
-        configdb = ConfigDBConnector(**db_kwargs)
+        data = dict()
+        configdb = ConfigDBConnector()
         configdb.connect()
         deep_update(data, FormatConverter.db_to_output(configdb.get_config()))
         self.configdbJsonIn =  FormatConverter.to_serialized(data)
@@ -209,8 +211,8 @@ def writeConfigDB(self, jDiff):
             void
         '''
         self.sysLog(doPrint=True, msg='Writing in Config DB')
-        db_kwargs = dict(); data = dict()
-        configdb = ConfigDBConnector(**db_kwargs)
+        data = dict()
+        configdb = ConfigDBConnector()
         configdb.connect(False)
         deep_update(data, FormatConverter.to_deserialized(jDiff))
         self.sysLog(msg="Write in DB: {}".format(data))
@@ -712,18 +714,31 @@ def _deleteHandler(diff, inp, outp, config):
             '''
             if isinstance(inp, dict):
                 # Example Case: diff = PORT': {delete: {u'Ethernet1': {...}}}}
+                self.sysLog(logLevel=syslog.LOG_DEBUG, \
+                    msg="Delete Dict diff:{}".format(diff))
                 for key in diff:
                     # make sure keys from diff are present in inp but not in outp
                     if key in inp and key not in outp:
-                        # assign key to None(null), redis will delete entire key
-                        config[key] = None
+                        if type(inp[key]) == list:
+                            self.sysLog(logLevel=syslog.LOG_DEBUG, \
+                                msg="Delete List key:{}".format(key))
+                            # assign current lists as empty.
+                            config[key] = []
+                        else:
+                            self.sysLog(logLevel=syslog.LOG_DEBUG, \
+                                msg="Delete Dict key:{}".format(key))
+                            # assign key to None(null), redis will delete entire key
+                            config[key] = None
                     else:
                         # should not happen
                         raise Exception('Invalid deletion of {} in diff'.format(key))
 
             elif isinstance(inp, list):
-                # Example case: {u'VLAN': {u'Vlan100': {'members': {delete: [(95, 'Ethernet1')]}}
-                # just take list from outputs
+                # Example case: diff: [(3, 'Ethernet10'), (2, 'Ethernet8')]
+                # inp:['Ethernet0', 'Ethernet4', 'Ethernet8', 'Ethernet10']
+                # outp:['Ethernet0', 'Ethernet4']
+                self.sysLog(logLevel=syslog.LOG_DEBUG, \
+                    msg="Delete List diff: {} inp:{} outp:{}".format(diff, inp, outp))
                 config.extend(outp)
             return
 
@@ -733,19 +748,31 @@ def _insertHandler(diff, inp, outp, config):
             '''
             if isinstance(outp, dict):
                 # Example Case: diff = PORT': {insert: {u'Ethernet1': {...}}}}
+                self.sysLog(logLevel=syslog.LOG_DEBUG, \
+                    msg="Insert Dict diff:{}".format(diff))
                 for key in diff:
                     # make sure keys are only in outp
                     if key not in inp and key in outp:
+                        self.sysLog(logLevel=syslog.LOG_DEBUG, \
+                            msg="Insert Dict key:{}".format(key))
                         # assign key in config same as outp
                         config[key] = outp[key]
                     else:
                         # should not happen
                         raise Exception('Invalid insertion of {} in diff'.format(key))
 
             elif isinstance(outp, list):
-                # just take list from output
-                # Example case: {u'VLAN': {u'Vlan100': {'members': {insert: [(95, 'Ethernet1')]}}
+                # Example diff:[(2, 'Ethernet8'), (3, 'Ethernet10')]
+                # in:['Ethernet0', 'Ethernet4']
+                # out:['Ethernet0', 'Ethernet4', 'Ethernet8', 'Ethernet10']
+                self.sysLog(logLevel=syslog.LOG_DEBUG, \
+                    msg="Insert list diff:{} inp:{} outp:{}".format(diff, inp, outp))
                 config.extend(outp)
+                # configDb stores []->[""], i.e. empty list as list of empty
+                # string. While adding default config for newly created ports,
+                # inp can be [""], in that case remove it from delta config.
+                if inp == ['']:
+                    config.remove('');
             return
 
         def _recurCreateConfig(diff, inp, outp, config):

config/kdump.py

@@ -1,7 +1,7 @@
 import os
 import click
 import utilities_common.cli as clicommon
-from swsssdk import ConfigDBConnector
+from swsscommon.swsscommon import ConfigDBConnector
 
 @click.group(cls=clicommon.AbbreviationGroup, name="kdump")
 def kdump():

config/main.py

@@ -16,8 +16,7 @@
 from portconfig import get_child_ports
 from sonic_py_common import device_info, multi_asic
 from sonic_py_common.interface import get_interface_table_name, get_port_table_name
-from swsssdk import ConfigDBConnector, SonicDBConfig
-from swsscommon.swsscommon import SonicV2Connector
+from swsscommon.swsscommon import SonicV2Connector, ConfigDBConnector, SonicDBConfig
 from utilities_common.db import Db
 from utilities_common.intf_filter import parse_interface_in_filter
 import utilities_common.cli as clicommon
@@ -668,6 +667,7 @@ def _get_disabled_services_list(config_db):
 
     return disabled_services_list
 
+
 def _stop_services():
     click.echo("Stopping SONiC target ...")
     clicommon.run_command("sudo systemctl stop sonic.target")
@@ -1761,14 +1761,14 @@ def is_dynamic_buffer_enabled(config_db):
 @click.option('-s', '--redis-unix-socket-path', help='unix socket path for redis connection')
 def warm_restart(ctx, redis_unix_socket_path):
     """warm_restart-related configuration tasks"""
-    kwargs = {}
-    if redis_unix_socket_path:
-        kwargs['unix_socket_path'] = redis_unix_socket_path
-    config_db = ConfigDBConnector(**kwargs)
+    # Note: redis_unix_socket_path is a path string, and the ground truth is now from database_config.json.
+    # We only use it as a bool indicator on either unix_socket_path or tcp port
+    use_unix_socket_path = bool(redis_unix_socket_path)
+    config_db = ConfigDBConnector(use_unix_socket_path=use_unix_socket_path)
     config_db.connect(wait_for_init=False)
 
     # warm restart enable/disable config is put in stateDB, not persistent across cold reboot, not saved to config_DB.json file
-    state_db = SonicV2Connector(host='127.0.0.1')
+    state_db = SonicV2Connector(use_unix_socket_path=use_unix_socket_path)
     state_db.connect(state_db.STATE_DB, False)
     TABLE_NAME_SEPARATOR = '|'
     prefix = 'WARM_RESTART_ENABLE_TABLE' + TABLE_NAME_SEPARATOR
@@ -2524,6 +2524,13 @@ def add(ctx, interface_name, ip_addr, gw):
         if interface_name is None:
             ctx.fail("'interface_name' is None!")
 
+    # Add a validation to check this interface is not a member in vlan before 
+    # changing it to a router port 
+    vlan_member_table = config_db.get_table('VLAN_MEMBER')
+    if (interface_is_in_vlan(vlan_member_table, interface_name)):
+            click.echo("Interface {} is a member of vlan\nAborting!".format(interface_name))
+            return
+
     try:
         net = ipaddress.ip_network(ip_addr, strict=False)
         if '/' not in ip_addr:
@@ -2952,7 +2959,7 @@ def bind(ctx, interface_name, vrf_name):
         state_db = SonicV2Connector(use_unix_socket_path=True, namespace=ctx.obj['namespace'])
     state_db.connect(state_db.STATE_DB, False)
     _hash = '{}{}'.format('INTERFACE_TABLE|', interface_name)
-    while state_db.get_all(state_db.STATE_DB, _hash) != None:
+    while state_db.exists(state_db.STATE_DB, _hash):
         time.sleep(0.01)
     state_db.close(state_db.STATE_DB)
     config_db.set_entry(table_name, interface_name, {"vrf_name": vrf_name})
@@ -3412,8 +3419,11 @@ def remove_reasons(counter_name, reasons, verbose):
 @click.option('-ymin', metavar='<yellow threshold min>', type=int, help="Set yellow min threshold")
 @click.option('-gmax', metavar='<green threshold max>', type=int, help="Set green max threshold")
 @click.option('-gmin', metavar='<green threshold min>', type=int, help="Set green min threshold")
+@click.option('-rdrop', metavar='<red drop probability>', type=click.IntRange(0, 100), help="Set red drop probability")
+@click.option('-ydrop', metavar='<yellow drop probability>', type=click.IntRange(0, 100), help="Set yellow drop probability")
+@click.option('-gdrop', metavar='<green drop probability>', type=click.IntRange(0, 100), help="Set green drop probability")
 @click.option('-v', '--verbose', is_flag=True, help="Enable verbose output")
-def ecn(profile, rmax, rmin, ymax, ymin, gmax, gmin, verbose):
+def ecn(profile, rmax, rmin, ymax, ymin, gmax, gmin, rdrop, ydrop, gdrop, verbose):
     """ECN-related configuration tasks"""
     log.log_info("'ecn -profile {}' executing...".format(profile))
     command = "ecnconfig -p %s" % profile
@@ -3423,6 +3433,9 @@ def ecn(profile, rmax, rmin, ymax, ymin, gmax, gmin, verbose):
     if ymin is not None: command += " -ymin %d" % ymin
     if gmax is not None: command += " -gmax %d" % gmax
     if gmin is not None: command += " -gmin %d" % gmin
+    if rdrop is not None: command += " -rdrop %d" % rdrop
+    if ydrop is not None: command += " -ydrop %d" % ydrop
+    if gdrop is not None: command += " -gdrop %d" % gdrop
     if verbose: command += " -vv"
     clicommon.run_command(command, display_cmd=verbose)
 

config/muxcable.py

@@ -5,8 +5,7 @@
 import click
 import utilities_common.cli as clicommon
 from sonic_py_common import multi_asic
-from swsssdk import ConfigDBConnector
-from swsscommon import swsscommon
+from swsscommon.swsscommon import SonicV2Connector, ConfigDBConnector
 from tabulate import tabulate
 from utilities_common import platform_sfputil_helper
 
@@ -71,7 +70,7 @@ def lookup_statedb_and_update_configdb(per_npu_statedb, config_db, port, state_c
     ipv6_value = get_value_for_key_in_config_tbl(config_db, port, "server_ipv6", "MUX_CABLE")
 
     state = get_value_for_key_in_dict(muxcable_statedb_dict, port, "state", "MUX_CABLE_TABLE")
-    if (state == "active" and configdb_state == "active") or (state == "standby" and configdb_state == "active") or (state == "unknown" and  configdb_state == "active") :
+    if (state == "active" and configdb_state == "active") or (state == "standby" and configdb_state == "active") or (state == "unknown" and configdb_state == "active"):
         if state_cfg_val == "active":
             # status is already active, so right back error
             port_status_dict[port] = 'OK'
@@ -90,7 +89,7 @@ def lookup_statedb_and_update_configdb(per_npu_statedb, config_db, port, state_c
             # dont write anything to db, write OK to user
             port_status_dict[port] = 'OK'
 
-    elif (state == "standby" and configdb_state == "auto") or (state == "unknown" and  configdb_state == "auto"):
+    elif (state == "standby" and configdb_state == "auto") or (state == "unknown" and configdb_state == "auto"):
         if state_cfg_val == "active":
             # make the state active
             config_db.set_entry("MUX_CABLE", port, {"state": "active",
@@ -123,7 +122,7 @@ def mode(state, port, json_output):
         # replace these with correct macros
         per_npu_configdb[asic_id] = ConfigDBConnector(use_unix_socket_path=True, namespace=namespace)
         per_npu_configdb[asic_id].connect()
-        per_npu_statedb[asic_id] = swsscommon.SonicV2Connector(use_unix_socket_path=True, namespace=namespace)
+        per_npu_statedb[asic_id] = SonicV2Connector(use_unix_socket_path=True, namespace=namespace)
         per_npu_statedb[asic_id].connect(per_npu_statedb[asic_id].STATE_DB)
 
         mux_tbl_cfg_db[asic_id] = per_npu_configdb[asic_id].get_table("MUX_CABLE")
@@ -188,3 +187,73 @@ def mode(state, port, json_output):
                 click.echo(tabulate(data, headers=headers))
 
         sys.exit(CONFIG_SUCCESSFUL)
+
+@muxcable.group(cls=clicommon.AbbreviationGroup)
+def prbs():
+    """Enable/disable PRBS mode on a port"""
+    pass
+
+@prbs.command()
+@click.argument('port', required=True, default=None, type=click.INT)
+@click.argument('target', required=True, default=None, type=click.INT)
+@click.argument('mode_value', required=True, default=None, type=click.INT)
+@click.argument('lane_map', required=True, default=None, type=click.INT)
+def enable(port, target, mode_value, lane_map):
+    """Enable PRBS mode on a port"""
+
+    import sonic_y_cable.y_cable
+    res = sonic_y_cable.y_cable.enable_prbs_mode(port, target, mode_value, lane_map)
+    if res != True:
+        click.echo("PRBS config unsuccesful")
+        sys.exit(CONFIG_FAIL)
+    click.echo("PRBS config sucessful")
+    sys.exit(CONFIG_SUCCESSFUL)
+
+@prbs.command()
+@click.argument('port', required=True, default=None, type=click.INT)
+@click.argument('target', required=True, default=None, type=click.INT)
+def disable(port, target):
+    """Disable PRBS mode on a port"""
+
+    import sonic_y_cable.y_cable
+    res = sonic_y_cable.y_cable.disable_prbs_mode(port, target)
+    if res != True:
+        click.echo("PRBS disable unsuccesful")
+        sys.exit(CONFIG_FAIL)
+    click.echo("PRBS disable sucessful")
+    sys.exit(CONFIG_SUCCESSFUL)
+
+@muxcable.group(cls=clicommon.AbbreviationGroup)
+def loopback():
+    """Enable/disable loopback mode on a port"""
+    pass
+
+
+@loopback.command()
+@click.argument('port', required=True, default=None, type=click.INT)
+@click.argument('target', required=True, default=None, type=click.INT)
+@click.argument('lane_map', required=True, default=None, type=click.INT)
+def enable(port, target, lane_map):
+    """Enable loopback mode on a port"""
+
+    import sonic_y_cable.y_cable
+    res = sonic_y_cable.y_cable.enable_loopback_mode(port, target, lane_map)
+    if res != True:
+        click.echo("loopback config unsuccesful")
+        sys.exit(CONFIG_FAIL)
+    click.echo("loopback config sucessful")
+    sys.exit(CONFIG_SUCCESSFUL)
+
+@loopback.command()
+@click.argument('port', required=True, default=None, type=click.INT)
+@click.argument('target', required=True, default=None, type=click.INT)
+def disable(port, target):
+    """Disable loopback mode on a port"""
+
+    import sonic_y_cable.y_cable
+    res = sonic_y_cable.y_cable.disable_loopback_mode(port, target)
+    if res != True:
+        click.echo("loopback disable unsuccesful")
+        sys.exit(CONFIG_FAIL)
+    click.echo("loopback disable sucessful")
+    sys.exit(CONFIG_SUCCESSFUL)

config/nat.py

@@ -1,8 +1,7 @@
 import ipaddress
 
 import click
-from swsssdk import ConfigDBConnector
-from swsscommon.swsscommon import SonicV2Connector
+from swsscommon.swsscommon import SonicV2Connector, ConfigDBConnector
 
 
 def is_valid_ipv4_address(address):

config/vlan.py

@@ -136,6 +136,9 @@ def add_vlan_member(db, vid, port, untagged):
     if (is_port and clicommon.is_port_router_interface(db.cfgdb, port)) or \
        (not is_port and clicommon.is_pc_router_interface(db.cfgdb, port)):
         ctx.fail("{} is a router interface!".format(port))
+
+    if (clicommon.interface_is_untagged_member(db.cfgdb, port) and untagged):
+        ctx.fail("{} is already untagged member!".format(port))
 
     db.cfgdb.set_entry('VLAN_MEMBER', (vlan, port), {'tagging_mode': "untagged" if untagged else "tagged" })