Merge remote-tracking branch 'origin/syslog_telemetry_shared' into re…

…manava_final

.github/workflows/label.yml

@@ -22,15 +22,6 @@ jobs:
   label:
     runs-on: ubuntu-latest
     steps:
-    - name: approve
-      env:
-        GITHUB_CONTEXT: ${{ toJson(github) }}
-      run: |
-        set -e
-        echo ${{ secrets.GITHUB_TOKEN }} | gh auth login --with-token
-        url=$(echo $GITHUB_CONTEXT | jq -r '.event.pull_request._links.html.href')
-        echo PR: $url
-        gh pr review $url --approve
     - uses: actions/labeler@main
       with:
         repo-token: "${{ secrets.GITHUB_TOKEN }}"

README.md

@@ -14,7 +14,6 @@
 [![Mellanox](https://dev.azure.com/mssonic/build/_apis/build/status/mellanox/Azure.sonic-buildimage.official.mellanox?branchName=master&label=Mellanox)](https://dev.azure.com/mssonic/build/_build/latest?definitionId=139&branchName=master)
 [![Marvell(armhf)](https://dev.azure.com/mssonic/build/_apis/build/status/marvell/Azure.sonic-buildimage.official.marvell-armhf?branchName=master&label=Marvell-armhf)](https://dev.azure.com/mssonic/build/_build/latest?definitionId=141&branchName=master)
 [![Nephos](https://dev.azure.com/mssonic/build/_apis/build/status/nephos/Azure.sonic-buildimage.official.nephos?branchName=master&label=Nephos)](https://dev.azure.com/mssonic/build/_build/latest?definitionId=149&branchName=master)
-[![P4](https://sonic-jenkins.westus2.cloudapp.azure.com/job/p4/job/buildimage-p4-all/badge/icon?subject=P4)](https://sonic-jenkins.westus2.cloudapp.azure.com/job/p4/job/buildimage-p4-all)
 [![VS](https://dev.azure.com/mssonic/build/_apis/build/status/vs/Azure.sonic-buildimage.official.vs?branchName=master&label=VS)](https://dev.azure.com/mssonic/build/_build/latest?definitionId=142&branchName=master)
 
 *202205 builds*:
@@ -70,25 +69,13 @@
 [![Nephos](https://dev.azure.com/mssonic/build/_apis/build/status/nephos/Azure.sonic-buildimage.official.nephos?branchName=201811&label=Nephos)](https://dev.azure.com/mssonic/build/_build/latest?definitionId=149&branchName=201811)
 [![VS](https://dev.azure.com/mssonic/build/_apis/build/status/vs/Azure.sonic-buildimage.official.vs?branchName=201811&label=VS)](https://dev.azure.com/mssonic/build/_build/latest?definitionId=142&branchName=201811)
 
-*201807 builds*:
-
-[![Broadcom](https://sonic-jenkins.westus2.cloudapp.azure.com/job/broadcom/job/buildimage-brcm-201807/badge/icon?subject=Broadcom)](https://sonic-jenkins.westus2.cloudapp.azure.com/job/broadcom/job/buildimage-brcm-201807/)
-[![Barefoot](https://sonic-jenkins.westus2.cloudapp.azure.com/job/barefoot/job/buildimage-bf-201807/badge/icon?subject=Barefoot)](https://sonic-jenkins.westus2.cloudapp.azure.com/job/barefoot/job/buildimage-bf-201807/)
-
-*201803 builds*:
-
-[![Broadcom](https://sonic-jenkins.westus2.cloudapp.azure.com/job/broadcom/job/buildimage-brcm-201803/badge/icon?subject=Broadcom)](https://sonic-jenkins.westus2.cloudapp.azure.com/job/broadcom/job/buildimage-brcm-201803/)
-[![Nephos](https://sonic-jenkins.westus2.cloudapp.azure.com/job/nephos/job/buildimage-nephos-201803/badge/icon?subject=Nephos)](https://sonic-jenkins.westus2.cloudapp.azure.com/job/nephos/job/buildimage-nephos-201803/)
-[![Marvell](https://sonic-jenkins.westus2.cloudapp.azure.com/job/marvell/job/buildimage-mrvl-201803/badge/icon?subject=Marvell)](https://sonic-jenkins.westus2.cloudapp.azure.com/job/marvell/job/buildimage-mrvl-201803/)
-[![Mellanox](https://sonic-jenkins.westus2.cloudapp.azure.com/job/mellanox/job/buildimage-mlnx-201803/badge/icon?subject=Mellanox)](https://sonic-jenkins.westus2.cloudapp.azure.com/job/mellanox/job/buildimage-mlnx-201803/)
-
 # sonic-buildimage
 
 ## Build SONiC Switch Images
 
 # Description
 
-Following is the instruction on how to build an [(ONIE)](https://github.com/opencomputeproject/onie) compatible network operating system (NOS) installer image for network switches, and also how to build docker images running inside the NOS. Note that SONiC image are build per ASIC platform. Switches using the same ASIC platform share a common image. For a list of supported switches and ASIC, please refer to this [list](https://github.com/sonic-net/SONiC/wiki/Supported-Devices-and-Platforms)
+Following are the instructions on how to build an [(ONIE)](https://github.com/opencomputeproject/onie) compatible network operating system (NOS) installer image for network switches, and also how to build docker images running inside the NOS. Note that SONiC images are build per ASIC platform. Switches using the same ASIC platform share a common image. For a list of supported switches and ASIC, please refer to this [list](https://github.com/sonic-net/SONiC/wiki/Supported-Devices-and-Platforms)
 
 # Hardware
 
@@ -156,12 +143,14 @@ To build SONiC installer image and docker images, run the following commands:
 - PLATFORM=vs
 
 ## Usage for ARM Architecture
-To build Arm32 bit for (ARMHF) platform
-    ARM build has dependency in docker version 18,
-    if docker version is 19, downgrade to 18 as below
-    sudo apt-get install --allow-downgrades  -y docker-ce=5:18.09.0~3-0~ubuntu-xenial
-    sudo apt-get install --allow-downgrades  -y docker-ce-cli=5:18.09.0~3-0~ubuntu-xenial
 
+ARM build has dependency in docker version 18. If docker version is 19, downgrade to 18 with:
+```
+sudo apt-get install --allow-downgrades -y docker-ce=5:18.09.0~3-0~ubuntu-xenial
+sudo apt-get install --allow-downgrades -y docker-ce-cli=5:18.09.0~3-0~ubuntu-xenial
+```
+To build Arm32 bit for (ARMHF) platform
+
     # Execute make configure once to configure ASIC and ARCH
 
     make configure PLATFORM=[ASIC_VENDOR] PLATFORM_ARCH=armhf
@@ -174,7 +163,7 @@ To build Arm32 bit for (ARMHF) platform
 
     make target/sonic-marvell-armhf.bin
 
-To build Arm32 bit for (ARMHF) Marvell platform on amd64 host for debian buster using cross-compilation run the following commands:
+To build Arm32 bit for (ARMHF) Marvell platform on amd64 host for debian buster using cross-compilation, run the following commands:
 
     # Execute make configure once to configure ASIC and ARCH for cross-compilation build
 
@@ -239,7 +228,7 @@ Every target has a clean target, so in order to clean swss, execute:
 It is recommended to use clean targets to clean all packages that are built together, like dev packages for instance. In order to be more familiar with build process and make some changes to it, it is recommended to read this short [Documentation](README.buildsystem.md).
 
 ## Build debug dockers and debug SONiC installer image:
-SONiC build system supports building dockers and ONIE-image with debug tools and debug symbols, to help with live & core debugging. For details refer to [(SONiC Buildimage Guide)](https://github.com/sonic-net/sonic-buildimage/blob/master/README.buildsystem.md).
+SONiC build system supports building dockers and ONIE-image with debug tools and debug symbols, to help with live & core debugging. For details refer to [SONiC Buildimage Guide](https://github.com/sonic-net/sonic-buildimage/blob/master/README.buildsystem.md).
 
 ## SAI Version
 Please refer to [SONiC roadmap](https://github.com/sonic-net/SONiC/wiki/Sonic-Roadmap-Planning) on the SAI version for each SONiC release.

device/arista/x86_64-arista_7050cx3_32s/Arista-7050CX3-32S-D48C8/qos.json.j2

@@ -170,7 +170,7 @@
             "3": "3",
             "4": "4",
             "5": "5",
-            "6": "6",
+            "6": "1",
             "7": "7",
             "8": "1"
         },

device/common/profiles/th2/7260/BALANCED/qos.json.j2

@@ -322,7 +322,7 @@
             "3": "3",
             "4": "4",
             "5": "5",
-            "6": "6",
+            "6": "1",
             "7": "7",
             "8": "1"
         },

device/common/profiles/th2/7260/RDMA-CENTRIC/qos.json.j2

@@ -321,7 +321,7 @@
             "3": "3",
             "4": "4",
             "5": "5",
-            "6": "6",
+            "6": "1",
             "7": "7",
             "8": "1"
         },

dockers/docker-fpm-frr/Dockerfile.j2

@@ -56,14 +56,14 @@ COPY ["TS", "/usr/bin/TS"]
 COPY ["files/supervisor-proc-exit-listener", "/usr/bin"]
 COPY ["zsocket.sh", "/usr/bin/"]
 COPY ["*.json", "/etc/rsyslog.d/"]
-COPY ["files/rsyslog_plugin.conf.j2", "/etc/rsyslog.d/"]
+# COPY ["files/rsyslog_plugin.conf.j2", "/etc/rsyslog.d/"]
 RUN chmod a+x /usr/bin/TSA && \
     chmod a+x /usr/bin/TSB && \
     chmod a+x /usr/bin/TSC && \
     chmod a+x /usr/bin/zsocket.sh
 
-RUN j2 -f json /etc/rsyslog.d/rsyslog_plugin.conf.j2 /etc/rsyslog.d/events_info.json  > /etc/rsyslog.d/bgp_events.conf
-RUN rm -f /etc/rsyslog.d/rsyslog_plugin.conf.j2*
+# RUN j2 -f json /etc/rsyslog.d/rsyslog_plugin.conf.j2 /etc/rsyslog.d/events_info.json  > /etc/rsyslog.d/bgp_events.conf
+# RUN rm -f /etc/rsyslog.d/rsyslog_plugin.conf.j2*
 RUN rm -f /etc/rsyslog.d/events_info.json*
 
 ENTRYPOINT ["/usr/bin/docker_init.sh"]

dockers/docker-macsec/cli/clear/plugins/clear_macsec_counter.py

@@ -0,0 +1,36 @@
+import os
+import click
+
+import show.plugins.macsec as show_macsec
+import utilities_common.cli as clicommon
+from sonic_py_common import multi_asic
+
+@click.group(cls=clicommon.AliasedGroup)
+def macsec():
+    pass
+
+
+@macsec.command('macsec')
+@click.option('--clean-cache', type=bool, required=False, default=False, help="If the option of clean cache is true, next show commands will show the raw counters which based on the service booted instead of the last clear command.")
+def macsec_clear_counters(clean_cache):
+    """ 
+        Clear MACsec counts.
+        This clear command will generated a cache for next show commands which will base on this cache as the zero baseline to show the increment of counters.
+    """
+
+    if clean_cache:
+        for namespace in multi_asic.get_namespace_list():
+            if os.path.isfile(show_macsec.CACHE_FILE.format(namespace)):
+                os.remove(show_macsec.CACHE_FILE.format(namespace))
+            print("Cleaned cache")
+        return
+
+    clicommon.run_command("show macsec --dump-file")
+    print("Clear MACsec counters")
+
+def register(cli):
+    cli.add_command(macsec_clear_counters)
+
+
+if __name__ == '__main__':
+    macsec_clear_counters(None)

dockers/docker-macsec/cli/show/plugins/show_macsec.py

@@ -1,12 +1,19 @@
 import typing
 from natsort import natsorted
+import datetime
+import pickle
+import os
+import copy
 
 import click
 from tabulate import tabulate
 
 import utilities_common.multi_asic as multi_asic_util
 from swsscommon.swsscommon import CounterTable, MacsecCounter
+from utilities_common.cli import UserCache
 
+CACHE_MANAGER = UserCache(app_name="macsec")
+CACHE_FILE = os.path.join(CACHE_MANAGER.get_directory(), "macsecstats{}")
 
 DB_CONNECTOR = None
 COUNTER_TABLE = None
@@ -15,12 +22,12 @@
 class MACsecAppMeta(object):
     def __init__(self, *args) -> None:
         SEPARATOR = DB_CONNECTOR.get_db_separator(DB_CONNECTOR.APPL_DB)
-        key = self.__class__.get_appl_table_name() + SEPARATOR + \
+        self.key = self.__class__.get_appl_table_name() + SEPARATOR + \
             SEPARATOR.join(args)
         self.meta = DB_CONNECTOR.get_all(
-            DB_CONNECTOR.APPL_DB, key)
+            DB_CONNECTOR.APPL_DB, self.key)
         if len(self.meta) == 0:
-            raise ValueError("No such MACsecAppMeta: {}".format(key))
+            raise ValueError("No such MACsecAppMeta: {}".format(self.key))
         for k, v in self.meta.items():
             setattr(self, k, v)
 
@@ -39,10 +46,15 @@ def __init__(self, port_name: str, sci: str, an: str) -> None:
         MACsecAppMeta.__init__(self, port_name, sci, an)
         MACsecCounters.__init__(self, port_name, sci, an)
 
-    def dump_str(self) -> str:
+    def dump_str(self, cache = None) -> str:
         buffer = self.get_header()
         meta = sorted(self.meta.items(), key=lambda x: x[0])
-        counters = sorted(self.counters.items(), key=lambda x: x[0])
+        counters = copy.deepcopy(self.counters)
+        if cache:
+            for k, v in counters.items():
+                if k in cache.counters:
+                    counters[k] = int(counters[k]) - int(cache.counters[k])
+        counters = sorted(counters.items(), key=lambda x: x[0])
         buffer += tabulate(meta + counters)
         buffer = "\n".join(["\t\t" + line for line in buffer.splitlines()])
         return buffer
@@ -87,7 +99,7 @@ def __init__(self, port_name: str, sci: str) -> None:
     def get_appl_table_name(cls) -> str:
         return "MACSEC_INGRESS_SC_TABLE"
 
-    def dump_str(self) -> str:
+    def dump_str(self, cache = None) -> str:
         buffer = self.get_header()
         buffer = "\n".join(["\t" + line for line in buffer.splitlines()])
         return buffer
@@ -104,7 +116,7 @@ def __init__(self, port_name: str, sci: str) -> None:
     def get_appl_table_name(cls) -> str:
         return "MACSEC_EGRESS_SC_TABLE"
 
-    def dump_str(self) -> str:
+    def dump_str(self, cache = None) -> str:
         buffer = self.get_header()
         buffer += tabulate(sorted(self.meta.items(), key=lambda x: x[0]))
         buffer = "\n".join(["\t" + line for line in buffer.splitlines()])
@@ -123,7 +135,7 @@ def __init__(self,  port_name: str) -> None:
     def get_appl_table_name(cls) -> str:
         return "MACSEC_PORT_TABLE"
 
-    def dump_str(self) -> str:
+    def dump_str(self, cache = None) -> str:
         buffer = self.get_header()
         buffer += tabulate(sorted(self.meta.items(), key=lambda x: x[0]))
         return buffer
@@ -149,6 +161,7 @@ def create_macsec_obj(key: str) -> MACsecAppMeta:
     except ValueError as e:
         return None
 
+
 def create_macsec_objs(interface_name: str) -> typing.List[MACsecAppMeta]:
     objs = []
     objs.append(create_macsec_obj(MACsecPort.get_appl_table_name() + ":" + interface_name))
@@ -179,12 +192,25 @@ def create_macsec_objs(interface_name: str) -> typing.List[MACsecAppMeta]:
     return objs
 
 
+def cache_find(cache: dict, target: MACsecAppMeta) -> MACsecAppMeta:
+    if not cache or not cache["objs"]:
+        return None
+    for obj in cache["objs"]:
+        if type(obj) == type(target) and obj.key == target.key:
+            # MACsec SA may be refreshed by a cycle that use the same key
+            # So, use the SA as the identifier
+            if isinstance(obj, MACsecSA) and obj.sak != target.sak:
+                continue
+            return obj
+    return None
+
+
 @click.command()
 @click.argument('interface_name', required=False)
+@click.option('--dump-file', is_flag=True, required=False, default=False)
 @multi_asic_util.multi_asic_click_options
-def macsec(interface_name, namespace, display):
-    MacsecContext(namespace, display).show(interface_name)
-
+def macsec(interface_name, dump_file, namespace, display):
+    MacsecContext(namespace, display).show(interface_name, dump_file)
 
 class MacsecContext(object):
 
@@ -194,7 +220,7 @@ def __init__(self, namespace_option, display_option):
             display_option, namespace_option)
 
     @multi_asic_util.run_on_multi_asic
-    def show(self, interface_name):
+    def show(self, interface_name, dump_file):
         global DB_CONNECTOR
         global COUNTER_TABLE
         DB_CONNECTOR = self.db
@@ -205,13 +231,29 @@ def show(self, interface_name):
             if interface_name not in interface_names:
                 return
             interface_names = [interface_name]
-
         objs = []
+
         for interface_name in natsorted(interface_names):
             objs += create_macsec_objs(interface_name)
-        for obj in objs:
-            print(obj.dump_str())
 
+        cache = {}
+        if os.path.isfile(CACHE_FILE.format(self.multi_asic.current_namespace)):
+            cache = pickle.load(open(CACHE_FILE.format(self.multi_asic.current_namespace), "rb"))
+
+        if not dump_file:
+            if cache and cache["time"] and objs:
+                print("Last cached time was {}".format(cache["time"]))
+            for obj in objs:
+                cache_obj = cache_find(cache, obj)
+                print(obj.dump_str(cache_obj))
+        else:
+            dump_obj = {
+                "time": datetime.datetime.now(),
+                "objs": objs
+            }
+            with open(CACHE_FILE.format(self.multi_asic.current_namespace), 'wb') as dump_file:
+                pickle.dump(dump_obj, dump_file)
+                dump_file.flush()
 
 def register(cli):
     cli.add_command(macsec)

files/build_templates/sonic_debian_extension.j2

@@ -800,8 +800,8 @@ sudo bash -c "echo { > $FILESYSTEM_ROOT_USR_SHARE_SONIC_TEMPLATES/ctr_image_name
 sudo bash -c "echo } >> $FILESYSTEM_ROOT_USR_SHARE_SONIC_TEMPLATES/ctr_image_names.json"
 
 # copy rsyslog plugin binary for use by all dockers that use plugin to publish events.
-sudo mkdir -p ${FILESYSTEM_ROOT_USR_SHARE_SONIC_SCRIPTS}
-sudo cp ${files_path}/rsyslog_plugin ${FILESYSTEM_ROOT_USR_SHARE_SONIC_SCRIPTS}/
+# sudo mkdir -p ${FILESYSTEM_ROOT_USR_SHARE_SONIC_SCRIPTS}
+# sudo cp ${files_path}/rsyslog_plugin ${FILESYSTEM_ROOT_USR_SHARE_SONIC_SCRIPTS}/
 
 {% for script in installer_start_scripts.split(' ') -%}
 if [ -f $TARGET_MACHINE"_{{script}}" ]; then

files/image_config/misc/docker-wait-any

@@ -46,10 +46,9 @@ g_dep_services = []
 
 
 def wait_for_container(docker_client, container_name):
-    while True:
-        while docker_client.inspect_container(container_name)['State']['Status'] != "running":
-            time.sleep(1)
+    log.log_info("Waiting on container '{}'".format(container_name))
 
+    while True:
         docker_client.wait(container_name)
 
         log.log_info("No longer waiting on container '{}'".format(container_name))

files/scripts/swss.sh

@@ -30,7 +30,7 @@ function read_dependent_services()
     fi
 
     if [[ -f ${ETC_SONIC_PATH}/${SERVICE}_multi_inst_dependent ]]; then
-        MULTI_INST_DEPENDENT="${MULTI_INST_DEPENDENT} cat ${ETC_SONIC_PATH}/${SERVICE}_multi_inst_dependent"
+        MULTI_INST_DEPENDENT="${MULTI_INST_DEPENDENT} $(cat ${ETC_SONIC_PATH}/${SERVICE}_multi_inst_dependent)"
     fi
 }
 
@@ -308,6 +308,19 @@ function check_peer_gbsyncd()
     fi
 }
 
+function check_macsec()
+{
+    MACSEC_STATE=`show feature status | grep macsec | awk '{print $2}'`
+
+    if [[ ${MACSEC_STATE} == 'enabled' ]]; then
+        if [ "$DEV" ]; then
+            DEPENDENT="${DEPENDENT} macsec@${DEV}"
+        else
+            DEPENDENT="${DEPENDENT} macsec"
+        fi
+    fi
+}
+
 if [ "$DEV" ]; then
     NET_NS="$NAMESPACE_PREFIX$DEV" #name of the network namespace
     SONIC_DB_CLI="sonic-db-cli -n $NET_NS"
@@ -319,6 +332,7 @@ else
 fi
 
 check_peer_gbsyncd
+check_macsec
 read_dependent_services
 
 case "$1" in