[rsyslog]: Use RELP instead of UDP for forwarding from container to host

build_debian.sh

@@ -358,6 +358,7 @@ sudo LANG=C DEBIAN_FRONTEND=noninteractive chroot $FILESYSTEM_ROOT apt-get -y in
     squashfs-tools          \
     $bootloader_packages    \
     rsyslog                 \
+    rsyslog-relp            \
     screen                  \
     hping3                  \
     tcptraceroute           \

dockers/docker-base-bookworm/Dockerfile.j2

@@ -47,6 +47,7 @@ RUN apt update &&            \
         python-is-python3    \
         vim-tiny             \
         rsyslog              \
+        rsyslog-relp         \
 # Install redis-tools
         redis-tools          \
 # common dependencies

dockers/docker-base-bookworm/etc/rsyslog.conf

@@ -9,24 +9,20 @@
 #### MODULES ####
 #################
 
-$ModLoad imuxsock # provides support for local system logging
-
 #
 # Set a rate limit on messages from the container
 #
-$SystemLogRateLimitInterval 300
-$SystemLogRateLimitBurst 20000
-
-#$ModLoad imklog  # provides kernel logging support
-#$ModLoad immark  # provides --MARK-- message capability
+module(load="imuxsock" SysSock.RateLimit.Interval="300" SysSock.RateLimit.Burst="20000") # provides support for local system logging
+#module(load="imklog")   # provides kernel logging support
+#module(load="immark")  # provides --MARK-- message capability
 
 # provides UDP syslog reception
-#$ModLoad imudp
-#$UDPServerRun 514
+#module(load="imudp")
+#input(type="imudp" port="514")
 
 # provides TCP syslog reception
-#$ModLoad imtcp
-#$InputTCPServerRun 514
+#module(load="imtcp")
+#input(type="imtcp" port="514")
 
 
 ###########################
@@ -37,7 +33,8 @@ set $.CONTAINER_NAME=getenv("CONTAINER_NAME");
 
 # Set remote syslog server
 template (name="ForwardFormatInContainer" type="string" string="<%PRI%>%TIMESTAMP:::date-rfc3339% %HOSTNAME% %$.CONTAINER_NAME%#%syslogtag%%msg:::sp-if-no-1st-sp%%msg%")
-*.* action(type="omfwd" target=`echo $SYSLOG_TARGET_IP` port="514" protocol="udp" Template="ForwardFormatInContainer")
+module(load="omrelp")
+*.* action(type="omrelp" target=`echo $SYSLOG_TARGET_IP` port="2514" action.resumeRetryCount="-1" queue.type="LinkedList" queue.size="20000" Template="ForwardFormatInContainer")
 
 #
 # Use traditional timestamp format.
@@ -75,4 +72,4 @@ $RepeatedMsgReduction on
 
 ###############
 #### RULES ####
-###############
+###############

dockers/docker-base-bullseye/Dockerfile.j2

@@ -62,7 +62,7 @@ RUN apt-get update &&        \
 
 # default rsyslog version is 8.2110.0 which has a bug on log rate limit,
 # use backport version 8.2206.0-1~bpo11+1
-RUN apt-get -t bullseye-backports -y install rsyslog
+RUN apt-get -t bullseye-backports -y install rsyslog rsyslog-relp
 
 # Upgrade pip via PyPI and uninstall the Debian version
 RUN pip3 install --upgrade pip

dockers/docker-base-bullseye/etc/rsyslog.conf

@@ -9,24 +9,20 @@
 #### MODULES ####
 #################
 
-$ModLoad imuxsock # provides support for local system logging
-
 #
 # Set a rate limit on messages from the container
 #
-$SystemLogRateLimitInterval 300
-$SystemLogRateLimitBurst 20000
-
-#$ModLoad imklog  # provides kernel logging support
-#$ModLoad immark  # provides --MARK-- message capability
+module(load="imuxsock" SysSock.RateLimit.Interval="300" SysSock.RateLimit.Burst="20000") # provides support for local system logging
+#module(load="imklog")   # provides kernel logging support
+#module(load="immark")  # provides --MARK-- message capability
 
 # provides UDP syslog reception
-#$ModLoad imudp
-#$UDPServerRun 514
+#module(load="imudp")
+#input(type="imudp" port="514")
 
 # provides TCP syslog reception
-#$ModLoad imtcp
-#$InputTCPServerRun 514
+#module(load="imtcp")
+#input(type="imtcp" port="514")
 
 
 ###########################
@@ -37,7 +33,8 @@ set $.CONTAINER_NAME=getenv("CONTAINER_NAME");
 
 # Set remote syslog server
 template (name="ForwardFormatInContainer" type="string" string="<%PRI%>%TIMESTAMP:::date-rfc3339% %HOSTNAME% %$.CONTAINER_NAME%#%syslogtag%%msg:::sp-if-no-1st-sp%%msg%")
-*.* action(type="omfwd" target=`echo $SYSLOG_TARGET_IP` port="514" protocol="udp" Template="ForwardFormatInContainer")
+module(load="omrelp")
+*.* action(type="omrelp" target=`echo $SYSLOG_TARGET_IP` port="2514" action.resumeRetryCount="-1" queue.type="LinkedList" queue.size="20000" Template="ForwardFormatInContainer")
 
 #
 # Use traditional timestamp format.
@@ -75,4 +72,4 @@ $RepeatedMsgReduction on
 
 ###############
 #### RULES ####
-###############
+###############

dockers/docker-platform-monitor/etc/rsyslog.conf

@@ -9,24 +9,20 @@
 #### MODULES ####
 #################
 
-$ModLoad imuxsock # provides support for local system logging
-
 #
 # Set a rate limit on messages from the container
 #
-$SystemLogRateLimitInterval 300
-$SystemLogRateLimitBurst 20000
-
-#$ModLoad imklog  # provides kernel logging support
-#$ModLoad immark  # provides --MARK-- message capability
+module(load="imuxsock" SysSock.RateLimit.Interval="300" SysSock.RateLimit.Burst="20000") # provides support for local system logging
+#module(load="imklog")   # provides kernel logging support
+#module(load="immark")  # provides --MARK-- message capability
 
 # provides UDP syslog reception
-#$ModLoad imudp
-#$UDPServerRun 514
+#module(load="imudp")
+#input(type="imudp" port="514")
 
 # provides TCP syslog reception
-#$ModLoad imtcp
-#$InputTCPServerRun 514
+#module(load="imtcp")
+#input(type="imtcp" port="514")
 
 
 ###########################
@@ -47,7 +43,8 @@ if $programname contains "sensord" and $msg contains "Error getting sensor data:
 
 # Set remote syslog server
 template (name="ForwardFormatInContainer" type="string" string="<%PRI%>%TIMESTAMP:::date-rfc3339% %HOSTNAME% %$.CONTAINER_NAME%#%syslogtag%%msg:::sp-if-no-1st-sp%%msg%")
-*.* action(type="omfwd" target=`echo $SYSLOG_TARGET_IP` port="514" protocol="udp" Template="ForwardFormatInContainer")
+module(load="omrelp")
+*.* action(type="omrelp" target=`echo $SYSLOG_TARGET_IP` port="2514" action.resumeRetryCount="-1" queue.type="LinkedList" queue.size="20000" Template="ForwardFormatInContainer")
 
 #
 # Use traditional timestamp format.
@@ -85,4 +82,4 @@ $RepeatedMsgReduction on
 
 ###############
 #### RULES ####
-###############
+###############

files/image_config/rsyslog/rsyslog-container.conf.j2

@@ -9,8 +9,6 @@
 #### MODULES ####
 #################
 
-$ModLoad imuxsock # provides support for local system logging
-
 #
 # Set a rate limit on messages from the container
 #
@@ -26,27 +24,17 @@ $ModLoad imuxsock # provides support for local system logging
 {% endif %}
 {% endif %}
 
-{% if rate_limit_interval is defined %}
-$SystemLogRateLimitInterval {{ rate_limit_interval }}
-{% else %}
-$SystemLogRateLimitInterval 300
-{% endif %}
-{% if rate_limit_burst is defined %}
-$SystemLogRateLimitBurst {{ rate_limit_burst }}
-{% else %}
-$SystemLogRateLimitBurst 20000
-{% endif %}
-
-#$ModLoad imklog  # provides kernel logging support
-#$ModLoad immark  # provides --MARK-- message capability
+module(load="imuxsock" SysSock.RateLimit.Interval="{{ rate_limit_interval|default('300') }}" SysSock.RateLimit.Burst="{{ rate_limit_burst|default('20000') }}") # provides support for local system logging
+#module(load="imklog")   # provides kernel logging support
+#module(load="immark")  # provides --MARK-- message capability
 
 # provides UDP syslog reception
-#$ModLoad imudp
-#$UDPServerRun 514
+#module(load="imudp")
+#input(type="imudp" port="514")
 
 # provides TCP syslog reception
-#$ModLoad imtcp
-#$InputTCPServerRun 514
+#module(load="imtcp")
+#input(type="imtcp" port="514")
 
 
 ###########################
@@ -71,17 +59,8 @@ if ($.PLATFORM == "x86_64-mlnx_msn2700-r0" or $.PLATFORM == "x86_64-mlnx_msn2700
 
 # Set remote syslog server
 template (name="ForwardFormatInContainer" type="string" string="<%PRI%>%TIMESTAMP:::date-rfc3339% %HOSTNAME% %$.CONTAINER_NAME%#%syslogtag%%msg:::sp-if-no-1st-sp%%msg%")
-*.* action(type="omfwd" target=`echo $SYSLOG_TARGET_IP` port="514" protocol="udp" Template="ForwardFormatInContainer")
-
-#
-# Use traditional timestamp format.
-# To enable high precision timestamps, comment out the following line.
-#
-#$ActionFileDefaultTemplate RSYSLOG_TraditionalFileFormat
-
-# Define a custom template
-$template SONiCFileFormat,"%TIMESTAMP%.%timestamp:::date-subseconds% %HOSTNAME% %syslogseverity-text:::uppercase% %$.CONTAINER_NAME%#%syslogtag%%msg:::sp-if-no-1st-sp%%msg:::drop-last-lf%\n"
-$ActionFileDefaultTemplate SONiCFileFormat
+module(load="omrelp")
+*.* action(type="omrelp" target=`echo $SYSLOG_TARGET_IP` port="2514" action.resumeRetryCount="-1" queue.type="LinkedList" queue.size="20000" Template="ForwardFormatInContainer")
 
 #
 # Set the default permissions for all log files.
@@ -109,4 +88,4 @@ $RepeatedMsgReduction on
 
 ###############
 #### RULES ####
-###############
+###############

files/image_config/rsyslog/rsyslog.conf.j2

@@ -13,34 +13,28 @@
 #### MODULES ####
 #################
 
-$ModLoad imuxsock # provides support for local system logging
-
 {% set gconf = (SYSLOG_CONFIG | d({})).get('GLOBAL', {}) -%}
-{% set rate_limit_interval = gconf.get('rate_limit_interval') %}
-{% set rate_limit_burst = gconf.get('rate_limit_burst') %}
+{% set rate_limit_interval = gconf.get('rate_limit_interval') -%}
+{% set rate_limit_burst = gconf.get('rate_limit_burst') -%}
 
-{% if rate_limit_interval is not none %}
-$SystemLogRateLimitInterval {{ rate_limit_interval }}
-{% endif %}
-{% if rate_limit_burst is not none %}
-$SystemLogRateLimitBurst {{ rate_limit_burst }}
-{% endif %}
-
-$ModLoad imklog   # provides kernel logging support
-#$ModLoad immark  # provides --MARK-- message capability
+module(load="imuxsock" {% if rate_limit_interval is not none %}SysSock.RateLimit.Interval="{{ rate_limit_interval }}"{% endif %} {% if rate_limit_burst is not none %}SysSock.RateLimit.Burst="{{ rate_limit_burst }}"{% endif %}) # provides support for local system logging
+module(load="imklog")   # provides kernel logging support
+#module(load="immark")  # provides --MARK-- message capability
 
 # provides UDP syslog reception
-$ModLoad imudp
-$UDPServerAddress {{udp_server_ip}}  #bind to localhost before udp server run
-$UDPServerRun 514
-{% if docker0_ip and docker0_ip != "" %}
-$UDPServerAddress {{docker0_ip}}
-$UDPServerRun 514
-{% endif%}
+#module(load="imudp")
+#input(type="imudp" port="514")
 
 # provides TCP syslog reception
-#$ModLoad imtcp
-#$InputTCPServerRun 514
+#module(load="imtcp")
+#input(type="imtcp" port="514")
+
+# provides RELP syslog reception
+module(load="imrelp")
+input(type="imrelp" address="{{udp_server_ip}}" port="2514")
+{% if docker0_ip and docker0_ip != "" %}
+input(type="imrelp" address="{{docker0_ip}}" port="2514")
+{% endif%}
 
 
 ###########################

src/sonic-config-engine/tests/sample_output/py3/rsyslog.conf

@@ -13,21 +13,21 @@
 #### MODULES ####
 #################
 
-$ModLoad imuxsock # provides support for local system logging
-
-
-
-$ModLoad imklog   # provides kernel logging support
-#$ModLoad immark  # provides --MARK-- message capability
+module(load="imuxsock"  ) # provides support for local system logging
+module(load="imklog")   # provides kernel logging support
+#module(load="immark")  # provides --MARK-- message capability
 
 # provides UDP syslog reception
-$ModLoad imudp
-$UDPServerAddress 1.1.1.1  #bind to localhost before udp server run
-$UDPServerRun 514
+#module(load="imudp")
+#input(type="imudp" port="514")
 
 # provides TCP syslog reception
-#$ModLoad imtcp
-#$InputTCPServerRun 514
+#module(load="imtcp")
+#input(type="imtcp" port="514")
+
+# provides RELP syslog reception
+module(load="imrelp")
+input(type="imrelp" address="1.1.1.1" port="2514")
 
 
 ###########################

src/sonic-config-engine/tests/sample_output/py3/rsyslog_with_docker0.conf

@@ -13,23 +13,22 @@
 #### MODULES ####
 #################
 
-$ModLoad imuxsock # provides support for local system logging
-
-
-
-$ModLoad imklog   # provides kernel logging support
-#$ModLoad immark  # provides --MARK-- message capability
+module(load="imuxsock"  ) # provides support for local system logging
+module(load="imklog")   # provides kernel logging support
+#module(load="immark")  # provides --MARK-- message capability
 
 # provides UDP syslog reception
-$ModLoad imudp
-$UDPServerAddress 1.1.1.1  #bind to localhost before udp server run
-$UDPServerRun 514
-$UDPServerAddress 2.2.2.2
-$UDPServerRun 514
+#module(load="imudp")
+#input(type="imudp" port="514")
 
 # provides TCP syslog reception
-#$ModLoad imtcp
-#$InputTCPServerRun 514
+#module(load="imtcp")
+#input(type="imtcp" port="514")
+
+# provides RELP syslog reception
+module(load="imrelp")
+input(type="imrelp" address="1.1.1.1" port="2514")
+input(type="imrelp" address="2.2.2.2" port="2514")
 
 
 ###########################

src/sonic-config-engine/tests/test_j2files.py

@@ -865,8 +865,8 @@ def test_rsyslog_conf(self):
             pattern = r'^action.*Device="eth0".*'
             for line in file:
                 assert not bool(re.match(pattern, line.strip())), "eth0 is not allowed in Mgfx device"
-        self.assertTrue(utils.cmp(os.path.join(self.test_dir, 'sample_output', utils.PYvX_DIR, 'rsyslog.conf'),
-                                  self.output_file))
+        expected = os.path.join(self.test_dir, 'sample_output', utils.PYvX_DIR, 'rsyslog.conf')
+        self.assertTrue(utils.cmp(expected, self.output_file), self.run_diff(expected, self.output_file))
 
     def test_rsyslog_conf_docker0_ip(self):
         if utils.PYvX_DIR != 'py3':
@@ -881,8 +881,8 @@ def test_rsyslog_conf_docker0_ip(self):
 
         argument = ['-j', config_db_json, '-t', conf_template, '-a', additional_data]
         self.run_script(argument, output_file=self.output_file)
-        self.assertTrue(utils.cmp(os.path.join(self.test_dir, 'sample_output', utils.PYvX_DIR,
-                                               'rsyslog_with_docker0.conf'), self.output_file))
+        expected = os.path.join(self.test_dir, 'sample_output', utils.PYvX_DIR, 'rsyslog_with_docker0.conf')
+        self.assertTrue(utils.cmp(expected, self.output_file), self.run_diff(expected, self.output_file))
 
     def tearDown(self):
         os.environ["CFGGEN_UNIT_TESTING"] = ""