Streaming structured events implementation

dockers/docker-eventd/Dockerfile.j2

@@ -0,0 +1,36 @@
+{% from "dockers/dockerfile-macros.j2" import install_debian_packages, install_python_wheels, copy_files %}
+FROM docker-config-engine-bullseye-{{DOCKER_USERNAME}}:{{DOCKER_USERTAG}}
+
+ARG docker_container_name
+ARG image_version
+RUN [ -f /etc/rsyslog.conf ] && sed -ri "s/%syslogtag%/$docker_container_name#%syslogtag%/;" /etc/rsyslog.conf
+
+# Make apt-get non-interactive
+ENV DEBIAN_FRONTEND=noninteractive
+
+# Pass the image_version to container
+ENV IMAGE_VERSION=$image_version
+
+# Update apt's cache of available packages
+RUN apt-get update
+
+{% if docker_eventd_debs.strip() -%}
+# Copy built Debian packages
+{{ copy_files("debs/", docker_eventd_debs.split(' '), "/debs/") }}
+
+# Install built Debian packages and implicitly install their dependencies
+{{ install_debian_packages(docker_eventd_debs.split(' ')) }}
+{%- endif %}
+
+# Clean up
+RUN apt-get clean -y         && \
+    apt-get autoclean -y     && \
+    apt-get autoremove -y    && \
+    rm -rf /debs
+
+COPY ["start.sh", "/usr/bin/"]
+COPY ["supervisord.conf", "/etc/supervisor/conf.d/"]
+COPY ["files/supervisor-proc-exit-listener", "/usr/bin"]
+COPY ["critical_processes", "/etc/supervisor"]
+
+ENTRYPOINT ["/usr/local/bin/supervisord"]

dockers/docker-eventd/critical_processes

@@ -0,0 +1 @@
+program:eventd

dockers/docker-eventd/start.sh

@@ -0,0 +1,6 @@
+#!/usr/bin/env bash
+
+if [ "${RUNTIME_OWNER}" == "" ]; then
+    RUNTIME_OWNER="kube"
+fi
+

dockers/docker-eventd/supervisord.conf

@@ -0,0 +1,52 @@
+[supervisord]
+logfile_maxbytes=1MB
+logfile_backups=2
+nodaemon=true
+
+[eventlistener:dependent-startup]
+command=python3 -m supervisord_dependent_startup
+autostart=true
+autorestart=unexpected
+startretries=0
+exitcodes=0,3
+events=PROCESS_STATE
+buffer_size=1024
+
+[eventlistener:supervisor-proc-exit-listener]
+command=/usr/bin/supervisor-proc-exit-listener --container-name eventd
+events=PROCESS_STATE_EXITED,PROCESS_STATE_RUNNING
+autostart=true
+autorestart=unexpected
+buffer_size=1024
+
+[program:rsyslogd]
+command=/usr/sbin/rsyslogd -n -iNONE
+priority=1
+autostart=false
+autorestart=unexpected
+stdout_logfile=syslog
+stderr_logfile=syslog
+dependent_startup=true
+
+[program:start]
+command=/usr/bin/start.sh
+priority=2
+autostart=false
+autorestart=false
+startsecs=0
+stdout_logfile=syslog
+stderr_logfile=syslog
+dependent_startup=true
+dependent_startup_wait_for=rsyslogd:running
+
+
+[program:eventd]
+command=/usr/sbin/eventd
+priority=3
+autostart=false
+autorestart=false
+stdout_logfile=syslog
+stderr_logfile=syslog
+dependent_startup=true
+dependent_startup_wait_for=start:exited
+

files/build_templates/eventd.service.j2

@@ -0,0 +1,17 @@
+[Unit]
+Description=EVENTD container
+Requires=updategraph.service
+After=updategraph.service
+BindsTo=sonic.target
+After=sonic.target
+StartLimitIntervalSec=1200
+StartLimitBurst=3
+
+[Service]
+ExecStartPre=/usr/bin/{{docker_container_name}}.sh start
+ExecStart=/usr/bin/{{docker_container_name}}.sh wait
+ExecStop=/usr/bin/{{docker_container_name}}.sh stop
+RestartSec=30
+
+[Install]
+WantedBy=sonic.target

files/build_templates/init_cfg.json.j2

@@ -34,6 +34,7 @@
                    ("pmon", "enabled", false, "enabled"),
                    ("radv", "enabled", false, "enabled"),
                    ("snmp", "enabled", true, "enabled"),
+                   ("eventd", "enabled", true, "enabled"),
                    ("swss", "enabled", false, "enabled"),
                    ("syncd", "enabled", false, "enabled"),
                    ("teamd", "enabled", false, "enabled")] %}
@@ -58,7 +59,7 @@
             "has_per_asic_scope": {% if feature + '@.service' in installer_services.split(' ') %}true{% else %}false{% endif %},
             "auto_restart": "{{autorestart}}",
 {%- if include_kubernetes == "y" %}
-{%- if feature in ["lldp", "pmon", "radv", "snmp", "telemetry"] %}
+{%- if feature in ["lldp", "pmon", "radv", "eventd", "snmp", "telemetry"] %}
             "set_owner": "kube", {% else %}
             "set_owner": "local", {% endif %} {% endif %}
             "high_mem_alert": "disabled"

files/build_templates/sonic_debian_extension.j2

@@ -806,6 +806,7 @@ sudo LANG=C cp $SCRIPTS_DIR/bgp.sh $FILESYSTEM_ROOT/usr/local/bin/bgp.sh
 sudo LANG=C cp $SCRIPTS_DIR/teamd.sh $FILESYSTEM_ROOT/usr/local/bin/teamd.sh
 sudo LANG=C cp $SCRIPTS_DIR/lldp.sh $FILESYSTEM_ROOT/usr/local/bin/lldp.sh
 sudo LANG=C cp $SCRIPTS_DIR/radv.sh $FILESYSTEM_ROOT/usr/local/bin/radv.sh
+sudo LANG=C cp $SCRIPTS_DIR/eventd.sh $FILESYSTEM_ROOT/usr/local/bin/eventd.sh
 sudo LANG=C cp $SCRIPTS_DIR/asic_status.sh $FILESYSTEM_ROOT/usr/local/bin/asic_status.sh
 sudo LANG=C cp $SCRIPTS_DIR/asic_status.py $FILESYSTEM_ROOT/usr/local/bin/asic_status.py
 

rules/docker-eventd.dep

@@ -0,0 +1,11 @@
+
+DPATH       := $($(DOCKER_EVENTD)_PATH)
+DEP_FILES   := $(SONIC_COMMON_FILES_LIST) rules/docker-eventd.mk rules/docker-eventd.dep   
+DEP_FILES   += $(SONIC_COMMON_BASE_FILES_LIST)
+DEP_FILES   += $(shell git ls-files $(DPATH))
+
+$(DOCKER_EVENTD)_CACHE_MODE  := GIT_CONTENT_SHA 
+$(DOCKER_EVENTD)_DEP_FLAGS   := $(SONIC_COMMON_FLAGS_LIST)
+$(DOCKER_EVENTD)_DEP_FILES   := $(DEP_FILES)
+
+$(eval $(call add_dbg_docker,$(DOCKER_EVENTD),$(DOCKER_EVENTD_DBG)))

rules/docker-eventd.mk

@@ -0,0 +1,38 @@
+# docker image for eventd
+
+DOCKER_EVENTD_STEM = docker-eventd
+DOCKER_EVENTD = $(DOCKER_EVENTD_STEM).gz
+DOCKER_EVENTD_DBG = $(DOCKER_EVENTD_STEM)-$(DBG_IMAGE_MARK).gz
+
+$(DOCKER_EVENTD)_DEPENDS += $(SONIC_EVENTD)
+
+$(DOCKER_EVENTD)_DBG_DEPENDS = $($(DOCKER_CONFIG_ENGINE_BULLSEYE)_DBG_DEPENDS)
+$(DOCKER_EVENTD)_DBG_DEPENDS += $(SONIC_EVENTD_DBG) $(LIBSWSSCOMMON_DBG)
+
+$(DOCKER_EVENTD)_DBG_IMAGE_PACKAGES = $($(DOCKER_CONFIG_ENGINE_BULLSEYE)_DBG_IMAGE_PACKAGES)
+
+$(DOCKER_EVENTD)_LOAD_DOCKERS = $(DOCKER_CONFIG_ENGINE_BULLSEYE)
+
+$(DOCKER_EVENTD)_PATH = $(DOCKERS_PATH)/$(DOCKER_EVENTD_STEM)
+
+$(DOCKER_EVENTD)_INSTALL_PYTHON_WHEELS = $(SONIC_UTILITIES_PY3)
+$(DOCKER_EVENTD)_INSTALL_DEBS = $(PYTHON3_SWSSCOMMON)
+
+$(DOCKER_EVENTD)_VERSION = 1.0.0
+$(DOCKER_EVENTD)_PACKAGE_NAME = eventd
+
+$(DOCKER_DHCP)_SERVICE_REQUIRES = updategraph
+$(DOCKER_DHCP)_SERVICE_AFTER = database
+
+SONIC_DOCKER_IMAGES += $(DOCKER_EVENTD)
+SONIC_INSTALL_DOCKER_IMAGES += $(DOCKER_EVENTD)
+
+SONIC_DOCKER_DBG_IMAGES += $(DOCKER_EVENTD_DBG)
+SONIC_INSTALL_DOCKER_DBG_IMAGES += $(DOCKER_EVENTD_DBG)
+
+$(DOCKER_EVENTD)_CONTAINER_NAME = eventd
+$(DOCKER_EVENTD)_RUN_OPT += --privileged -t
+$(DOCKER_EVENTD)_RUN_OPT += -v /etc/sonic:/etc/sonic:ro
+
+SONIC_BULLSEYE_DOCKERS += $(DOCKER_EVENTD)
+SONIC_BULLSEYE_DBG_DOCKERS += $(DOCKER_EVENTD_DBG)

rules/eventd.dep

@@ -0,0 +1,10 @@
+
+SPATH       := $($(SONIC_EVENTD)_SRC_PATH)
+DEP_FILES   := $(SONIC_COMMON_FILES_LIST) rules/eventd.mk rules/eventd.dep   
+DEP_FILES   += $(SONIC_COMMON_BASE_FILES_LIST)
+DEP_FILES := $(addprefix $(SPATH)/,$(shell cd $(SPATH) && git ls-files))
+
+$(SONIC_EVENTD)_CACHE_MODE  := GIT_CONTENT_SHA 
+$(SONIC_EVENTD)_DEP_FLAGS   := $(SONIC_COMMON_FLAGS_LIST)
+$(SONIC_EVENTD)_DEP_FILES   := $(DEP_FILES)
+

rules/eventd.mk

@@ -0,0 +1,19 @@
+# eventd package
+
+SONIC_EVENTD_VERSION = 1.0.0-0
+SONIC_EVENTD_PKG_NAME = eventd
+
+SONIC_EVENTD = sonic-$(SONIC_EVENTD_PKG_NAME)_$(SONIC_EVENTD_VERSION)_$(CONFIGURED_ARCH).deb
+$(SONIC_EVENTD)_SRC_PATH = $(SRC_PATH)/sonic-eventd
+$(SONIC_EVENTD)_DEPENDS += $(LIBSWSSCOMMON)  $(LIBSWSSCOMMON_DEV)
+
+SONIC_DPKG_DEBS += $(SONIC_EVENTD)
+
+SONIC_EVENTD_DBG = sonic-$(SONIC_EVENTD_PKG_NAME)-dbgsym_$(SONIC_EVENTD_VERSION)_$(CONFIGURED_ARCH).deb
+$(eval $(call add_derived_package,$(SONIC_EVENTD),$(SONIC_EVENTD_DBG)))
+
+# The .c, .cpp, .h & .hpp files under src/{$DBG_SRC_ARCHIVE list}
+# are archived into debug one image to facilitate debugging.
+#
+DBG_SRC_ARCHIVE += sonic-eventd
+

src/sonic-eventd/Makefile

@@ -0,0 +1,83 @@
+RM := rm -rf
+EVENTD_TARGET := eventd
+EVENTD_TEST := tests/tests
+EVENTD_TOOL := tools/events_tool
+RSYSLOG-PLUGIN_TARGET := rsyslog_plugin
+RSYSLOG-PLUGIN_TEST: rsyslog_plugin_tests/tests
+CP := cp
+MKDIR := mkdir
+CC := g++
+MV := mv
+LIBS := -levent -lhiredis -lswsscommon -lpthread -lboost_thread -lboost_system -lzmq -lboost_serialization -luuid
+TEST_LIBS := -L/usr/src/gtest -lgtest -lgtest_main -lgmock -lgmock_main
+
+CFLAGS += -Wall -std=c++17 -fPIE -I$(PWD)/../sonic-swss-common/common
+PWD := $(shell pwd)
+
+ifneq ($(MAKECMDGOALS),clean)
+ifneq ($(strip $(C_DEPS)),)
+-include $(C_DEPS) $(OBJS)
+endif
+endif
+
+-include src/subdir.mk
+-include tests/subdir.mk
+-include tools/subdir.mk
+-include rsyslog_plugin/subdir.mk
+-include rsyslog_plugin_tests/subdir.mk
+
+all: sonic-eventd eventd-tests eventd-tool rsyslog-plugin rsyslog-plugin-tests
+
+sonic-eventd: $(OBJS)
+	@echo 'Building target: $@'
+	@echo 'Invoking: G++ Linker'
+	$(CC) $(LDFLAGS) -o $(EVENTD_TARGET) $(OBJS) $(LIBS)
+	@echo 'Finished building target: $@'
+	@echo ' '
+
+eventd-tool: $(TOOL_OBJS)
+	@echo 'Building target: $@'
+	@echo 'Invoking: G++ Linker'
+	$(CC) $(LDFLAGS) -o $(EVENTD_TOOL) $(TOOL_OBJS) $(LIBS)
+	@echo 'Finished building target: $@'
+	@echo ' '
+
+rsyslog-plugin: $(RSYSLOG-PLUGIN_OBJS)
+	@echo 'Buidling Target: $@'
+	@echo 'Invoking: G++ Linker'
+	$(CC) $(LDFLAGS) -o $(RSYSLOG-PLUGIN_TARGET) $(RSYSLOG-PLUGIN_OBJS) $(LIBS)
+	@echo 'Finished building target: $@'
+	@echo ' '
+
+eventd-tests: $(TEST_OBJS)
+	@echo 'Building target: $@'
+	@echo 'Invoking: G++ Linker'
+	$(CC) $(LDFLAGS) -o $(EVENTD_TEST) $(TEST_OBJS) $(LIBS) $(TEST_LIBS)
+	@echo 'Finished building target: $@'
+	$(EVENTD_TEST)
+	@echo 'Finished running tests'
+	@echo ' '
+
+rsyslog-plugin-tests: $(RSYSLOG-PLUGIN-TEST_OBJS)
+	@echo 'BUILDING target: $@'
+	@echo 'Invoking G++ Linker'
+	$(CC) $(LDFLAGS) -o $(RSYSLOG-PLUGIN_TEST) $(RSYSLOG-PLUGIN-TEST_OBJS) $(LIBS) $(TEST_LIBS)
+	@echo 'Finished building target: $@'
+	$(RSYSLOG-PLUGIN_TEST)
+	@echo 'Finished running tests'
+	@echo ' '
+
+install:
+	$(MKDIR) -p $(DESTDIR)/usr/sbin
+	$(MV) $(EVENTD_TARGET) $(DESTDIR)/usr/sbin
+	$(MV) $(EVENTD_TOOL) $(DESTDIR)/usr/sbin
+
+deinstall:
+	$(RM) $(DESTDIR)/usr/sbin/$(EVENTD_TARGET)
+	$(RM) $(DESTDIR)/usr/sbin/$(RSYSLOG-PLUGIN_TARGET)
+	$(RM) -rf $(DESTDIR)/usr/sbin
+
+clean:
+	-@echo ' '
+
+.PHONY: all clean dependents

src/sonic-eventd/debian/changelog

@@ -0,0 +1,5 @@
+sonic-eventd (1.0.0-0) UNRELEASED; urgency=medium
+
+  * Initial release.
+
+-- Renuka Manavalan <remanava@microsoft.com>  

src/sonic-eventd/debian/compat

@@ -0,0 +1 @@
+12

src/sonic-eventd/debian/control

@@ -0,0 +1,14 @@
+Source: sonic-eventd
+Section: devel
+Priority: optional
+Maintainer: Renuka Manavalan <remanava@microsoft.com>
+Build-Depends: debhelper (>= 12.0.0), libevent-dev, libboost-thread-dev, libboost-system-dev, libswsscommon-dev
+Standards-Version: 3.9.3
+Homepage: https://github.com/Azure/sonic-buildimage
+XS-Go-Import-Path: github.com/Azure/sonic-buildimage
+
+Package: sonic-eventd
+Architecture: any
+Built-Using: ${misc:Built-Using}
+Depends: ${shlibs:Depends}
+Description: SONiC event service

src/sonic-eventd/debian/rules

@@ -0,0 +1,6 @@
+#!/usr/bin/make -f
+
+export DEB_BUILD_MAINT_OPTIONS=hardening=+all
+
+%:
+	dh $@ --parallel

src/sonic-eventd/rsyslog_plugin/main.cpp

@@ -0,0 +1,57 @@
+#include <iostream>
+#include <memory>
+#include <unistd.h>
+#include "rsyslog_plugin.h"
+
+#define SUCCESS_CODE 0
+#define INVALID_REGEX_ERROR_CODE 1
+#define EVENT_INIT_PUBLISH_ERROR_CODE 2
+#define MISSING_ARGS_ERROR_CODE 3
+
+void showUsage() {
+    cout << "Usage for rsyslog_plugin: \n" << "options\n"
+        << "\t-r,required,type=string\t\tPath to regex file\n"
+        << "\t-m,required,type=string\t\tYANG module name of source generating syslog message\n"
+        << "\t-h                     \t\tHelp"
+        << endl;
+}
+
+int main(int argc, char** argv) {
+    string regexPath;
+    string moduleName;
+    int optionVal;
+
+    while((optionVal = getopt(argc, argv, "r:m:h")) != -1) {
+        switch(optionVal) {
+            case 'r':
+                regexPath = optarg;
+                break;
+            case 'm':
+                moduleName = optarg;
+                break;
+            case 'h':
+            case '?':
+            default:
+                showUsage();
+                return 1;
+        }
+    }
+
+    if(regexPath.empty() || moduleName.empty()) { // Missing required rc path
+        cerr << "Error: Missing regexPath and moduleName." << endl;
+        return MISSING_ARGS_ERROR_CODE;
+    }
+
+    unique_ptr<RsyslogPlugin> plugin(new RsyslogPlugin(moduleName, regexPath));
+    int returnCode = plugin->onInit();
+    if(returnCode == INVALID_REGEX_ERROR_CODE) {
+        SWSS_LOG_ERROR("Rsyslog plugin was not able to be initialized due to invalid regex file provided.\n");
+        return returnCode;
+    } else if(returnCode == EVENT_INIT_PUBLISH_ERROR_CODE) {
+        SWSS_LOG_ERROR("Rsyslog plugin was not able to be initialized due to event_init_publish call failing.\n");
+	return returnCode;
+    }
+
+    plugin->run();
+    return SUCCESS_CODE;
+}