Merge pull request #6 from renukamanavalan/lua_support

Add lua code and formatter for rsyslog plugin

src/sonic-eventd/Makefile

@@ -7,8 +7,7 @@ RSYSLOG-PLUGIN_TEST := rsyslog_plugin_tests/tests
 CP := cp
 MKDIR := mkdir
 CC := g++
-CP := cp
-LIBS := -levent -lhiredis -lswsscommon -lpthread -lboost_thread -lboost_system -lzmq -lboost_serialization -luuid
+LIBS := -levent -lhiredis -lswsscommon -lpthread -lboost_thread -lboost_system -lzmq -lboost_serialization -luuid -llua5.1
 TEST_LIBS := -L/usr/src/gtest -lgtest -lgtest_main -lgmock -lgmock_main
 
 CFLAGS += -Wall -std=c++17 -fPIE -I$(PWD)/../sonic-swss-common/common
@@ -26,7 +25,7 @@ endif
 -include rsyslog_plugin/subdir.mk
 -include rsyslog_plugin_tests/subdir.mk
 
-all: sonic-eventd eventd-tests eventd-tool rsyslog-plugin
+all: sonic-eventd eventd-tests eventd-tool rsyslog-plugin rsyslog-plugin-tests
 
 sonic-eventd: $(OBJS)
 	@echo 'Building target: $@'

src/sonic-eventd/rsyslog_plugin/rsyslog_plugin.cpp

@@ -2,20 +2,22 @@
 #include <vector>
 #include <fstream>
 #include <regex>
+#include <ctime>
+#include <unordered_map>
 #include "rsyslog_plugin.h"
 #include "json.hpp"
 
 using json = nlohmann::json;
 
-bool RsyslogPlugin::onMessage(string msg) {
+bool RsyslogPlugin::onMessage(string msg, lua_State* luaState) {
     string tag;
     event_params_t paramDict;
-    if(!m_parser->parseMessage(msg, tag, paramDict)) {
+    if(!m_parser->parseMessage(msg, tag, paramDict, luaState)) {
         SWSS_LOG_DEBUG("%s was not able to be parsed into a structured event\n", msg.c_str());
         return false;
     } else {
         int returnCode = event_publish(m_eventHandle, tag, &paramDict);
-        if (returnCode != 0) {
+        if(returnCode != 0) {
             SWSS_LOG_ERROR("rsyslog_plugin was not able to publish event for %s.\n", tag.c_str());
             return false;
         }
@@ -42,9 +44,14 @@ bool RsyslogPlugin::createRegexList() {
 
     for(long unsigned int i = 0; i < m_parser->m_regexList.size(); i++) {
         try {
-            regexString = m_parser->m_regexList[i]["regex"];
+	    string timestampRegex = "^([a-zA-Z]{3})?\\s*([0-9]{1,2})?\\s*([0-9]{2}:[0-9]{2}:[0-9]{2}.[0-9]{0,6})?\\s*"; 
+            string eventRegex = m_parser->m_regexList[i]["regex"];
+	    regexString = timestampRegex + eventRegex; 
             string tag = m_parser->m_regexList[i]["tag"];
             vector<string> params = m_parser->m_regexList[i]["params"];
+	    vector<string> timestampParams = { "month", "day", "time" };
+	    params.insert(params.begin(), timestampParams.begin(), timestampParams.end());
+	    m_parser->m_regexList[i]["params"] = params;
             regex expr(regexString);
             expression = expr;
         } catch (domain_error& deException) {
@@ -56,23 +63,28 @@ bool RsyslogPlugin::createRegexList() {
 	}
         m_parser->m_expressions.push_back(expression);
     }
+
     if(m_parser->m_expressions.empty()) {
         SWSS_LOG_ERROR("Empty list of regex expressions.\n");
         return false;
     }
+
     regexFile.close();
     return true;
 }
 
 void RsyslogPlugin::run() {
+    lua_State* luaState = luaL_newstate();
+    luaL_openlibs(luaState);
     while(true) {
         string line;
         getline(cin, line);
         if(line.empty()) {
             continue;
         }
-        onMessage(line);
+        onMessage(line, luaState);
     }
+    lua_close(luaState);
 }
 
 int RsyslogPlugin::onInit() {

src/sonic-eventd/rsyslog_plugin/rsyslog_plugin.h

@@ -1,6 +1,12 @@
 #ifndef RSYSLOG_PLUGIN_H
 #define RSYSLOG_PLUGIN_H
 
+extern "C"
+{
+    #include <lua5.1/lua.h>
+    #include <lua5.1/lualib.h>
+    #include <lua5.1/lauxlib.h>
+}
 #include <string>
 #include <memory>
 #include "syslog_parser.h"
@@ -19,7 +25,7 @@ using namespace swss;
 class RsyslogPlugin {
 public:
     int onInit();
-    bool onMessage(string msg);
+    bool onMessage(string msg, lua_State* luaState);
     void run();
     RsyslogPlugin(string moduleName, string regexPath);
 private:

src/sonic-eventd/rsyslog_plugin/subdir.mk

@@ -1,9 +1,9 @@
 CC := g++
 
-RSYSLOG-PLUGIN_TEST_OBJS += ./rsyslog_plugin/rsyslog_plugin.o ./rsyslog_plugin/syslog_parser.o
-RSYSLOG-PLUGIN_OBJS += ./rsyslog_plugin/rsyslog_plugin.o ./rsyslog_plugin/syslog_parser.o ./rsyslog_plugin/main.o
+RSYSLOG-PLUGIN-TEST_OBJS += ./rsyslog_plugin/rsyslog_plugin.o ./rsyslog_plugin/syslog_parser.o ./rsyslog_plugin/timestamp_formatter.o
+RSYSLOG-PLUGIN_OBJS += ./rsyslog_plugin/rsyslog_plugin.o ./rsyslog_plugin/syslog_parser.o ./rsyslog_plugin/timestamp_formatter.o ./rsyslog_plugin/main.o
 
-C_DEPS += ./rsyslog_plugin/rsyslog_plugin.d ./rsyslog_plugin/syslog_parser.d ./rsyslog_plugin/main.d
+C_DEPS += ./rsyslog_plugin/rsyslog_plugin.d ./rsyslog_plugin/syslog_parser.d ./rsyslog_plugin/timestamp_formatter.d ./rsyslog_plugin/main.d
 
 rsyslog_plugin/%.o: rsyslog_plugin/%.cpp
 	@echo 'Building file: $<'

src/sonic-eventd/rsyslog_plugin/syslog_parser.cpp

@@ -1,4 +1,5 @@
 #include <iostream>
+#include <ctime>
 #include "syslog_parser.h"
 #include "logger.h"
 
@@ -10,19 +11,59 @@
  *
 */
 
-bool SyslogParser::parseMessage(string message, string& eventTag, event_params_t& paramMap) {
+bool SyslogParser::parseMessage(string message, string& eventTag, event_params_t& paramMap, lua_State* luaState) {
     for(long unsigned int i = 0; i < m_regexList.size(); i++) {
         smatch matchResults;
         vector<string> params = m_regexList[i]["params"];
-        if(!regex_search(message, matchResults, m_expressions[i]) || params.size() != matchResults.size() - 1) {
+        if(!regex_search(message, matchResults, m_expressions[i]) || params.size() != matchResults.size() - 1 || matchResults.size() < 4) {
             continue;
         }
+
+        if(!matchResults[1].str().empty() && !matchResults[2].str().empty() && !matchResults[3].str().empty()) { // found timestamp components
+            string formattedTimestamp = m_timestampFormatter->changeTimestampFormat({ matchResults[1].str(), matchResults[2].str(), matchResults[3].str() });
+	    if(!formattedTimestamp.empty()) {
+                paramMap["timestamp"] = formattedTimestamp;
+	    } else {
+                SWSS_LOG_ERROR("Timestamp is invalid and is not able to be formatted");
+	    }
+	}
         // found matching regex
         eventTag = m_regexList[i]["tag"];
-        transform(params.begin(), params.end(), matchResults.begin() + 1, inserter(paramMap, paramMap.end()), [](string a, string b) {
-            return make_pair(a,b);
-        });
+	// check params for lua code	
+	for(long unsigned int j = 3; j < params.size(); j++) {
+            auto delimPos = params[j].find(':');
+	    string resultValue = matchResults[j + 1].str();
+	    if(delimPos == string::npos) { // no lua code
+		paramMap[params[j]] = resultValue;
+		continue;
+	    }
+            // have to execute lua script
+            string param = params[j].substr(0, delimPos);
+            string luaString = params[j].substr(delimPos + 1); 
+	    if(luaString.empty()) { // empty lua code
+                SWSS_LOG_INFO("Lua code missing after :, skipping operation");
+                paramMap[param] = resultValue;
+                continue; 
+	    }
+            const char* luaCode = luaString.c_str();
+            lua_pushstring(luaState, resultValue.c_str());
+            lua_setglobal(luaState, "arg");
+            if(luaL_dostring(luaState, luaCode) == 0) {
+                lua_pop(luaState, lua_gettop(luaState));
+	    } else {
+		SWSS_LOG_ERROR("Invalid lua code, unable to do operation.\n");
+		paramMap[param] = resultValue;
+		continue;
+            }
+            lua_getglobal(luaState, "ret");
+            paramMap[param] = lua_tostring(luaState, -1);
+            lua_pop(luaState, 1);
+	}
         return true;
     }
     return false;
 }
+
+SyslogParser::SyslogParser() {
+    m_timestampFormatter = unique_ptr<TimestampFormatter>(new TimestampFormatter());
+}

src/sonic-eventd/rsyslog_plugin/syslog_parser.h

@@ -1,11 +1,19 @@
 #ifndef SYSLOG_PARSER_H
 #define SYSLOG_PARSER_H
 
+extern "C"
+{
+    #include <lua5.1/lua.h>
+    #include <lua5.1/lualib.h>
+    #include <lua5.1/lauxlib.h>
+}
+
 #include <vector>
 #include <string>
 #include <regex>
 #include "json.hpp"
 #include "events.h"
+#include "timestamp_formatter.h"
 
 using namespace std;
 using json = nlohmann::json;
@@ -18,9 +26,11 @@ using json = nlohmann::json;
 
 class SyslogParser {
 public:
+    unique_ptr<TimestampFormatter> m_timestampFormatter;
     vector<regex> m_expressions;
     json m_regexList = json::array();
-    bool parseMessage(string message, string& tag, event_params_t& paramDict);
+    bool parseMessage(string message, string& tag, event_params_t& paramDict, lua_State* luaState);
+    SyslogParser();
 };
 
 #endif

src/sonic-eventd/rsyslog_plugin/timestamp_formatter.cpp

@@ -0,0 +1,74 @@
+#include <iostream>
+#include "timestamp_formatter.h"
+#include "logger.h"
+#include "events.h"
+
+using namespace std;
+
+/***
+ *
+ * Formats given string into string needed by YANG model
+ *
+ * @param timestamp parsed from syslog message
+ * @return formatted timestamp that conforms to YANG model
+ *
+ */
+
+static const unordered_map<string, string> g_monthDict {
+    { "Jan", "01" },
+    { "Feb", "02" },
+    { "Mar", "03" },
+    { "Apr", "04" },
+    { "May", "05" },
+    { "Jun", "06" },
+    { "Jul", "07" },
+    { "Aug", "08" },
+    { "Sep", "09" },
+    { "Oct", "10" },
+    { "Nov", "11" },
+    { "Dec", "12" }
+};
+
+string TimestampFormatter::getYear(string timestamp) {
+    if(!m_storedTimestamp.empty()) {
+        if(m_storedTimestamp.compare(timestamp) <= 0) {
+            m_storedTimestamp = timestamp;
+            return m_storedYear;
+        }
+    }
+    // no last timestamp or year change
+    time_t currentTime = time(nullptr);
+    tm* const localTime = localtime(&currentTime);
+    stringstream ss;
+    auto currentYear = 1900 + localTime->tm_year;
+    ss << currentYear; // get current year
+    string year = ss.str();
+    m_storedTimestamp = timestamp;
+    m_storedYear = year;
+    return year;
+}
+
+string TimestampFormatter::changeTimestampFormat(vector<string> dateComponents) {
+    if(dateComponents.size() < 3) {
+        SWSS_LOG_ERROR("Timestamp formatter unable to format due to invalid input");
+        return "";
+    }
+    string formattedTimestamp; // need to change format of Mmm dd hh:mm:ss.SSSSSS to YYYY-mm-ddThh:mm:ss.SSSSSSZ
+    string month;
+    auto it = g_monthDict.find(dateComponents[0]);
+    if(it != g_monthDict.end()) {
+        month = it->second;
+    } else {
+        SWSS_LOG_ERROR("Timestamp month was given in wrong format.\n");
+        return "";
+    }
+    string day = dateComponents[1];
+    if(day.size() == 1) { // convert 1 -> 01
+       day.insert(day.begin(), '0');
+    }
+    string time = dateComponents[2];
+    string currentTimestamp = month + day + time;
+    string year = getYear(currentTimestamp);
+    formattedTimestamp = year + "-" + month + "-" + day + "T" + time + "Z";
+    return formattedTimestamp;
+}

src/sonic-eventd/rsyslog_plugin/timestamp_formatter.h

@@ -0,0 +1,27 @@
+#ifndef TIMESTAMP_FORMATTER_H
+#define TIMESTAMP_FORMATTER_H
+
+#include <iostream>
+#include <string>
+#include <regex>
+#include <ctime>
+#include <vector>
+
+using namespace std;
+
+/***
+ *
+ * TimestampFormatter is responsible for formatting the timestamps received in syslog messages and to format them into the type needed by YANG model
+ *
+ */
+
+class TimestampFormatter {
+public:
+    string changeTimestampFormat(vector<string> dateComponents);
+    string m_storedTimestamp;
+    string m_storedYear;
+private:
+    string getYear(string timestamp);
+};
+
+#endif