Merge pull request #917 from redHJ/pdr-8421

支持linux审计日志(/var/log/audit/)标准化，redhat机型

mgr/dataflow.go

@@ -387,13 +387,13 @@ func checkSampleData(sampleData []string, logParser parser.Parser) ([]string, er
 }
 
 func getTransformerCreator(transformerConfig map[string]interface{}) (transforms.Creator, error) {
-	transformKeyType, ok := transformerConfig[transforms.KeyType]
+	transformKeyType, ok := transformerConfig[KeyType]
 	if !ok {
-		return nil, fmt.Errorf("missing param %s", transforms.KeyType)
+		return nil, fmt.Errorf("missing param %s", KeyType)
 	}
 	transformKeyTypeStr, ok := transformKeyType.(string)
 	if !ok {
-		return nil, fmt.Errorf("param %s must be of type string", transforms.KeyType)
+		return nil, fmt.Errorf("param %s must be of type string", KeyType)
 	}
 
 	create, ok := transforms.Transformers[transformKeyTypeStr]

mgr/runner.go

@@ -22,6 +22,7 @@ import (
 	"github.com/qiniu/logkit/parser"
 	_ "github.com/qiniu/logkit/parser/builtin"
 	"github.com/qiniu/logkit/parser/config"
+	parserconfig "github.com/qiniu/logkit/parser/config"
 	"github.com/qiniu/logkit/parser/qiniu"
 	"github.com/qiniu/logkit/reader"
 	_ "github.com/qiniu/logkit/reader/builtin"
@@ -267,16 +268,22 @@ func NewLogExportRunner(rc RunnerConfig, cleanChan chan<- cleaner.CleanSignal, r
 			return nil, err
 		}
 	}
-	parser, err := pr.NewLogParser(rc.ParserConf)
+	ps, err := pr.NewLogParser(rc.ParserConf)
 	if err != nil {
 		return nil, err
 	}
 
+	var serverConfigs = make([]map[string]interface{}, 0, 10)
+	if serverParser, ok := ps.(parser.ServerParser); ok {
+		if serverParser.ServerConfig() != nil {
+			serverConfigs = append(serverConfigs, serverParser.ServerConfig())
+		}
+	}
+
 	transformers, err := createTransformers(rc)
 	if err != nil {
 		return nil, err
 	}
-	var serverConfigs = make([]map[string]interface{}, 0, len(transformers))
 	for _, transform := range transformers {
 		if serverTransformer, ok := transform.(transforms.ServerTansformer); ok {
 			if serverTransformer.ServerConfig() != nil {
@@ -317,7 +324,7 @@ func NewLogExportRunner(rc RunnerConfig, cleanChan chan<- cleaner.CleanSignal, r
 	if err != nil {
 		return nil, fmt.Errorf("runner %v add sender router error, %v", rc.RunnerName, err)
 	}
-	runner, err = NewLogExportRunnerWithService(runnerInfo, rd, cl, parser, transformers, senders, router, meta)
+	runner, err = NewLogExportRunnerWithService(runnerInfo, rd, cl, ps, transformers, senders, router, meta)
 	if err != nil {
 		return runner, err
 	}
@@ -335,7 +342,7 @@ func createTransformers(rc RunnerConfig) ([]transforms.Transformer, error) {
 	transformers := make([]transforms.Transformer, 0)
 	for idx := range rc.Transforms {
 		tConf := rc.Transforms[idx]
-		tp := tConf[transforms.KeyType]
+		tp := tConf[KeyType]
 		if tp == nil {
 			return nil, fmt.Errorf("transformer config type is empty %v", tConf)
 		}
@@ -1484,12 +1491,12 @@ func setPandoraServerConfig(senderConfig conf.MapConf, serverConfigs []map[strin
 
 	var err error
 	for _, serverConfig := range serverConfigs {
-		keyType, ok := serverConfig[transforms.KeyType].(string)
+		keyType, ok := serverConfig[KeyType].(string)
 		if !ok {
 			continue
 		}
 		switch keyType {
-		case ip.Name:
+		case ip.Name, parserconfig.TypeLinuxAudit:
 			if senderConfig, err = setIPConfig(senderConfig, serverConfig); err != nil {
 				return senderConfig, err
 			}
@@ -1506,13 +1513,13 @@ func setIPConfig(senderConfig conf.MapConf, serverConfig map[string]interface{})
 	}
 
 	autoCreate := senderConfig[senderConf.KeyPandoraAutoCreate]
-	transformAt, transformAtOk := serverConfig[transforms.TransformAt].(string)
-	if !transformAtOk {
+	processAt, processAtOk := serverConfig[ProcessAt].(string)
+	if !processAtOk {
 		return senderConfig, nil
 	}
 
 	senderConfig[senderConf.KeyPandoraAutoCreate] = removeServerIPSchema(senderConfig[senderConf.KeyPandoraAutoCreate], key)
-	if transformAt == ip.Local {
+	if processAt == Local {
 		return senderConfig, nil
 	}
 

mgr/runner_test.go

@@ -2036,8 +2036,8 @@ func Test_setSenderConfig(t *testing.T) {
 
 	serverConfigs := []map[string]interface{}{
 		{
-			transforms.KeyType:     ip.Name,
-			transforms.TransformAt: ip.Server,
+			KeyType:   ip.Name,
+			ProcessAt: Server,
 		},
 	}
 	actualConfig, err := setPandoraServerConfig(senderConfig, serverConfigs)
@@ -2046,9 +2046,9 @@ func Test_setSenderConfig(t *testing.T) {
 
 	serverConfigs = []map[string]interface{}{
 		{
-			transforms.KeyType:     ip.Name,
-			transforms.TransformAt: ip.Server,
-			"key": "ip",
+			KeyType:   ip.Name,
+			ProcessAt: Server,
+			"key":     "ip",
 		},
 	}
 	actualConfig, err = setPandoraServerConfig(senderConfig, serverConfigs)
@@ -2060,9 +2060,9 @@ func Test_setSenderConfig(t *testing.T) {
 	}
 	serverConfigs = []map[string]interface{}{
 		{
-			transforms.KeyType:     ip.Name,
-			transforms.TransformAt: ip.Local,
-			"key": "a.b",
+			KeyType:   ip.Name,
+			ProcessAt: Local,
+			"key":     "a.b",
 		},
 	}
 	actualConfig, err = setPandoraServerConfig(senderConfig, serverConfigs)
@@ -2071,7 +2071,7 @@ func Test_setSenderConfig(t *testing.T) {
 
 	serverConfigs = []map[string]interface{}{
 		{
-			transforms.KeyType: "other",
+			KeyType: "other",
 		},
 	}
 	actualConfig, err = setPandoraServerConfig(senderConfig, serverConfigs)
@@ -2080,9 +2080,9 @@ func Test_setSenderConfig(t *testing.T) {
 
 	serverConfigs = []map[string]interface{}{
 		{
-			transforms.KeyType:     ip.Name,
-			transforms.TransformAt: ip.Server,
-			"key": "ip.ip",
+			KeyType:   ip.Name,
+			ProcessAt: Server,
+			"key":     "ip.ip",
 		},
 	}
 	actualConfig, err = setPandoraServerConfig(senderConfig, serverConfigs)

parser/builtin/builtin.go

@@ -7,6 +7,7 @@ import (
 	_ "github.com/qiniu/logkit/parser/grok"
 	_ "github.com/qiniu/logkit/parser/json"
 	_ "github.com/qiniu/logkit/parser/kafkarest"
+	_ "github.com/qiniu/logkit/parser/linuxaudit"
 	_ "github.com/qiniu/logkit/parser/logfmt"
 	_ "github.com/qiniu/logkit/parser/mysql"
 	_ "github.com/qiniu/logkit/parser/nginx"

parser/config/config.go

@@ -60,17 +60,18 @@ const (
 // ModeUsages 和 ModeTooltips 用途说明
 var (
 	ModeUsages = KeyValueSlice{
-		{TypeRaw, "按原始日志逐行发送", ""},
-		{TypeJSON, "按 json 格式解析", ""},
-		{TypeNginx, "按 nginx 日志解析", ""},
-		{TypeGrok, "按 grok 格式解析", ""},
-		{TypeCSV, "按 csv 格式解析", ""},
-		{TypeSyslog, "按 syslog 格式解析", ""},
-		{TypeLogv1, "按七牛日志库格式解析", ""},
-		{TypeKafkaRest, "按 kafkarest 日志解析", ""},
+		{TypeRaw, "原始日志逐行发送", ""},
+		{TypeJSON, "json 格式解析", ""},
+		{TypeNginx, "nginx 日志解析", ""},
+		{TypeGrok, "grok 格式解析", ""},
+		{TypeCSV, "csv 格式解析", ""},
+		{TypeSyslog, "syslog 格式解析", ""},
+		{TypeLogv1, "七牛日志库格式解析", ""},
+		{TypeKafkaRest, "kafkarest 日志解析", ""},
 		{TypeEmpty, "通过解析清空数据", ""},
-		{TypeMySQL, "按 mysql 慢请求日志解析", ""},
+		{TypeMySQL, "mysql 慢请求日志解析", ""},
 		{TypeKeyValue, "key value 日志解析", ""},
+		{TypeLinuxAudit, "redhat 审计日志解析", ""},
 	}
 
 	ModeToolTips = KeyValueSlice{
@@ -85,6 +86,7 @@ var (
 		{TypeEmpty, "通过解析清空数据", ""},
 		{TypeMySQL, "解析mysql的慢请求日志。", ""},
 		{TypeKeyValue, "按照key value解析日志", ""},
+		{TypeLinuxAudit, "按 redhat 审计日志解析", ""},
 	}
 )
 
@@ -369,6 +371,30 @@ var ModeKeyOptions = map[string][]Option{
 		OptionKeepRawData,
 	},
 	TypeLogfmt: {
+		{
+			KeyName:      KeySplitter,
+			ChooseOnly:   false,
+			Default:      "=",
+			DefaultNoUse: false,
+			Description:  "分隔符(splitter)",
+		},
+		OptionParserName,
+		OptionDisableRecordErrData,
+		OptionKeepRawData,
+	},
+	TypeKeyValue: {
+		{
+			KeyName:      KeySplitter,
+			ChooseOnly:   false,
+			Default:      "=",
+			DefaultNoUse: false,
+			Description:  "分隔符(splitter)",
+		},
+		OptionParserName,
+		OptionDisableRecordErrData,
+		OptionKeepRawData,
+	},
+	TypeLinuxAudit: {
 		OptionParserName,
 		OptionDisableRecordErrData,
 		OptionKeepRawData,
@@ -396,4 +422,7 @@ SELECT count(*) from mysql.rds_replication_status WHERE master_host IS NOT NULL
 #`,
 	TypeLogfmt: `ts=2018-01-02T03:04:05.123Z lvl=5 msg="error" log_id=123456abc
 method=PUT duration=1.23 log_id=123456abc`,
+	TypeKeyValue: `ts=2018-01-02T03:04:05.123Z lvl=5 msg="error" log_id=123456abc
+method=PUT duration=1.23 log_id=123456abc`,
+	TypeLinuxAudit: `type=SYSCALL msg=audit(1364481363.243:24287): arch=c000003e syscall=2 success=no exit=-13 a0=7fffd19c5592 a1=0    a2=7fffd19c4b50`,
 }

parser/config/models.go

@@ -30,6 +30,7 @@ const (
 	TypeMySQL      = "mysqllog"
 	TypeLogfmt     = "logfmt"
 	TypeKeyValue   = "KV"
+	TypeLinuxAudit = "linuxaudit"
 )
 
 // 数据常量类型