Merge pull request #715 from wonderflow/fixlogk

make qiniu log parser more flexiable
qiniu · Aug 17, 2018 · fab1e92 · fab1e92
2 parents c322d4c + cd97c7f
commit fab1e92
Show file tree

Hide file tree

Showing 6 changed files with 187 additions and 132 deletions.
diff --git a/mgr/runner.go b/mgr/runner.go
@@ -20,6 +20,7 @@ import (
 	"github.com/qiniu/logkit/conf"
 	"github.com/qiniu/logkit/parser"
 	_ "github.com/qiniu/logkit/parser/builtin"
+	"github.com/qiniu/logkit/parser/qiniu"
 	"github.com/qiniu/logkit/reader"
 	_ "github.com/qiniu/logkit/reader/builtin"
 	"github.com/qiniu/logkit/reader/cloudtrail"
@@ -89,7 +90,6 @@ type LogExportRunner struct {
 }
 
 const defaultSendIntervalSeconds = 60
-const qiniulogHeadPatthern = "[1-9]\\d{3}/[0-1]\\d/[0-3]\\d [0-2]\\d:[0-6]\\d:[0-6]\\d(\\.\\d{6})?"
 
 // NewRunner 创建Runner
 func NewRunner(rc RunnerConfig, cleanChan chan<- cleaner.CleanSignal) (runner Runner, err error) {
@@ -1043,13 +1043,13 @@ func Compatible(rc RunnerConfig) RunnerConfig {
 	}
 	pattern, _ := rc.ReaderConfig.GetStringOr(reader.KeyHeadPattern, "")
 	if parserType == parser.TypeLogv1 && pattern == "" {
-		prefix, _ := rc.ParserConf.GetStringOr(parser.KeyQiniulogPrefix, "")
+		prefix, _ := rc.ParserConf.GetStringOr(qiniu.KeyPrefix, "")
 		prefix = strings.TrimSpace(prefix)
 		var readpattern string
 		if len(prefix) > 0 {
-			readpattern = "^" + prefix + " " + qiniulogHeadPatthern
+			readpattern = "^" + prefix + " " + qiniu.HeadPatthern
 		} else {
-			readpattern = "^" + qiniulogHeadPatthern
+			readpattern = "^" + qiniu.HeadPatthern
 		}
 		rc.ReaderConfig[reader.KeyHeadPattern] = readpattern
 	}

diff --git a/mgr/runner_test.go b/mgr/runner_test.go
@@ -19,6 +19,7 @@ import (
 	"github.com/qiniu/logkit/cleaner"
 	"github.com/qiniu/logkit/conf"
 	"github.com/qiniu/logkit/parser"
+	"github.com/qiniu/logkit/parser/qiniu"
 	"github.com/qiniu/logkit/reader"
 	"github.com/qiniu/logkit/router"
 	"github.com/qiniu/logkit/sender"
@@ -287,7 +288,7 @@ func Test_RunForEnvTag(t *testing.T) {
 		t.Error(err)
 	}
 	senderConfigs := []conf.MapConf{
-		conf.MapConf{
+		{
 			"name":        "mock_sender",
 			"sender_type": "mock",
 		},
@@ -517,7 +518,7 @@ func Test_Compatible(t *testing.T) {
 			"mode":           "dir",
 			"read_from":      "oldest",
 			"datasource_tag": "testtag",
-			"head_pattern":   "^" + qiniulogHeadPatthern,
+			"head_pattern":   "^" + qiniu.HeadPatthern,
 		},
 		ParserConf: conf.MapConf{
 			"type": "qiniulog",
@@ -545,7 +546,7 @@ func Test_Compatible(t *testing.T) {
 			"mode":           "dir",
 			"read_from":      "oldest",
 			"datasource_tag": "testtag",
-			"head_pattern":   "^PREX " + qiniulogHeadPatthern,
+			"head_pattern":   "^PREX " + qiniu.HeadPatthern,
 		},
 		ParserConf: conf.MapConf{
 			"type":            "qiniulog",
@@ -593,9 +594,11 @@ func Test_QiniulogRun(t *testing.T) {
 2016/10/20 17:20:30.642662 [123][WARN] disk.go:241: github.com/qiniu/logkit/queue/disk.go 1
 `
 	log3 := `2016/10/20 17:20:30.642662 [124][WARN] disk.go:456: xxxxxx`
-	expfiles := []string{`[REQ_END] 200 0.010k 3.792ms \t\t[WARN][SLdoIrCDZj7pmZsU] disk.go <job.freezeDeamon> pop() failed: not found`,
+	expfiles := []string{`[REQ_END] 200 0.010k 3.792ms
+		[WARN][SLdoIrCDZj7pmZsU] disk.go <job.freezeDeamon> pop() failed: not found`,
 		`Service: POST 10.200.20.25:9100/user/info, Code: 200, Xlog: AC, Time: 1ms`,
-		`github.com/qiniu/logkit/queue/disk.go:241 \t1234 3243xsaxs`, `github.com/qiniu/logkit/queue/disk.go 1`,
+		`github.com/qiniu/logkit/queue/disk.go:241
+	1234 3243xsaxs`, `github.com/qiniu/logkit/queue/disk.go 1`,
 		`xxxxxx`}
 	expreqid := []string{"X-ZsU", "2pyKMukqvwSd-ZsU", "", "123", "124"}
 	if err := ioutil.WriteFile(filepath.Join(logpath, "log1"), []byte(log1), 0666); err != nil {
@@ -697,7 +700,11 @@ func Test_QiniulogRun(t *testing.T) {
 	}
 	for idx, dt := range dts {
 		assert.Equal(t, expfiles[idx], dt["log"], "equl log test")
-		assert.Equal(t, expreqid[idx], dt["reqid"], "equal reqid test")
+		if expreqid[idx] == "" {
+			assert.Nil(t, dt["reqid"])
+		} else {
+			assert.Equal(t, expreqid[idx], dt["reqid"], "equal reqid test")
+		}
 	}
 	ls, err := runner.LagStats()
 	assert.NoError(t, err)

diff --git a/parser/mysql/mysql.go b/parser/mysql/mysql.go
@@ -1,24 +1,16 @@
 package mysql
 
 import (
-	"regexp"
 	"strings"
 
 	"github.com/Preetam/mysqllog"
 
 	"github.com/qiniu/logkit/conf"
 	"github.com/qiniu/logkit/parser"
-	"github.com/qiniu/logkit/parser/qiniu"
 	. "github.com/qiniu/logkit/utils/models"
 )
 
 func init() {
-	qiniu.CompliedPatterns = make(map[string]*regexp.Regexp)
-	for k, v := range qiniu.HeaderPattern {
-		c, _ := regexp.Compile(v)
-		qiniu.CompliedPatterns[k] = c
-	}
-
 	parser.RegisterConstructor(parser.TypeMySQL, NewParser)
 }