Move parsers outside of filestream input so others can use them as well

filebeat/input/filestream/config.go

@@ -25,6 +25,7 @@ import (
 
 	"github.com/elastic/beats/v7/libbeat/common"
 	"github.com/elastic/beats/v7/libbeat/common/match"
+	"github.com/elastic/beats/v7/libbeat/reader/parser"
 	"github.com/elastic/beats/v7/libbeat/reader/readfile"
 )
 
@@ -71,7 +72,7 @@ type readerConfig struct {
 	MaxBytes       int                     `config:"message_max_bytes" validate:"min=0,nonzero"`
 	Tail           bool                    `config:"seek_to_tail"`
 
-	Parsers []common.ConfigNamespace `config:"parsers"`
+	Parsers parser.Config `config:",inline"`
 }
 
 type backoffConfig struct {
@@ -127,7 +128,6 @@ func defaultReaderConfig() readerConfig {
 		LineTerminator: readfile.AutoLineTerminator,
 		MaxBytes:       10 * humanize.MiByte,
 		Tail:           false,
-		Parsers:        make([]common.ConfigNamespace, 0),
 	}
 }
 
@@ -136,9 +136,5 @@ func (c *config) Validate() error {
 		return fmt.Errorf("no path is configured")
 	}
 
-	if err := validateParserConfig(parserConfig{maxBytes: c.Reader.MaxBytes, lineTerminator: c.Reader.LineTerminator}, c.Reader.Parsers); err != nil {
-		return fmt.Errorf("cannot parse parser configuration: %+v", err)
-	}
-
 	return nil
 }

filebeat/input/filestream/input.go

@@ -35,6 +35,7 @@ import (
 	"github.com/elastic/beats/v7/libbeat/logp"
 	"github.com/elastic/beats/v7/libbeat/reader"
 	"github.com/elastic/beats/v7/libbeat/reader/debug"
+	"github.com/elastic/beats/v7/libbeat/reader/parser"
 	"github.com/elastic/beats/v7/libbeat/reader/readfile"
 	"github.com/elastic/beats/v7/libbeat/reader/readfile/encoding"
 )
@@ -57,7 +58,7 @@ type filestream struct {
 	encodingFactory encoding.EncodingFactory
 	encoding        encoding.Encoding
 	closerConfig    closerConfig
-	parserConfig    []common.ConfigNamespace
+	parsers         parser.Config
 }
 
 // Plugin creates a new filestream input plugin for creating a stateful input.
@@ -97,6 +98,7 @@ func configure(cfg *common.Config) (loginp.Prospector, loginp.Harvester, error)
 		readerConfig:    config.Reader,
 		encodingFactory: encodingFactory,
 		closerConfig:    config.Close,
+		parsers:         config.Reader.Parsers,
 	}
 
 	return prospector, filestream, nil
@@ -219,10 +221,7 @@ func (inp *filestream) open(log *logp.Logger, canceler input.Canceler, fs fileSo
 
 	r = readfile.NewFilemeta(r, fs.newPath)
 
-	r, err = newParsers(r, parserConfig{maxBytes: inp.readerConfig.MaxBytes, lineTerminator: inp.readerConfig.LineTerminator}, inp.readerConfig.Parsers)
-	if err != nil {
-		return nil, err
-	}
+	r = inp.parsers.Create(r)
 
 	r = readfile.NewLimitReader(r, inp.readerConfig.MaxBytes)
 

filebeat/input/filestream/parser.go → libbeat/reader/parser/parser.go

@@ -15,13 +15,15 @@
 // specific language governing permissions and limitations
 // under the License.
 
-package filestream
+package parser
 
 import (
 	"errors"
 	"fmt"
 	"io"
 
+	"github.com/dustin/go-humanize"
+
 	"github.com/elastic/beats/v7/libbeat/common"
 	"github.com/elastic/beats/v7/libbeat/reader"
 	"github.com/elastic/beats/v7/libbeat/reader/multiline"
@@ -35,20 +37,48 @@ var (
 
 // parser transforms or translates the Content attribute of a Message.
 // They are able to aggregate two or more Messages into a single one.
-type parser interface {
+type Parser interface {
 	io.Closer
 	Next() (reader.Message, error)
 }
 
-type parserConfig struct {
-	maxBytes       int
-	lineTerminator readfile.LineTerminator
+type CommonConfig struct {
+	MaxBytes       int                     `config:"max_bytes"`
+	LineTerminator readfile.LineTerminator `config:"line_terminator"`
 }
 
-func newParsers(in reader.Reader, pCfg parserConfig, c []common.ConfigNamespace) (parser, error) {
-	p := in
+type Config struct {
+	pCfg    CommonConfig
+	parsers []common.ConfigNamespace
+}
+
+func (c *Config) Unpack(cc *common.Config) error {
+	tmp := struct {
+		Common  CommonConfig             `config:",inline"`
+		Parsers []common.ConfigNamespace `config:"parsers"`
+	}{
+		CommonConfig{
+			MaxBytes:       10 * humanize.MiByte,
+			LineTerminator: readfile.AutoLineTerminator,
+		},
+		nil,
+	}
+	err := cc.Unpack(&tmp)
+	if err != nil {
+		return err
+	}
 
-	for _, ns := range c {
+	newC, err := NewConfig(tmp.Common, tmp.Parsers)
+	if err != nil {
+		return err
+	}
+	*c = *newC
+
+	return nil
+}
+
+func NewConfig(pCfg CommonConfig, parsers []common.ConfigNamespace) (*Config, error) {
+	for _, ns := range parsers {
 		name := ns.Name()
 		switch name {
 		case "multiline":
@@ -58,63 +88,68 @@ func newParsers(in reader.Reader, pCfg parserConfig, c []common.ConfigNamespace)
 			if err != nil {
 				return nil, fmt.Errorf("error while parsing multiline parser config: %+v", err)
 			}
-			p, err = multiline.New(p, "\n", pCfg.maxBytes, &config)
-			if err != nil {
-				return nil, fmt.Errorf("error while creating multiline parser: %+v", err)
-			}
 		case "ndjson":
 			var config readjson.ParserConfig
 			cfg := ns.Config()
 			err := cfg.Unpack(&config)
 			if err != nil {
 				return nil, fmt.Errorf("error while parsing ndjson parser config: %+v", err)
 			}
-			p = readjson.NewJSONParser(p, &config)
 		case "container":
 			config := readjson.DefaultContainerConfig()
 			cfg := ns.Config()
 			err := cfg.Unpack(&config)
 			if err != nil {
 				return nil, fmt.Errorf("error while parsing container parser config: %+v", err)
 			}
-			p = readjson.NewContainerParser(p, &config)
 		default:
 			return nil, fmt.Errorf("%s: %s", ErrNoSuchParser, name)
 		}
 	}
 
-	return p, nil
+	return &Config{
+		pCfg:    pCfg,
+		parsers: parsers,
+	}, nil
+
 }
 
-func validateParserConfig(pCfg parserConfig, c []common.ConfigNamespace) error {
-	for _, ns := range c {
+func (c *Config) Create(in reader.Reader) Parser {
+	p := in
+	for _, ns := range c.parsers {
 		name := ns.Name()
 		switch name {
 		case "multiline":
 			var config multiline.Config
 			cfg := ns.Config()
 			err := cfg.Unpack(&config)
 			if err != nil {
-				return fmt.Errorf("error while parsing multiline parser config: %+v", err)
+				return p
+			}
+			p, err = multiline.New(p, "\n", c.pCfg.MaxBytes, &config)
+			if err != nil {
+				return p
 			}
 		case "ndjson":
-			var config readjson.Config
+			var config readjson.ParserConfig
 			cfg := ns.Config()
 			err := cfg.Unpack(&config)
 			if err != nil {
-				return fmt.Errorf("error while parsing ndjson parser config: %+v", err)
+				return p
 			}
+			p = readjson.NewJSONParser(p, &config)
 		case "container":
 			config := readjson.DefaultContainerConfig()
 			cfg := ns.Config()
 			err := cfg.Unpack(&config)
 			if err != nil {
-				return fmt.Errorf("error while parsing container parser config: %+v", err)
+				return p
 			}
+			p = readjson.NewContainerParser(p, &config)
 		default:
-			return fmt.Errorf("%s: %s", ErrNoSuchParser, name)
+			return p
 		}
 	}
 
-	return nil
+	return p
 }

libbeat/reader/parser/parser_example_test.go

@@ -0,0 +1,94 @@
+// Licensed to Elasticsearch B.V. under one or more contributor
+// license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright
+// ownership. Elasticsearch B.V. licenses this file to you under
+// the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package parser
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/require"
+
+	"github.com/elastic/beats/v7/libbeat/common"
+	"github.com/elastic/beats/v7/libbeat/reader/readfile"
+)
+
+type inputParsersConfig struct {
+	MaxBytes       int                     `config:"max_bytes"`
+	LineTerminator readfile.LineTerminator `config:"line_terminator"`
+	Parsers        Config                  `config:",inline"`
+}
+
+func TestParsersExampleInline(t *testing.T) {
+	tests := map[string]struct {
+		lines            string
+		parsers          map[string]interface{}
+		expectedMessages []string
+	}{
+		"multiline docker logs parser": {
+			lines: `{"log":"[log] The following are log messages\n","stream":"stdout","time":"2016-03-02T22:58:51.338462311Z"}
+{"log":"[log] This one is\n","stream":"stdout","time":"2016-03-02T22:58:51.338462311Z"}
+{"log":" on multiple\n","stream":"stdout","time":"2016-03-02T22:58:51.338462311Z"}
+{"log":" lines","stream":"stdout","time":"2016-03-02T22:58:51.338462311Z"}
+{"log":"[log] In total there should be 3 events\n","stream":"stdout","time":"2016-03-02T22:58:51.338462311Z"}
+`,
+			parsers: map[string]interface{}{
+				"max_bytes":       1024,
+				"line_terminator": "auto",
+				"parsers": []map[string]interface{}{
+					map[string]interface{}{
+						"ndjson": map[string]interface{}{
+							"keys_under_root": true,
+							"message_key":     "log",
+						},
+					},
+					map[string]interface{}{
+						"multiline": map[string]interface{}{
+							"match":   "after",
+							"negate":  true,
+							"pattern": "^\\[log\\]",
+						},
+					},
+				},
+			},
+			expectedMessages: []string{
+				"[log] The following are log messages\n",
+				"[log] This one is\n\n on multiple\n\n lines",
+				"[log] In total there should be 3 events\n",
+			},
+		},
+	}
+
+	for name, test := range tests {
+		test := test
+		t.Run(name, func(t *testing.T) {
+			cfg := common.MustNewConfigFrom(test.parsers)
+			var c inputParsersConfig
+			err := cfg.Unpack(&c)
+			require.NoError(t, err)
+
+			p := c.Parsers.Create(testReader(test.lines))
+
+			i := 0
+			msg, err := p.Next()
+			for err == nil {
+				require.Equal(t, test.expectedMessages[i], string(msg.Content))
+				i++
+				msg, err = p.Next()
+			}
+		})
+	}
+}