change type of max_bytes to ByteType (#26699)

* change type of max_bytes to ByteType allows for both number and humanize values in the config. Without this max_bytes in awss3 input could only be numbers.
elastic · Jul 7, 2021 · 2af8ab9 · 2af8ab9
1 parent 584ae74
commit 2af8ab9
Show file tree

Hide file tree

Showing 3 changed files with 34 additions and 3 deletions.
diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc
@@ -281,6 +281,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Fix default config template values for paths on oracle module: {pull}26276[26276]
 - Do not close filestream harvester if an unexpected error is returned when close.on_state_change.* is enabled. {pull}26411[26411]
 - Fix Elasticsearch compatibility for modules that use `copy_from` in `set` processors. {issue}26629[26629]
+- Change type of max_bytes in all configs to be cfgtype.ByteSize {pull}26699[26699]
 
 *Filebeat*
 

diff --git a/libbeat/reader/parser/parser.go b/libbeat/reader/parser/parser.go
@@ -25,6 +25,7 @@ import (
 	"github.com/dustin/go-humanize"
 
 	"github.com/elastic/beats/v7/libbeat/common"
+	"github.com/elastic/beats/v7/libbeat/common/cfgtype"
 	"github.com/elastic/beats/v7/libbeat/reader"
 	"github.com/elastic/beats/v7/libbeat/reader/multiline"
 	"github.com/elastic/beats/v7/libbeat/reader/readfile"
@@ -43,7 +44,7 @@ type Parser interface {
 }
 
 type CommonConfig struct {
-	MaxBytes       int                     `config:"max_bytes"`
+	MaxBytes       cfgtype.ByteSize        `config:"max_bytes"`
 	LineTerminator readfile.LineTerminator `config:"line_terminator"`
 }
 
@@ -126,7 +127,7 @@ func (c *Config) Create(in reader.Reader) Parser {
 			if err != nil {
 				return p
 			}
-			p, err = multiline.New(p, "\n", c.pCfg.MaxBytes, &config)
+			p, err = multiline.New(p, "\n", int(c.pCfg.MaxBytes), &config)
 			if err != nil {
 				return p
 			}

diff --git a/libbeat/reader/parser/parser_example_test.go b/libbeat/reader/parser/parser_example_test.go
@@ -23,11 +23,12 @@ import (
 	"github.com/stretchr/testify/require"
 
 	"github.com/elastic/beats/v7/libbeat/common"
+	"github.com/elastic/beats/v7/libbeat/common/cfgtype"
 	"github.com/elastic/beats/v7/libbeat/reader/readfile"
 )
 
 type inputParsersConfig struct {
-	MaxBytes       int                     `config:"max_bytes"`
+	MaxBytes       cfgtype.ByteSize        `config:"max_bytes"`
 	LineTerminator readfile.LineTerminator `config:"line_terminator"`
 	Parsers        Config                  `config:",inline"`
 }
@@ -70,6 +71,34 @@ func TestParsersExampleInline(t *testing.T) {
 				"[log] In total there should be 3 events\n",
 			},
 		},
+		"humanize max_bytes, multiline XML": {
+			lines: `<Event><Data>
+	A
+	B
+	C</Data></Event>
+<Event><Data>
+	D
+	E
+	F</Data></Event>
+`,
+			parsers: map[string]interface{}{
+				"max_bytes":       "4 KiB",
+				"line_terminator": "auto",
+				"parsers": []map[string]interface{}{
+					map[string]interface{}{
+						"multiline": map[string]interface{}{
+							"match":   "after",
+							"negate":  true,
+							"pattern": "^<Event",
+						},
+					},
+				},
+			},
+			expectedMessages: []string{
+				"<Event><Data>\n\n\tA\n\n\tB\n\n\tC</Data></Event>\n",
+				"<Event><Data>\n\n\tD\n\n\tE\n\n\tF</Data></Event>\n",
+			},
+		},
 	}
 
 	for name, test := range tests {