change type of max_bytes to ByteType (#26699)

leehinman · mergify-bot · commit f9f9f74a283c · 2021-07-07T15:11:15.000Z
* change type of max_bytes to ByteType allows for both number and humanize values in the config. Without this max_bytes in awss3 input could only be numbers. (cherry picked from commit 2af8ab9)
diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc
@@ -220,6 +220,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Fix default config template values for paths on oracle module: {pull}26276[26276]
 - Do not close filestream harvester if an unexpected error is returned when close.on_state_change.* is enabled. {pull}26411[26411]
 - Fix Elasticsearch compatibility for modules that use `copy_from` in `set` processors. {issue}26629[26629]
+- Change type of max_bytes in all configs to be cfgtype.ByteSize {pull}26699[26699]
 
 *Filebeat*
 
diff --git a/libbeat/reader/parser/parser.go b/libbeat/reader/parser/parser.go
@@ -25,6 +25,7 @@ import (
 	"github.com/dustin/go-humanize"
 
 	"github.com/elastic/beats/v7/libbeat/common"
+	"github.com/elastic/beats/v7/libbeat/common/cfgtype"
 	"github.com/elastic/beats/v7/libbeat/reader"
 	"github.com/elastic/beats/v7/libbeat/reader/multiline"
 	"github.com/elastic/beats/v7/libbeat/reader/readfile"
@@ -43,7 +44,7 @@ type Parser interface {
 }
 
 type CommonConfig struct {
-	MaxBytes       int                     `config:"max_bytes"`
+	MaxBytes       cfgtype.ByteSize        `config:"max_bytes"`
 	LineTerminator readfile.LineTerminator `config:"line_terminator"`
 }
 
@@ -126,7 +127,7 @@ func (c *Config) Create(in reader.Reader) Parser {
 			if err != nil {
 				return p
 			}
-			p, err = multiline.New(p, "\n", c.pCfg.MaxBytes, &config)
+			p, err = multiline.New(p, "\n", int(c.pCfg.MaxBytes), &config)
 			if err != nil {
 				return p
 			}
diff --git a/libbeat/reader/parser/parser_example_test.go b/libbeat/reader/parser/parser_example_test.go
@@ -23,11 +23,12 @@ import (
 	"github.com/stretchr/testify/require"
 
 	"github.com/elastic/beats/v7/libbeat/common"
+	"github.com/elastic/beats/v7/libbeat/common/cfgtype"
 	"github.com/elastic/beats/v7/libbeat/reader/readfile"
 )
 
 type inputParsersConfig struct {
-	MaxBytes       int                     `config:"max_bytes"`
+	MaxBytes       cfgtype.ByteSize        `config:"max_bytes"`
 	LineTerminator readfile.LineTerminator `config:"line_terminator"`
 	Parsers        Config                  `config:",inline"`
 }
@@ -70,6 +71,34 @@ func TestParsersExampleInline(t *testing.T) {
 				"[log] In total there should be 3 events\n",
 			},
 		},
+		"humanize max_bytes, multiline XML": {
+			lines: `<Event><Data>
+	A
+	B
+	C</Data></Event>
+<Event><Data>
+	D
+	E
+	F</Data></Event>
+`,
+			parsers: map[string]interface{}{
+				"max_bytes":       "4 KiB",
+				"line_terminator": "auto",
+				"parsers": []map[string]interface{}{
+					map[string]interface{}{
+						"multiline": map[string]interface{}{
+							"match":   "after",
+							"negate":  true,
+							"pattern": "^<Event",
+						},
+					},
+				},
+			},
+			expectedMessages: []string{
+				"<Event><Data>\n\n\tA\n\n\tB\n\n\tC</Data></Event>\n",
+				"<Event><Data>\n\n\tD\n\n\tE\n\n\tF</Data></Event>\n",
+			},
+		},
 	}
 
 	for name, test := range tests {

Original file line number	Diff line number	Diff line change
`@@ -25,6 +25,7 @@ import (`
`25`	`25`	`"github.com/dustin/go-humanize"`
`26`	`26`
`27`	`27`	`"github.com/elastic/beats/v7/libbeat/common"`
	`28`	`+ "github.com/elastic/beats/v7/libbeat/common/cfgtype"`
`28`	`29`	`"github.com/elastic/beats/v7/libbeat/reader"`
`29`	`30`	`"github.com/elastic/beats/v7/libbeat/reader/multiline"`
`30`	`31`	`"github.com/elastic/beats/v7/libbeat/reader/readfile"`
`@@ -43,7 +44,7 @@ type Parser interface {`
`43`	`44`	`}`
`44`	`45`
`45`	`46`	`type CommonConfig struct {`
`46`		- MaxBytes int `config:"max_bytes"`
	`47`	+ MaxBytes cfgtype.ByteSize `config:"max_bytes"`
`47`	`48`	LineTerminator readfile.LineTerminator `config:"line_terminator"`
`48`	`49`	`}`
`49`	`50`
`@@ -126,7 +127,7 @@ func (c *Config) Create(in reader.Reader) Parser {`
`126`	`127`	`if err != nil {`
`127`	`128`	`return p`
`128`	`129`	`}`
`129`		`- p, err = multiline.New(p, "\n", c.pCfg.MaxBytes, &config)`
	`130`	`+ p, err = multiline.New(p, "\n", int(c.pCfg.MaxBytes), &config)`
`130`	`131`	`if err != nil {`
`131`	`132`	`return p`
`132`	`133`	`}`