Skip to content

Commit 3953756

Browse files
wph95wph95
authored
Add RFC5424 format support for syslog input (#23954)
* - add PRI schema - add VERSION schema * - make test work - clean code - change parser.go to parser/parser_rfc5424 * - Add TIMESTAMP format * - Add "AppName" format - Add "ProcID" format - Add "MsgID" format - Add "HostName" format - improve test * add STRUCTURED_DATA support * add MESSAGE support * - syslog input config support rfc5424 - add auto detect format * add param value escape support * - clean up the code - add more test * update mod
1 parent 6981ca3 commit 3953756

18 files changed

Lines changed: 12378 additions & 305 deletions

‎filebeat/input/syslog/config.go‎

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -36,13 +36,31 @@ import (
3636

3737
type config struct {
3838
harvester.ForwarderConfig `config:",inline"`
39+
Format syslogFormat `config:"format"`
3940
Protocol common.ConfigNamespace `config:"protocol"`
4041
}
4142

43+
type syslogFormat int
44+
45+
const (
46+
syslogFormatRFC3164 = iota
47+
syslogFormatRFC5424
48+
syslogFormatAuto
49+
)
50+
51+
var (
52+
syslogFormats = map[string]syslogFormat{
53+
"rfc3164": syslogFormatRFC3164,
54+
"rfc5424": syslogFormatRFC5424,
55+
"auto": syslogFormatAuto,
56+
}
57+
)
58+
4259
var defaultConfig = config{
4360
ForwarderConfig: harvester.ForwarderConfig{
4461
Type: "syslog",
4562
},
63+
Format: syslogFormatRFC3164,
4664
}
4765

4866
type syslogTCP struct {
@@ -122,3 +140,12 @@ func factory(
122140
return nil, fmt.Errorf("you must choose between TCP or UDP")
123141
}
124142
}
143+
144+
func (f *syslogFormat) Unpack(value string) error {
145+
format, ok := syslogFormats[value]
146+
if !ok {
147+
return fmt.Errorf("invalid format '%s'", value)
148+
}
149+
*f = format
150+
return nil
151+
}

‎filebeat/input/syslog/event.go‎

Lines changed: 81 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@
1818
package syslog
1919

2020
import (
21+
"bytes"
2122
"math"
2223
"time"
2324
)
@@ -72,8 +73,17 @@ type event struct {
7273
year int
7374
loc *time.Location
7475
sequence int
76+
77+
// RFC 5424
78+
version int
79+
appName string
80+
msgID string
81+
processID string
82+
data EventData
7583
}
7684

85+
type EventData map[string]map[string]string
86+
7787
// newEvent() return a new event.
7888
func newEvent() *event {
7989
return &event{
@@ -86,6 +96,7 @@ func newEvent() *event {
8696
second: -1,
8797
year: time.Now().Year(),
8898
sequence: -1,
99+
version: -1,
89100
}
90101
}
91102

@@ -198,7 +209,12 @@ func (s *event) Year() int {
198209

199210
// SetMessage sets the message.
200211
func (s *event) SetMessage(b []byte) {
201-
s.message = string(b)
212+
// remove BOM
213+
if b[0] == 0xef && b[1] == 0xbb && b[2] == 0xbf {
214+
s.message = string(b[3:])
215+
} else {
216+
s.message = string(b)
217+
}
202218
}
203219

204220
// Message returns the message.
@@ -298,6 +314,39 @@ func (s *event) Nanosecond() int {
298314
return s.nanosecond
299315
}
300316

317+
// SetVersion sets the version.
318+
func (s *event) SetVersion(version []byte) {
319+
s.version = bytesToInt(version)
320+
}
321+
322+
func (s *event) Version() int {
323+
return s.version
324+
}
325+
326+
func (s *event) SetAppName(appname []byte) {
327+
s.appName = string(appname)
328+
}
329+
330+
func (s *event) AppName() string {
331+
return s.appName
332+
}
333+
334+
func (s *event) SetMsgID(msgID []byte) {
335+
s.msgID = string(msgID)
336+
}
337+
338+
func (s *event) MsgID() string {
339+
return s.msgID
340+
}
341+
342+
func (s *event) SetProcID(processID []byte) {
343+
s.processID = string(processID)
344+
}
345+
346+
func (s *event) ProcID() string {
347+
return s.processID
348+
}
349+
301350
// Timestamp return the timestamp in UTC.
302351
func (s *event) Timestamp(timezone *time.Location) time.Time {
303352
var t *time.Location
@@ -319,9 +368,39 @@ func (s *event) Timestamp(timezone *time.Location) time.Time {
319368
).UTC()
320369
}
321370

371+
func (s *event) IsDataEmpty() bool {
372+
if s.data == nil {
373+
return true
374+
}
375+
return len(s.data) == 0
376+
}
377+
322378
// IsValid returns true if the date and the message are present.
323379
func (s *event) IsValid() bool {
324-
return s.day != -1 && s.hour != -1 && s.minute != -1 && s.second != -1 && s.message != ""
380+
return s.day != -1 && s.hour != -1 && s.minute != -1 && s.second != -1 && (s.message != "" || !s.IsDataEmpty())
381+
}
382+
383+
func (s *event) SetData(id string, key string, data []byte, start int, end int, bs []int) {
384+
var v string
385+
386+
// param value escape
387+
// https://tools.ietf.org/html/rfc5424#section-6.3.3
388+
if len(bs) > 0 {
389+
buf := bytes.NewBufferString("")
390+
for _, i := range bs {
391+
buf.Write(data[start:i])
392+
start = i + 1
393+
}
394+
if start <= end {
395+
buf.Write(data[start:end])
396+
}
397+
v = buf.String()
398+
} else {
399+
v = string(data[start:end])
400+
}
401+
if element, ok := s.data[id]; ok {
402+
element[key] = v
403+
}
325404
}
326405

327406
// BytesToInt takes a variable length of bytes and assume ascii chars and convert it to int, this is

0 commit comments

Comments
 (0)