Skip to content

Commit 31ddbd9

Browse files
tetianakravchenkomergify-bot
tetianakravchenko
authored and
mergify-bot
committed
[Filebeat] [MongoDB] Support MongoDB 4.4 json logs (#24774)
MongoDB 4.4 uses structured JSON format for logging, add support for this format while keeping support for the old one too. (cherry picked from commit 3752526)
1 parent ed515e1 commit 31ddbd9

12 files changed

Lines changed: 302 additions & 32 deletions

‎CHANGELOG.next.asciidoc‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -567,6 +567,8 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
567567
- Add Content-Type override to aws-s3 input. {issue}25697[25697] {pull}25772[25772]
568568
- In Cisco Umbrella fileset add users from cisco.umbrella.identities to related.user. {pull}25776[25776]
569569
- Add fingerprint processor to generate fixed ids for `google_workspace` events. {pull}25841[25841]
570+
- Update PanOS module to parse HIP Match logs. {issue}24350[24350] {pull}25686[25686]
571+
- Support MongoDB 4.4 in filebeat's MongoDB module. {issue}20501[20501] {pull}24774[24774]
570572
- Enhance GCP module to populate orchestrator.* fields for GKE / K8S logs {pull}25368[25368]
571573
- Move Filebeat azure module to GA. {pull}26114[26114] {pull}26168[26168]
572574
- Make `filestream` input GA. {pull}26127[26127]

‎filebeat/docs/fields.asciidoc‎

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -95635,6 +95635,30 @@ alias to: message
9563595635

9563695636
--
9563795637

95638+
*`mongodb.log.msg`*::
95639+
+
95640+
--
95641+
String representing the raw log output message as passed from the server or driver
95642+
95643+
95644+
type: text
95645+
95646+
example: MongoDB starting
95647+
95648+
--
95649+
95650+
*`mongodb.log.id`*::
95651+
+
95652+
--
95653+
Integer representing the unique identifier of the log statement
95654+
95655+
95656+
type: long
95657+
95658+
example: 4615611
95659+
95660+
--
95661+
9563895662
[[exported-fields-mssql]]
9563995663
== mssql fields
9564095664

‎filebeat/docs/modules/mongodb.asciidoc‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,7 @@ include::../include/gs-link.asciidoc[]
1818
[float]
1919
=== Compatibility
2020

21-
The +{modulename}+ module was tested with logs from versions v3.2.11 on Debian.
21+
The +{modulename}+ module was tested with plaintext logs from version v3.2.11 on Debian and json logs from version v4.4.4 on Ubuntu.
2222

2323
include::../include/configuring-intro.asciidoc[]
2424

‎filebeat/module/mongodb/_meta/docs.asciidoc‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,7 @@ include::../include/gs-link.asciidoc[]
1313
[float]
1414
=== Compatibility
1515

16-
The +{modulename}+ module was tested with logs from versions v3.2.11 on Debian.
16+
The +{modulename}+ module was tested with plaintext logs from version v3.2.11 on Debian and json logs from version v4.4.4 on Ubuntu.
1717

1818
include::../include/configuring-intro.asciidoc[]
1919

‎filebeat/module/mongodb/fields.go‎

Lines changed: 1 addition & 1 deletion
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎filebeat/module/mongodb/log/_meta/fields.yml‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -22,3 +22,13 @@
2222
type: alias
2323
path: message
2424
migration: true
25+
- name: msg
26+
description: >
27+
String representing the raw log output message as passed from the server or driver
28+
example: MongoDB starting
29+
type: text
30+
- name: id
31+
description: >
32+
Integer representing the unique identifier of the log statement
33+
example: 4615611
34+
type: long

‎filebeat/module/mongodb/log/ingest/pipeline-json.yml‎

Lines changed: 47 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,47 @@
1+
description: Pipeline for parsing MongoDB logs in JSON format
2+
processors:
3+
- json:
4+
field: message
5+
target_field: mongodb.log
6+
- date:
7+
field: mongodb.log.t.$date
8+
target_field: '@timestamp'
9+
formats:
10+
- yyyy-MM-dd'T'HH:mm:ss.SSSZZZZZ
11+
- rename:
12+
field: mongodb.log.s
13+
target_field: log.level
14+
- rename:
15+
field: mongodb.log.c
16+
target_field: mongodb.log.component
17+
- rename:
18+
field: mongodb.log.ctx
19+
target_field: mongodb.log.context
20+
- append:
21+
field: event.type
22+
value: access
23+
if: ctx.mongodb.log.component == 'ACCESS'
24+
- append:
25+
field: event.type
26+
value: change
27+
if: ctx.mongodb.log.component == 'WRITE'
28+
- append:
29+
field: event.type
30+
value: info
31+
if: ctx.mongodb.log.component != 'WRITE' && ctx.mongodb.log.component != 'ACCESS'
32+
- append:
33+
field: event.type
34+
value: error
35+
if: ctx.log.level == 'F' || ctx.log.level == 'E'
36+
- remove:
37+
field:
38+
- mongodb.log.t
39+
- mongodb.log.attr
40+
- mongodb.log.tags
41+
- mongodb.log.truncated
42+
- mongodb.log.size
43+
ignore_missing: true
44+
on_failure:
45+
- set:
46+
field: error.message
47+
value: '{{ _ingest.on_failure_message }}'

‎filebeat/module/mongodb/log/ingest/pipeline-plaintext.yml‎

Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,34 @@
1+
description: Pipeline for parsing MongoDB logs in plaintext
2+
processors:
3+
- grok:
4+
field: message
5+
patterns:
6+
- '%{TIMESTAMP_ISO8601:mongodb.log.timestamp}%{SPACE}%{MONGO3_SEVERITY:log.level}%{SPACE}%{MONGO3_COMPONENT:mongodb.log.component}%{SPACE}(?:\[%{DATA:mongodb.log.context}\])?%{SPACE}%{GREEDYDATA:message}'
7+
ignore_missing: true
8+
- date:
9+
field: mongodb.log.timestamp
10+
target_field: '@timestamp'
11+
formats:
12+
- yyyy-MM-dd'T'HH:mm:ss.SSSZZ
13+
- remove:
14+
field: mongodb.log.timestamp
15+
- append:
16+
field: event.type
17+
value: access
18+
if: "ctx?.mongodb?.log?.component == 'ACCESS'"
19+
- append:
20+
field: event.type
21+
value: change
22+
if: "ctx?.mongodb?.log?.component == 'WRITE'"
23+
- append:
24+
field: event.type
25+
value: info
26+
if: "ctx?.mongodb?.log?.component != 'WRITE' && ctx?.mongodb?.log?.component != 'ACCESS'"
27+
- append:
28+
field: event.type
29+
value: error
30+
if: "ctx?.log?.level == 'F' || ctx?.log?.level == 'E'"
31+
on_failure:
32+
- set:
33+
field: error.message
34+
value: '{{ _ingest.on_failure_message }}'

‎filebeat/module/mongodb/log/ingest/pipeline.yml‎

Lines changed: 15 additions & 28 deletions
Original file line numberDiff line numberDiff line change
@@ -3,43 +3,30 @@ processors:
33
- set:
44
field: event.ingested
55
value: '{{_ingest.timestamp}}'
6-
- grok:
7-
field: message
8-
patterns:
9-
- '%{TIMESTAMP_ISO8601:mongodb.log.timestamp}%{SPACE}%{MONGO3_SEVERITY:log.level}%{SPACE}%{MONGO3_COMPONENT:mongodb.log.component}%{SPACE}(?:\[%{DATA:mongodb.log.context}\])?%{SPACE}%{GREEDYDATA:message}'
10-
ignore_missing: true
116
- rename:
127
field: '@timestamp'
138
target_field: event.created
14-
- date:
15-
field: mongodb.log.timestamp
16-
target_field: '@timestamp'
17-
formats:
18-
- yyyy-MM-dd'T'HH:mm:ss.SSSZZ
19-
- remove:
20-
field: mongodb.log.timestamp
9+
- grok:
10+
field: message
11+
patterns:
12+
- ^%{CHAR:first_char}
13+
pattern_definitions:
14+
CHAR: .
15+
- pipeline:
16+
if: ctx.first_char != '{'
17+
name: '{< IngestPipeline "pipeline-plaintext" >}'
18+
- pipeline:
19+
if: ctx.first_char == '{'
20+
name: '{< IngestPipeline "pipeline-json" >}'
2121
- set:
2222
field: event.kind
2323
value: event
2424
- append:
2525
field: event.category
2626
value: database
27-
- append:
28-
field: event.type
29-
value: access
30-
if: "ctx?.mongodb?.log?.component == 'ACCESS'"
31-
- append:
32-
field: event.type
33-
value: change
34-
if: "ctx?.mongodb?.log?.component == 'WRITE'"
35-
- append:
36-
field: event.type
37-
value: info
38-
if: "ctx?.mongodb?.log?.component != 'WRITE' && ctx?.mongodb?.log?.component != 'ACCESS'"
39-
- append:
40-
field: event.type
41-
value: error
42-
if: "ctx?.log?.level == 'F' || ctx?.log?.level == 'E'"
27+
- remove:
28+
field:
29+
- first_char
4330
on_failure:
4431
- set:
4532
field: error.message

‎filebeat/module/mongodb/log/manifest.yml‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7,5 +7,8 @@ var:
77
os.windows:
88
- c:\data\log\mongod.log
99

10-
ingest_pipeline: ingest/pipeline.yml
10+
ingest_pipeline:
11+
- ingest/pipeline.yml
12+
- ingest/pipeline-plaintext.yml
13+
- ingest/pipeline-json.yml
1114
input: config/log.yml

0 commit comments

Comments
 (0)