Skip to content

Commit 24a4da8

Browse files
author
kaiyan-sheng
authored
Drop pkt_dstaddr and pkt_srcaddr when equals to "-" (#22721)
* Add painless script to remove all empty fields
1 parent 3a1d1ae commit 24a4da8

6 files changed

Lines changed: 90 additions & 3 deletions

File tree

‎CHANGELOG.next.asciidoc‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -335,6 +335,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
335335
- Fix handing missing eventtime and assignip field being set to N/A for fortinet module. {pull}22361[22361]
336336
- Fix Zeek dashboard reference to `zeek.ssl.server.name` field. {pull}21696[21696]
337337
- Fix for `field [source] not present as part of path [source.ip]` error in azure pipelines. {pull}22377[22377]
338+
- Drop aws.vpcflow.pkt_srcaddr and aws.vpcflow.pkt_dstaddr when equal to "-". {pull}22721[22721] {issue}22716[22716]
338339

339340
*Heartbeat*
340341

‎x-pack/filebeat/module/aws/vpcflow/ingest/pipeline.yml‎

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,32 @@ processors:
2828
field: ["aws.vpcflow.start", "aws.vpcflow.end"]
2929
ignore_missing: true
3030

31+
- script:
32+
lang: painless
33+
ignore_failure: true
34+
if: ctx?.aws != null
35+
source: >-
36+
void handleMap(Map map) {
37+
for (def x : map.values()) {
38+
if (x instanceof Map) {
39+
handleMap(x);
40+
} else if (x instanceof List) {
41+
handleList(x);
42+
}
43+
}
44+
map.values().removeIf(v -> v instanceof String && v == "-");
45+
}
46+
void handleList(List list) {
47+
for (def x : list) {
48+
if (x instanceof Map) {
49+
handleMap(x);
50+
} else if (x instanceof List) {
51+
handleList(x);
52+
}
53+
}
54+
}
55+
handleMap(ctx.aws);
56+
3157
# IP Geolocation Lookup
3258
- geoip:
3359
field: source.ip

‎x-pack/filebeat/module/aws/vpcflow/test/custom-nat-gateway.log-expected.json‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,5 @@
11
[
22
{
3-
"aws.vpcflow.instance_id": "-",
43
"aws.vpcflow.interface_id": "eni-1235b8ca123456789",
54
"aws.vpcflow.pkt_dstaddr": "203.0.113.5",
65
"aws.vpcflow.pkt_srcaddr": "10.0.1.5",

‎x-pack/filebeat/module/aws/vpcflow/test/no-data-skip-data.log-expected.json‎

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,6 @@
22
{
33
"@timestamp": "2015-05-10T18:02:14.000Z",
44
"aws.vpcflow.account_id": "123456789010",
5-
"aws.vpcflow.action": "-",
65
"aws.vpcflow.interface_id": "eni-1235b8ca123456789",
76
"aws.vpcflow.log_status": "NODATA",
87
"aws.vpcflow.version": "2",
@@ -27,7 +26,6 @@
2726
{
2827
"@timestamp": "2015-05-10T18:02:14.000Z",
2928
"aws.vpcflow.account_id": "123456789010",
30-
"aws.vpcflow.action": "-",
3129
"aws.vpcflow.interface_id": "eni-11111111aaaaaaaaa",
3230
"aws.vpcflow.log_status": "SKIPDATA",
3331
"aws.vpcflow.version": "2",

‎x-pack/filebeat/module/aws/vpcflow/test/tcp-flag-sequence-skip-data.log‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
version vpc-id subnet-id instance-id interface-id account-id type srcaddr dstaddr srcport dstport pkt-srcaddr pkt-dstaddr protocol bytes packets start end action tcp-flags log-status
2+
3 vpc-abcdefab012345678 subnet-aaaaaaaa012345678 i-01234567890123456 eni-1235b8ca123456789 123456789010 - - - - - - - - - - 1566848875 1566848933 - - SKIPDATA
3+
4+
version vpc-id subnet-id instance-id interface-id account-id type srcaddr dstaddr srcport dstport pkt-srcaddr pkt-dstaddr protocol bytes packets start end action tcp-flags log-status
5+
3 vpc-abcdefab012345678 subnet-aaaaaaaa012345678 i-01234567890123456 eni-1235b8ca123456789 123456789010 - - - - - - - - - - 1566848875 1566848933 - - NODATA

‎x-pack/filebeat/module/aws/vpcflow/test/tcp-flag-sequence-skip-data.log-expected.json‎

Lines changed: 58 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,58 @@
1+
[
2+
{
3+
"@timestamp": "2019-08-26T19:48:53.000Z",
4+
"aws.vpcflow.account_id": "123456789010",
5+
"aws.vpcflow.instance_id": "i-01234567890123456",
6+
"aws.vpcflow.interface_id": "eni-1235b8ca123456789",
7+
"aws.vpcflow.log_status": "SKIPDATA",
8+
"aws.vpcflow.subnet_id": "subnet-aaaaaaaa012345678",
9+
"aws.vpcflow.version": "3",
10+
"aws.vpcflow.vpc_id": "vpc-abcdefab012345678",
11+
"cloud.account.id": "123456789010",
12+
"cloud.instance.id": "i-01234567890123456",
13+
"cloud.provider": "aws",
14+
"event.category": "network_traffic",
15+
"event.dataset": "aws.vpcflow",
16+
"event.end": "2019-08-26T19:48:53.000Z",
17+
"event.kind": "event",
18+
"event.module": "aws",
19+
"event.original": "3 vpc-abcdefab012345678 subnet-aaaaaaaa012345678 i-01234567890123456 eni-1235b8ca123456789 123456789010 - - - - - - - - - - 1566848875 1566848933 - - SKIPDATA",
20+
"event.start": "2019-08-26T19:47:55.000Z",
21+
"event.type": "flow",
22+
"fileset.name": "vpcflow",
23+
"input.type": "log",
24+
"log.offset": 183,
25+
"service.type": "aws",
26+
"tags": [
27+
"forwarded"
28+
]
29+
},
30+
{
31+
"@timestamp": "2019-08-26T19:48:53.000Z",
32+
"aws.vpcflow.account_id": "123456789010",
33+
"aws.vpcflow.instance_id": "i-01234567890123456",
34+
"aws.vpcflow.interface_id": "eni-1235b8ca123456789",
35+
"aws.vpcflow.log_status": "NODATA",
36+
"aws.vpcflow.subnet_id": "subnet-aaaaaaaa012345678",
37+
"aws.vpcflow.version": "3",
38+
"aws.vpcflow.vpc_id": "vpc-abcdefab012345678",
39+
"cloud.account.id": "123456789010",
40+
"cloud.instance.id": "i-01234567890123456",
41+
"cloud.provider": "aws",
42+
"event.category": "network_traffic",
43+
"event.dataset": "aws.vpcflow",
44+
"event.end": "2019-08-26T19:48:53.000Z",
45+
"event.kind": "event",
46+
"event.module": "aws",
47+
"event.original": "3 vpc-abcdefab012345678 subnet-aaaaaaaa012345678 i-01234567890123456 eni-1235b8ca123456789 123456789010 - - - - - - - - - - 1566848875 1566848933 - - NODATA",
48+
"event.start": "2019-08-26T19:47:55.000Z",
49+
"event.type": "flow",
50+
"fileset.name": "vpcflow",
51+
"input.type": "log",
52+
"log.offset": 526,
53+
"service.type": "aws",
54+
"tags": [
55+
"forwarded"
56+
]
57+
}
58+
]

0 commit comments

Comments
 (0)