Skip to content

Commit 990a5f7

Browse files
leehinmanleehinman
authored
[Filebeat] Add ECS tls & categorization fields to apache module (#16121)
* Add ECS tls & categorization fields to apache module - tls.cipher (access) - tls.protocol (access) - tls.protocol_version (access) - event.kind (access) - event.category (access) - event.outcome (access) - lowercase http.request.method for ECS compliance (access) - event.kind (error) - event.category (error) - event.type (error) Closes #16032
1 parent 52e5f49 commit 990a5f7

15 files changed

Lines changed: 327 additions & 236 deletions

‎CHANGELOG.next.asciidoc‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -111,6 +111,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
111111
- Add ECS tls fields to zeek:smtp,rdp,ssl and aws:s3access,elb {issue}15757[15757] {pull}15935[15936]
112112
- Add custom string mapping to CEF module to support Forcepoint NGFW {issue}14663[14663] {pull}15910[15910]
113113
- move create-[module,fileset,fields] to mage and enable in x-pack/filebeat {pull}15836[15836]
114+
- Add ECS tls and categorization fields to apache module. {issue}16032[16032] {pull}16121[16121]
114115

115116
*Heartbeat*
116117

‎filebeat/module/apache/access/ingest/default.json‎

Lines changed: 0 additions & 100 deletions
This file was deleted.

‎filebeat/module/apache/access/ingest/pipeline.yml‎

Lines changed: 103 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,103 @@
1+
description: "Pipeline for parsing Apache HTTP Server access logs. Requires the geoip and user_agent plugins."
2+
3+
processors:
4+
- grok:
5+
field: message
6+
patterns:
7+
- '%{IPORHOST:destination.domain} %{IPORHOST:source.ip} - %{DATA:user.name} \[%{HTTPDATE:apache.access.time}\]
8+
"(?:%{WORD:http.request.method} %{DATA:url.original} HTTP/%{NUMBER:http.version}|-)?"
9+
%{NUMBER:http.response.status_code:long} (?:%{NUMBER:http.response.body.bytes:long}|-)(
10+
"%{DATA:http.request.referrer}")?( "%{DATA:user_agent.original}")?'
11+
- '%{IPORHOST:source.address} - %{DATA:user.name} \[%{HTTPDATE:apache.access.time}\]
12+
"(?:%{WORD:http.request.method} %{DATA:url.original} HTTP/%{NUMBER:http.version}|-)?"
13+
%{NUMBER:http.response.status_code:long} (?:%{NUMBER:http.response.body.bytes:long}|-)(
14+
"%{DATA:http.request.referrer}")?( "%{DATA:user_agent.original}")?'
15+
- '%{IPORHOST:source.address} - %{DATA:user.name} \[%{HTTPDATE:apache.access.time}\]
16+
"-" %{NUMBER:http.response.status_code:long} -'
17+
- \[%{HTTPDATE:apache.access.time}\] %{IPORHOST:source.address} %{DATA:apache.access.ssl.protocol}
18+
%{DATA:apache.access.ssl.cipher} "%{WORD:http.request.method} %{DATA:url.original}
19+
HTTP/%{NUMBER:http.version}" (-|%{NUMBER:http.response.body.bytes:long})
20+
ignore_missing: true
21+
- remove:
22+
field: message
23+
- set:
24+
field: event.kind
25+
value: event
26+
- set:
27+
field: event.category
28+
value: web
29+
- set:
30+
field: event.outcome
31+
value: success
32+
if: "ctx?.http?.response?.status_code != null && ctx.http.response.status_code < 400"
33+
- set:
34+
field: event.outcome
35+
value: failure
36+
if: "ctx?.http?.response?.status_code != null && ctx.http.response.status_code > 399"
37+
- lowercase:
38+
field: http.request.method
39+
ignore_missing: true
40+
- grok:
41+
field: source.address
42+
ignore_missing: true
43+
patterns:
44+
- ^(%{IP:source.ip}|%{HOSTNAME:source.domain})$
45+
- rename:
46+
field: '@timestamp'
47+
target_field: event.created
48+
- date:
49+
field: apache.access.time
50+
target_field: '@timestamp'
51+
formats:
52+
- dd/MMM/yyyy:H:m:s Z
53+
ignore_failure: true
54+
- remove:
55+
field: apache.access.time
56+
ignore_failure: true
57+
- user_agent:
58+
field: user_agent.original
59+
ignore_failure: true
60+
- geoip:
61+
field: source.ip
62+
target_field: source.geo
63+
ignore_missing: true
64+
- geoip:
65+
database_file: GeoLite2-ASN.mmdb
66+
field: source.ip
67+
target_field: source.as
68+
properties:
69+
- asn
70+
- organization_name
71+
ignore_missing: true
72+
- rename:
73+
field: source.as.asn
74+
target_field: source.as.number
75+
ignore_missing: true
76+
- rename:
77+
field: source.as.organization_name
78+
target_field: source.as.organization.name
79+
ignore_missing: true
80+
- set:
81+
field: tls.cipher
82+
value: '{{apache.access.ssl.cipher}}'
83+
if: ctx?.apache?.access?.ssl?.cipher != null
84+
85+
- script:
86+
lang: painless
87+
if: ctx?.apache?.access?.ssl?.protocol != null
88+
source: >-
89+
def parts = ctx.apache.access.ssl.protocol.toLowerCase().splitOnToken("v");
90+
if (parts.length != 2) {
91+
return;
92+
}
93+
if (parts[1].contains(".")) {
94+
ctx.tls.version = parts[1];
95+
} else {
96+
ctx.tls.version = parts[1] + ".0";
97+
}
98+
ctx.tls.version_protocol = parts[0];
99+
100+
on_failure:
101+
- set:
102+
field: error.message
103+
value: '{{ _ingest.on_failure_message }}'

‎filebeat/module/apache/access/manifest.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,7 @@ var:
1212
- "C:/tools/Apache/httpd-2.*/Apache24/logs/access.log*"
1313
- "C:/Program Files/Apache Software Foundation/Apache2.*/logs/access.log*"
1414

15-
ingest_pipeline: ingest/default.json
15+
ingest_pipeline: ingest/pipeline.yml
1616
input: config/access.yml
1717

1818
requires.processors:

‎filebeat/module/apache/access/test/darwin-2.4.23.log-expected.json‎

Lines changed: 23 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,13 @@
11
[
22
{
33
"@timestamp": "2016-12-26T14:16:28.000Z",
4+
"event.category": "web",
45
"event.dataset": "apache.access",
6+
"event.kind": "event",
57
"event.module": "apache",
8+
"event.outcome": "success",
69
"fileset.name": "access",
7-
"http.request.method": "GET",
10+
"http.request.method": "get",
811
"http.response.body.bytes": 45,
912
"http.response.status_code": 200,
1013
"http.version": "1.1",
@@ -18,10 +21,13 @@
1821
},
1922
{
2023
"@timestamp": "2016-12-26T14:16:29.000Z",
24+
"event.category": "web",
2125
"event.dataset": "apache.access",
26+
"event.kind": "event",
2227
"event.module": "apache",
28+
"event.outcome": "failure",
2329
"fileset.name": "access",
24-
"http.request.method": "GET",
30+
"http.request.method": "get",
2531
"http.response.body.bytes": 209,
2632
"http.response.status_code": 404,
2733
"http.version": "1.1",
@@ -35,8 +41,11 @@
3541
},
3642
{
3743
"@timestamp": "2016-12-26T14:16:48.000Z",
44+
"event.category": "web",
3845
"event.dataset": "apache.access",
46+
"event.kind": "event",
3947
"event.module": "apache",
48+
"event.outcome": "failure",
4049
"fileset.name": "access",
4150
"http.response.status_code": 408,
4251
"input.type": "log",
@@ -48,10 +57,13 @@
4857
},
4958
{
5059
"@timestamp": "2016-12-26T16:23:35.000Z",
60+
"event.category": "web",
5161
"event.dataset": "apache.access",
62+
"event.kind": "event",
5263
"event.module": "apache",
64+
"event.outcome": "success",
5365
"fileset.name": "access",
54-
"http.request.method": "GET",
66+
"http.request.method": "get",
5567
"http.response.body.bytes": 45,
5668
"http.response.status_code": 200,
5769
"http.version": "1.1",
@@ -74,10 +86,13 @@
7486
},
7587
{
7688
"@timestamp": "2016-12-26T16:23:41.000Z",
89+
"event.category": "web",
7790
"event.dataset": "apache.access",
91+
"event.kind": "event",
7892
"event.module": "apache",
93+
"event.outcome": "failure",
7994
"fileset.name": "access",
80-
"http.request.method": "GET",
95+
"http.request.method": "get",
8196
"http.response.body.bytes": 206,
8297
"http.response.status_code": 404,
8398
"http.version": "1.1",
@@ -100,10 +115,13 @@
100115
},
101116
{
102117
"@timestamp": "2016-12-26T16:23:45.000Z",
118+
"event.category": "web",
103119
"event.dataset": "apache.access",
120+
"event.kind": "event",
104121
"event.module": "apache",
122+
"event.outcome": "failure",
105123
"fileset.name": "access",
106-
"http.request.method": "GET",
124+
"http.request.method": "get",
107125
"http.response.body.bytes": 201,
108126
"http.response.status_code": 404,
109127
"http.version": "1.1",

‎filebeat/module/apache/access/test/ssl-request.log-expected.json‎

Lines changed: 12 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -3,27 +3,34 @@
33
"@timestamp": "2018-08-10T07:45:56.000Z",
44
"apache.access.ssl.cipher": "ECDHE-RSA-AES128-GCM-SHA256",
55
"apache.access.ssl.protocol": "TLSv1.2",
6+
"event.category": "web",
67
"event.dataset": "apache.access",
8+
"event.kind": "event",
79
"event.module": "apache",
810
"fileset.name": "access",
9-
"http.request.method": "GET",
11+
"http.request.method": "get",
1012
"http.response.body.bytes": 1375,
1113
"http.version": "1.1",
1214
"input.type": "log",
1315
"log.offset": 0,
1416
"service.type": "apache",
1517
"source.address": "172.30.0.119",
1618
"source.ip": "172.30.0.119",
19+
"tls.cipher": "ECDHE-RSA-AES128-GCM-SHA256",
20+
"tls.version": "1.2",
21+
"tls.version_protocol": "tls",
1722
"url.original": "/nagiosxi/ajaxhelper.php?cmd=getxicoreajax&amp;opts=%7B%22func%22%3A%22get_admin_tasks_html%22%2C%22args%22%3A%22%22%7D&amp;nsp=b5c7d5d4b6f7d0cf0c92f9cbdf737f6a5c838218425e6ae21"
1823
},
1924
{
2025
"@timestamp": "2019-10-16T09:53:47.000Z",
2126
"apache.access.ssl.cipher": "ECDHE-RSA-AES128-GCM-SHA256",
2227
"apache.access.ssl.protocol": "TLSv1.2",
28+
"event.category": "web",
2329
"event.dataset": "apache.access",
30+
"event.kind": "event",
2431
"event.module": "apache",
2532
"fileset.name": "access",
26-
"http.request.method": "GET",
33+
"http.request.method": "get",
2734
"http.version": "1.1",
2835
"input.type": "log",
2936
"log.offset": 276,
@@ -34,6 +41,9 @@
3441
"source.geo.location.lat": 37.751,
3542
"source.geo.location.lon": -97.822,
3643
"source.ip": "11.19.0.217",
44+
"tls.cipher": "ECDHE-RSA-AES128-GCM-SHA256",
45+
"tls.version": "1.2",
46+
"tls.version_protocol": "tls",
3747
"url.original": "/appl/ajaxhelper.php?cmd=getxicoreajax&opts=%7B%22func%22%3A%22get_pagetop_alert_content_html%22%2C%22args%22%3A%22%22%7D&nsp=c2700eab9797eda8a9f65a3ab17a6adbceccd60a6cca7708650a5923950d"
3848
}
3949
]

‎filebeat/module/apache/access/test/test-vhost.log-expected.json‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,10 +2,13 @@
22
{
33
"@timestamp": "2016-12-26T16:22:14.000Z",
44
"destination.domain": "vhost1.domaine.fr",
5+
"event.category": "web",
56
"event.dataset": "apache.access",
7+
"event.kind": "event",
68
"event.module": "apache",
9+
"event.outcome": "failure",
710
"fileset.name": "access",
8-
"http.request.method": "GET",
11+
"http.request.method": "get",
912
"http.request.referrer": "-",
1013
"http.response.body.bytes": 499,
1114
"http.response.status_code": 404,

0 commit comments

Comments
 (0)