Skip to content

Commit 74f1ad1

Browse files
marc-grmarc-gr
authored
Merge branch 'feature-ecs-1.8' into panw-ecs-1.8
2 parents c31db04 + c957e58 commit 74f1ad1

25 files changed

Lines changed: 330 additions & 38 deletions

‎CHANGELOG.next.asciidoc‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -379,6 +379,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
379379
- Fix handling of ModifiedProperties field in Office 365. {pull}23777[23777]
380380
- Fix goroutines leak with some inputs in autodiscover. {pull}23722[23722]
381381
- Fix various processing errors in the Suricata module. {pull}23236[23236]
382+
- aws/s3access dataset was populating event.duration using the wrong unit. {pull}23920[23920]
382383

383384
*Heartbeat*
384385

@@ -836,6 +837,8 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
836837
- Upgrade CEF module to ECS 1.8.0. {pull}23832[23832]
837838
- Upgrade fortinet/firewall to ECS 1.8 {issue}23118[23118] {pull}23902[23902]
838839
- Upgrade Zeek to ECS 1.8.0. {issue}23118[23118] {pull}23847[23847]
840+
- Updated azure module to ECS 1.8. {issue}23118[23118] {pull}23927[23927]
841+
- Update aws/s3access to ECS 1.8. {issue}23118[23118] {pull}23920[23920]
839842
- Upgrade panw module to ecs 1.8 {issue}23118[23118] {pull}23931[23931]
840843

841844
*Heartbeat*

‎x-pack/filebeat/module/aws/s3access/config/aws-s3.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -52,4 +52,4 @@ processors:
5252
- add_fields:
5353
target: ''
5454
fields:
55-
ecs.version: 1.7.0
55+
ecs.version: 1.8.0

‎x-pack/filebeat/module/aws/s3access/config/file.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -11,4 +11,4 @@ processors:
1111
- add_fields:
1212
target: ''
1313
fields:
14-
ecs.version: 1.7.0
14+
ecs.version: 1.8.0

‎x-pack/filebeat/module/aws/s3access/ingest/pipeline.yml‎

Lines changed: 49 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,12 @@
11
description: "Pipeline for s3 server access logs"
22

33
processors:
4+
- set:
5+
field: event.category
6+
value: web
7+
- append:
8+
field: event.type
9+
value: access
410
- set:
511
field: event.ingested
612
value: '{{_ingest.timestamp}}'
@@ -24,6 +30,22 @@ processors:
2430
S3ID: "[a-zA-Z0-9\\/_\\.\\-%+=]+"
2531
S3VERSION: "[a-zA-Z0-9.]+"
2632

33+
- grok:
34+
field: aws.s3access.request_uri
35+
ignore_failure: true
36+
patterns:
37+
- '%{NOTSPACE:http.request.method} %{NOTSPACE:url.original} [hH][tT][tT][pP]/%{NOTSPACE:http.version}'
38+
39+
#
40+
# Best-effort parse of url.original in the form /path?query"
41+
#
42+
- grok:
43+
field: url.original
44+
ignore_failure: true
45+
patterns:
46+
- '^%{ABS_PATH:url.path}(?:\?%{DATA:url.query})?$'
47+
pattern_definitions:
48+
ABS_PATH: '/[^?]*'
2749
- append:
2850
if: "ctx?.aws?.s3access?.bucket_owner != null"
2951
field: related.user
@@ -99,10 +121,25 @@ processors:
99121
field: event.outcome
100122
value: success
101123

102-
- set:
103-
field: event.duration
104-
value: "{{aws.s3access.total_time}}"
105-
ignore_empty_value: true
124+
- convert:
125+
field: aws.s3access.bytes_sent
126+
target_field: http.response.body.bytes
127+
type: long
128+
ignore_failure: true
129+
130+
- convert:
131+
field: aws.s3access.total_time
132+
target_field: event.duration
133+
type: long
134+
ignore_failure: true
135+
136+
- script:
137+
lang: painless
138+
if: ctx.event?.duration != null
139+
params:
140+
MS_TO_NS: 1000000
141+
source: >-
142+
ctx.event.duration *= params.MS_TO_NS;
106143
107144
- set:
108145
field: http.request.referrer
@@ -137,13 +174,18 @@ processors:
137174
field: event.kind
138175
value: event
139176

177+
#
178+
# Save original message into event.original
179+
#
180+
- rename:
181+
field: "message"
182+
target_field: "event.original"
183+
140184
#
141185
# Remove temporary fields
142186
#
143187
- remove:
144-
field:
145-
- message
146-
- _temp_
188+
field: _temp_
147189
ignore_missing: true
148190

149191
on_failure:

‎x-pack/filebeat/module/aws/s3access/test/s3_server_access.log-expected.json‎

Lines changed: 58 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -23,12 +23,17 @@
2323
"client.user.id": "arn:aws:sts::123456:assumed-role/AWSServiceRoleForTrustedAdvisor/TrustedAdvisor_627959692251_784ab70b-8cc9-4d37-a2ec-2ff4d0c08af9",
2424
"cloud.provider": "aws",
2525
"event.action": "REST.GET.LOCATION",
26+
"event.category": "web",
2627
"event.dataset": "aws.s3access",
27-
"event.duration": "17",
28+
"event.duration": 17000000,
2829
"event.id": "44EE8651683CB4DA",
2930
"event.kind": "event",
3031
"event.module": "aws",
32+
"event.original": "36c1f05b76016b78528454e6e0c60e2b7ff7aa20c0a5e4c748276e5b0a2debd2 test-s3-ks [01/Aug/2019:00:24:41 +0000] 72.21.217.31 arn:aws:sts::123456:assumed-role/AWSServiceRoleForTrustedAdvisor/TrustedAdvisor_627959692251_784ab70b-8cc9-4d37-a2ec-2ff4d0c08af9 44EE8651683CB4DA REST.GET.LOCATION - \"GET /test-s3-ks/?location&aws-account=627959692251 HTTP/1.1\" 200 - 142 - 17 - \"-\" \"AWS-Support-TrustedAdvisor, aws-internal/3 aws-sdk-java/1.11.590 Linux/4.9.137-0.1.ac.218.74.329.metal1.x86_64 OpenJDK_64-Bit_Server_VM/25.212-b03 java/1.8.0_212 vendor/Oracle_Corporation\" - BsCfJedfuSnds2QFoxi+E/O7M6OEWzJnw4dUaes/2hyA363sONRJKzB7EOY+Bt9DTHYUn+HoHxI= SigV4 ECDHE-RSA-AES128-SHA AuthHeader s3.ap-southeast-1.amazonaws.com TLSv1.2",
3133
"event.outcome": "success",
34+
"event.type": [
35+
"access"
36+
],
3237
"fileset.name": "s3access",
3338
"geo.city_name": "Ashburn",
3439
"geo.continent_name": "North America",
@@ -38,7 +43,10 @@
3843
"geo.location.lon": -77.4728,
3944
"geo.region_iso_code": "US-VA",
4045
"geo.region_name": "Virginia",
46+
"http.request.method": "GET",
47+
"http.response.body.bytes": 142,
4148
"http.response.status_code": 200,
49+
"http.version": "1.1",
4250
"input.type": "log",
4351
"log.offset": 0,
4452
"related.ip": [
@@ -54,6 +62,9 @@
5462
"tls.cipher": "ECDHE-RSA-AES128-SHA",
5563
"tls.version": "1.2",
5664
"tls.version_protocol": "tls",
65+
"url.original": "/test-s3-ks/?location&aws-account=627959692251",
66+
"url.path": "/test-s3-ks/",
67+
"url.query": "location&aws-account=627959692251",
5768
"user_agent.device.name": "Other",
5869
"user_agent.name": "aws-sdk-java",
5970
"user_agent.original": "AWS-Support-TrustedAdvisor, aws-internal/3 aws-sdk-java/1.11.590 Linux/4.9.137-0.1.ac.218.74.329.metal1.x86_64 OpenJDK_64-Bit_Server_VM/25.212-b03 java/1.8.0_212 vendor/Oracle_Corporation",
@@ -86,12 +97,17 @@
8697
"client.user.id": "arn:aws:sts::123456:assumed-role/AWSServiceRoleForTrustedAdvisor/TrustedAdvisor_627959692251_784ab70b-8cc9-4d37-a2ec-2ff4d0c08af9",
8798
"cloud.provider": "aws",
8899
"event.action": "REST.GET.LOCATION",
100+
"event.category": "web",
89101
"event.dataset": "aws.s3access",
90-
"event.duration": "3",
102+
"event.duration": 3000000,
91103
"event.id": "E26222010BCC32B6",
92104
"event.kind": "event",
93105
"event.module": "aws",
106+
"event.original": "36c1f05b76016b78528454e6e0c60e2b7ff7aa20c0a5e4c748276e5b0a2debd2 test-s3-ks [01/Aug/2019:00:24:42 +0000] 72.21.217.31 arn:aws:sts::123456:assumed-role/AWSServiceRoleForTrustedAdvisor/TrustedAdvisor_627959692251_784ab70b-8cc9-4d37-a2ec-2ff4d0c08af9 E26222010BCC32B6 REST.GET.LOCATION - \"GET /test-s3-ks/?location&aws-account=627959692251 HTTP/1.1\" 200 - 142 - 3 - \"-\" \"AWS-Support-TrustedAdvisor, aws-internal/3 aws-sdk-java/1.11.590 Linux/4.9.137-0.1.ac.218.74.329.metal1.x86_64 OpenJDK_64-Bit_Server_VM/25.212-b03 java/1.8.0_212 vendor/Oracle_Corporation\" - gNl/Q1IzY6nGTBygqI3rnMz/ZFOFwOTDpSMrNca+IcEmMAd6sCIs1ZRLYDekD8LB9lrj9UdQLWE= SigV4 ECDHE-RSA-AES128-SHA AuthHeader s3.ap-southeast-1.amazonaws.com TLSv1.2",
94107
"event.outcome": "success",
108+
"event.type": [
109+
"access"
110+
],
95111
"fileset.name": "s3access",
96112
"geo.city_name": "Ashburn",
97113
"geo.continent_name": "North America",
@@ -101,7 +117,10 @@
101117
"geo.location.lon": -77.4728,
102118
"geo.region_iso_code": "US-VA",
103119
"geo.region_name": "Virginia",
120+
"http.request.method": "GET",
121+
"http.response.body.bytes": 142,
104122
"http.response.status_code": 200,
123+
"http.version": "1.1",
105124
"input.type": "log",
106125
"log.offset": 715,
107126
"related.ip": [
@@ -117,6 +136,9 @@
117136
"tls.cipher": "ECDHE-RSA-AES128-SHA",
118137
"tls.version": "1.2",
119138
"tls.version_protocol": "tls",
139+
"url.original": "/test-s3-ks/?location&aws-account=627959692251",
140+
"url.path": "/test-s3-ks/",
141+
"url.query": "location&aws-account=627959692251",
120142
"user_agent.device.name": "Other",
121143
"user_agent.name": "aws-sdk-java",
122144
"user_agent.original": "AWS-Support-TrustedAdvisor, aws-internal/3 aws-sdk-java/1.11.590 Linux/4.9.137-0.1.ac.218.74.329.metal1.x86_64 OpenJDK_64-Bit_Server_VM/25.212-b03 java/1.8.0_212 vendor/Oracle_Corporation",
@@ -150,12 +172,17 @@
150172
"client.user.id": "arn:aws:sts::123456:assumed-role/AWSServiceRoleForTrustedAdvisor/TrustedAdvisor_627959692251_784ab70b-8cc9-4d37-a2ec-2ff4d0c08af9",
151173
"cloud.provider": "aws",
152174
"event.action": "REST.GET.BUCKET",
175+
"event.category": "web",
153176
"event.dataset": "aws.s3access",
154-
"event.duration": "2",
177+
"event.duration": 2000000,
155178
"event.id": "4DD6D17D1C5C401C",
156179
"event.kind": "event",
157180
"event.module": "aws",
181+
"event.original": "36c1f05b76016b78528454e6e0c60e2b7ff7aa20c0a5e4c748276e5b0a2debd2 test-s3-ks [01/Aug/2019:00:24:43 +0000] 72.21.217.31 arn:aws:sts::123456:assumed-role/AWSServiceRoleForTrustedAdvisor/TrustedAdvisor_627959692251_784ab70b-8cc9-4d37-a2ec-2ff4d0c08af9 4DD6D17D1C5C401C REST.GET.BUCKET - \"GET /test-s3-ks/?max-keys=0&encoding-type=url&aws-account=627959692251 HTTP/1.1\" 200 - 265 - 2 1 \"-\" \"AWS-Support-TrustedAdvisor, aws-internal/3 aws-sdk-java/1.11.590 Linux/4.9.137-0.1.ac.218.74.329.metal1.x86_64 OpenJDK_64-Bit_Server_VM/25.212-b03 java/1.8.0_212 vendor/Oracle_Corporation\" - KzvchfojYQnuFC4PABYVJVxIlv/f6r17LRaTSvw7x+bxj4PkkPKT1kX9x8wbqtq40iD4PC881iE= SigV4 ECDHE-RSA-AES128-SHA AuthHeader s3.ap-southeast-1.amazonaws.com TLSv1.2",
158182
"event.outcome": "success",
183+
"event.type": [
184+
"access"
185+
],
159186
"fileset.name": "s3access",
160187
"geo.city_name": "Ashburn",
161188
"geo.continent_name": "North America",
@@ -165,7 +192,10 @@
165192
"geo.location.lon": -77.4728,
166193
"geo.region_iso_code": "US-VA",
167194
"geo.region_name": "Virginia",
195+
"http.request.method": "GET",
196+
"http.response.body.bytes": 265,
168197
"http.response.status_code": 200,
198+
"http.version": "1.1",
169199
"input.type": "log",
170200
"log.offset": 1429,
171201
"related.ip": [
@@ -181,6 +211,9 @@
181211
"tls.cipher": "ECDHE-RSA-AES128-SHA",
182212
"tls.version": "1.2",
183213
"tls.version_protocol": "tls",
214+
"url.original": "/test-s3-ks/?max-keys=0&encoding-type=url&aws-account=627959692251",
215+
"url.path": "/test-s3-ks/",
216+
"url.query": "max-keys=0&encoding-type=url&aws-account=627959692251",
184217
"user_agent.device.name": "Other",
185218
"user_agent.name": "aws-sdk-java",
186219
"user_agent.original": "AWS-Support-TrustedAdvisor, aws-internal/3 aws-sdk-java/1.11.590 Linux/4.9.137-0.1.ac.218.74.329.metal1.x86_64 OpenJDK_64-Bit_Server_VM/25.212-b03 java/1.8.0_212 vendor/Oracle_Corporation",
@@ -213,12 +246,17 @@
213246
"client.user.id": "arn:aws:sts::123456:assumed-role/AWSServiceRoleForTrustedAdvisor/TrustedAdvisor_627959692251_784ab70b-8cc9-4d37-a2ec-2ff4d0c08af9",
214247
"cloud.provider": "aws",
215248
"event.action": "REST.GET.LOCATION",
249+
"event.category": "web",
216250
"event.dataset": "aws.s3access",
217-
"event.duration": "4",
251+
"event.duration": 4000000,
218252
"event.id": "706992E2F3CC3C3D",
219253
"event.kind": "event",
220254
"event.module": "aws",
255+
"event.original": "36c1f05b76016b78528454e6e0c60e2b7ff7aa20c0a5e4c748276e5b0a2debd2 test-s3-ks [01/Aug/2019:00:24:43 +0000] 72.21.217.31 arn:aws:sts::123456:assumed-role/AWSServiceRoleForTrustedAdvisor/TrustedAdvisor_627959692251_784ab70b-8cc9-4d37-a2ec-2ff4d0c08af9 706992E2F3CC3C3D REST.GET.LOCATION - \"GET /test-s3-ks/?location&aws-account=627959692251 HTTP/1.1\" 200 - 142 - 4 - \"-\" \"AWS-Support-TrustedAdvisor, aws-internal/3 aws-sdk-java/1.11.590 Linux/4.9.137-0.1.ac.218.74.329.metal1.x86_64 OpenJDK_64-Bit_Server_VM/25.212-b03 java/1.8.0_212 vendor/Oracle_Corporation\" - cIN12KTrJwx+uTBZD+opZUPE4iGypi8oG/oXGPzFk9CMuHQGuEpmAeNELdtYKDxf2TDor25Nikg= SigV4 ECDHE-RSA-AES128-SHA AuthHeader s3.ap-southeast-1.amazonaws.com TLSv1.2",
221256
"event.outcome": "success",
257+
"event.type": [
258+
"access"
259+
],
222260
"fileset.name": "s3access",
223261
"geo.city_name": "Ashburn",
224262
"geo.continent_name": "North America",
@@ -228,7 +266,10 @@
228266
"geo.location.lon": -77.4728,
229267
"geo.region_iso_code": "US-VA",
230268
"geo.region_name": "Virginia",
269+
"http.request.method": "GET",
270+
"http.response.body.bytes": 142,
231271
"http.response.status_code": 200,
272+
"http.version": "1.1",
232273
"input.type": "log",
233274
"log.offset": 2161,
234275
"related.ip": [
@@ -244,6 +285,9 @@
244285
"tls.cipher": "ECDHE-RSA-AES128-SHA",
245286
"tls.version": "1.2",
246287
"tls.version_protocol": "tls",
288+
"url.original": "/test-s3-ks/?location&aws-account=627959692251",
289+
"url.path": "/test-s3-ks/",
290+
"url.query": "location&aws-account=627959692251",
247291
"user_agent.device.name": "Other",
248292
"user_agent.name": "aws-sdk-java",
249293
"user_agent.original": "AWS-Support-TrustedAdvisor, aws-internal/3 aws-sdk-java/1.11.590 Linux/4.9.137-0.1.ac.218.74.329.metal1.x86_64 OpenJDK_64-Bit_Server_VM/25.212-b03 java/1.8.0_212 vendor/Oracle_Corporation",
@@ -274,11 +318,16 @@
274318
"client.user.id": "arn:aws:iam::123456:user/test@elastic.co",
275319
"cloud.provider": "aws",
276320
"event.action": "BATCH.DELETE.OBJECT",
321+
"event.category": "web",
277322
"event.dataset": "aws.s3access",
278323
"event.id": "8CD7A4A71E2E5C9E",
279324
"event.kind": "event",
280325
"event.module": "aws",
326+
"event.original": "36c1f05b76016b78528454e6e0c60e2b7ff7aa20c0a5e4c748276e5b0a2debd2 jsoriano-s3-test [10/Sep/2019:15:11:07 +0000] 77.227.156.41 arn:aws:iam::123456:user/test@elastic.co 8CD7A4A71E2E5C9E BATCH.DELETE.OBJECT jolokia-war-1.5.0.war - 204 - - 344017 - - - - - IeDW5I3wefFxU8iHOcAzi5qr+O+1bdRlcQ0AO2WGjFh7JwYM6qCoKq+1TrUshrXMlBxPFtg97Vk= SigV4 ECDHE-RSA-AES128-SHA AuthHeader s3.eu-central-1.amazonaws.com TLSv1.2",
281327
"event.outcome": "success",
328+
"event.type": [
329+
"access"
330+
],
282331
"fileset.name": "s3access",
283332
"geo.city_name": "Teruel",
284333
"geo.continent_name": "Europe",
@@ -327,11 +376,16 @@
327376
"client.user.id": "arn:aws:iam::123456:user/test@elastic.co",
328377
"cloud.provider": "aws",
329378
"event.action": "BATCH.DELETE.OBJECT",
379+
"event.category": "web",
330380
"event.dataset": "aws.s3access",
331381
"event.id": "6CE38F1312D32BDD",
332382
"event.kind": "event",
333383
"event.module": "aws",
384+
"event.original": "36c1f05b76016b78528454e6e0c60e2b7ff7aa20c0a5e4c748276e5b0a2debd2 test-s3-ks [19/Sep/2019:17:06:39 +0000] 174.29.206.152 arn:aws:iam::123456:user/test@elastic.co 6CE38F1312D32BDD BATCH.DELETE.OBJECT Screen+Shot+2019-09-09+at+9.08.44+AM.png - 204 - - 57138 - - - - - LwRa4w6DbuU48GKQiH3jDbjfTyLCbwasFBsdttugRQ+9lH4jK8lT91+HhGZKMYI3sPyKuQ9LvU0= SigV4 ECDHE-RSA-AES128-SHA AuthHeader s3-ap-southeast-1.amazonaws.com TLSv1.2",
334385
"event.outcome": "success",
386+
"event.type": [
387+
"access"
388+
],
335389
"fileset.name": "s3access",
336390
"geo.city_name": "Denver",
337391
"geo.continent_name": "North America",

‎x-pack/filebeat/module/aws/s3access/test/test.log‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,3 +3,4 @@
33
79a59df900b949e55d96a1e698fbacedfd6e09d98eacf8f8d5218e7cd47ef2be awsexamplebucket [06/Feb/2019:00:00:38 +0000] 192.0.2.3 79a59df900b949e55d96a1e698fbacedfd6e09d98eacf8f8d5218e7cd47ef2be A1206F460EXAMPLE REST.GET.BUCKETPOLICY - "GET /awsexamplebucket?policy HTTP/1.1" 404 NoSuchBucketPolicy 297 - 38 - "-" "S3Console/0.4" - BNaBsXZQQDbssi6xMBdBU2sLt+Yf5kZDmeBUP35sFoKa3sLLeMC78iwEIWxs99CRUrbS4n11234= SigV2 ECDHE-RSA-AES128-GCM-SHA256 AuthHeader awsexamplebucket.s3.amazonaws.com TLSV1.1
44
79a59df900b949e55d96a1e698fbacedfd6e09d98eacf8f8d5218e7cd47ef2be awsexamplebucket [06/Feb/2019:00:01:00 +0000] 192.0.2.3 79a59df900b949e55d96a1e698fbacedfd6e09d98eacf8f8d5218e7cd47ef2be 7B4A0FABBEXAMPLE REST.GET.VERSIONING - "GET /awsexamplebucket?versioning HTTP/1.1" 200 - 113 - 33 - "-" "S3Console/0.4" - Ke1bUcazaN1jWuUlPJaxF64cQVpUEhoZKEG/hmy/gijN/I1DeWqDfFvnpybfEseEME/u7ME1234= SigV2 ECDHE-RSA-AES128-GCM-SHA256 AuthHeader awsexamplebucket.s3.amazonaws.com TLSV1.1
55
79a59df900b949e55d96a1e698fbacedfd6e09d98eacf8f8d5218e7cd47ef2be awsexamplebucket [06/Feb/2019:00:01:57 +0000] 192.0.2.3 79a59df900b949e55d96a1e698fbacedfd6e09d98eacf8f8d5218e7cd47ef2be DD6CC733AEXAMPLE REST.PUT.OBJECT s3-dg.pdf "PUT /awsexamplebucket/s3-dg.pdf HTTP/1.1" 200 - - 4406583 41754 28 "-" "S3Console/0.4" - 10S62Zv81kBW7BB6SX4XJ48o6kpcl6LPwEoizZQQxJd5qDSCTLX0TgS37kYUBKQW3+bPdrg1234= SigV4 ECDHE-RSA-AES128-SHA AuthHeader awsexamplebucket.s3.amazonaws.com TLSV1.1
6+
79a59df900b949e55d96a1e698fbacedfd6e09d98eacf8f8d5218e7cd47ef2be faketest [09/Feb/2021:14:48:42 +0200] 192.0.2.3 79a59df900b949e55d96a1e698fbacedfd6e09d98eacf8f8d5218e7cd47ef2be DD6CC733AEXAMPLE REST.OPTIONS.FAKE s3-dg.pdf "OPTIONS * HTTP/1.0" 200 - - 4406583 41754 28 "-" "S3Console/0.4" - 10S62Zv81kBW7BB6SX4XJ48o6kpcl6LPwEoizZQQxJd5qDSCTLX0TgS37kYUBKQW3+bPdrg1234= SigV4 ECDHE-RSA-AES128-SHA AuthHeader awsexamplebucket.s3.amazonaws.com TLSV1.1

0 commit comments

Comments
 (0)