add source_region field

kaiyan-sheng · kaiyan-sheng · commit e4efcf0680b6 · 2026-03-11T22:43:08.000-06:00
diff --git a/packages/aws/changelog.yml b/packages/aws/changelog.yml
@@ -1,7 +1,7 @@
 # newer versions go on top
 - version: "6.3.1"
   changes:
-    - description: Fix s3access ingest pipeline to handle special characters in S3 bucket names.
+    - description: Fix s3access ingest pipeline to handle host headers with ports and Source region field parsing.
       type: bugfix
       link: https://github.com/elastic/integrations/pull/17748
 - version: "6.3.0"
diff --git a/packages/aws/data_stream/s3access/_dev/test/pipeline/test-s3-server-access.log-expected.json b/packages/aws/data_stream/s3access/_dev/test/pipeline/test-s3-server-access.log-expected.json
@@ -1060,7 +1060,6 @@
             "@timestamp": "2026-03-09T18:02:43.000Z",
             "aws": {
                 "s3access": {
-                    "aclrequired": "us-east-1",
                     "bucket": "flow-log-test",
                     "bucket_owner": "28bdbd9102b76c3da07c44e094b16b382f90f94be71626f4d9589b1309d2d7ef",
                     "bytes_sent": 243,
@@ -1072,6 +1071,7 @@
                     "remote_ip": "89.160.20.156",
                     "request_id": "366DB3C4B325AB11",
                     "request_uri": "GET / HTTP/1.1",
+                    "source_region": "us-east-1",
                     "total_time": 13,
                     "user_agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) HeadlessChrome/143.0.7499.4 Safari/537.36"
                 }
diff --git a/packages/aws/data_stream/s3access/elasticsearch/ingest_pipeline/default.yml b/packages/aws/data_stream/s3access/elasticsearch/ingest_pipeline/default.yml
@@ -1,5 +1,5 @@
 ---
-description: "Pipeline for s3 server access logs"
+description: "Pipeline for S3 server access logs"
 
 processors:
   - set:
@@ -33,15 +33,16 @@ processors:
           (?:-|%{NUMBER:aws.s3access.object_size:long}) (?:-|%{NUMBER:aws.s3access.total_time:long}) (?:-|%{NUMBER:aws.s3access.turn_around_time:long})
           (?:-|\"-\"|\"%{DATA:aws.s3access.referrer}\") (?:-|\"(-|%{DATA:aws.s3access.user_agent})\") (?:-|%{S3KEY:aws.s3access.version_id})
           (?:-|%{S3ID:aws.s3access.host_id}) (?:-|%{S3VERSION:aws.s3access.signature_version}) (?:-|%{S3KEY:aws.s3access.cipher_suite})
-          (?:-|%{WORD:aws.s3access.authentication_type}) (?:-|%{S3ID:aws.s3access.host_header}) (?:-|%{S3VERSION:aws.s3access.tls_version})(?: (?:-|%{S3REQUESTER:aws.s3access.access_point_arn}))?(?: (?:-|%{S3ACLREQUIRED:aws.s3access.aclrequired}))?
+          (?:-|%{WORD:aws.s3access.authentication_type}) (?:-|%{S3ID:aws.s3access.host_header}) (?:-|%{S3VERSION:aws.s3access.tls_version})(?: (?:-|%{S3REQUESTER:aws.s3access.access_point_arn}))?(?: (?:-|%{S3ACLREQUIRED:aws.s3access.aclrequired}))?(?: (?:-|%{S3REGION:aws.s3access.source_region}))?(?: %{S3REGION:aws.s3access.source_region})?
       pattern_definitions:
         S3REQUESTER: "[a-zA-Z0-9\\/_\\.\\-%:@]+"
         S3REQUESTID: "[a-zA-Z0-9]+"
         S3OPERATION: "%{WORD}.%{WORD}.%{WORD}"
         S3KEY: "[a-zA-Z0-9\\/\\_\\!\\-\\.\\*\\'\\(\\)\\%\\+]+"
         S3ID: "[a-zA-Z0-9\\/_\\.\\-%+=:]+"
         S3VERSION: "[a-zA-Z0-9.]+"
-        S3ACLREQUIRED: "[a-zA-Z0-9_-]+"
+        S3ACLREQUIRED: "(-|Yes)"
+        S3REGION: "[a-zA-Z][a-zA-Z0-9-]*"
   - grok:
       field: aws.s3access.host_header
       ignore_missing: true
@@ -188,6 +189,10 @@ processors:
       field: aws.s3access.aclrequired
       value: '{{aws.s3access.aclrequired}}'
       ignore_empty_value: true
+  - set:
+      field: aws.s3access.source_region
+      value: '{{aws.s3access.source_region}}'
+      ignore_empty_value: true
   - set:
       field: cloud.provider
       value: aws
diff --git a/packages/aws/data_stream/s3access/fields/fields.yml b/packages/aws/data_stream/s3access/fields/fields.yml
@@ -97,6 +97,10 @@
       type: keyword
       description: |
         Whether the ACL was required for the request.
+    - name: source_region
+      type: keyword
+      description: |
+        The AWS Region from which the request originated.
     - name: access_point_arn
       type: keyword
       description: |
diff --git a/packages/aws/docs/s3.md b/packages/aws/docs/s3.md
@@ -82,6 +82,7 @@ Refer to the following [document](https://www.elastic.co/guide/en/ecs/current/ec
 | aws.s3access.request_uri | The Request-URI part of the HTTP request message. | keyword |
 | aws.s3access.requester | The canonical user ID of the requester, or a - for unauthenticated requests. | keyword |
 | aws.s3access.signature_version | The signature version, SigV2 or SigV4, that was used to authenticate the request or a - for unauthenticated requests. | keyword |
+| aws.s3access.source_region | The AWS Region from which the request originated. | keyword |
 | aws.s3access.tls_version | The Transport Layer Security (TLS) version negotiated by the client. | keyword |
 | aws.s3access.total_time | The total amount of time in milliseconds the request was in flight from the server's perspective. | long |
 | aws.s3access.turn_around_time | The total amount of time in milliseconds that Amazon S3 spent processing your request. | long |
