feat: improve AWS S3 acess log regex for S3 key

Kavindu-Dodan · Kavindu-Dodan · commit 56c3dfc54f4e · 2025-12-24T12:29:25.000-08:00
Signed-off-by: Kavindu Dodanduwa &lt;kavindu.dodanduwa@elastic.co&gt;
diff --git a/packages/aws/changelog.yml b/packages/aws/changelog.yml
@@ -1,4 +1,9 @@
 # newer versions go on top
+- version: "5.3.2"
+  changes:
+    - description: S3 Access logs, improve S3KEY regex to handle special characters.
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/16695
 - version: "5.3.1"
   changes:
     - description: Add `cookies` field in cloudfront logs datastream.
diff --git a/packages/aws/data_stream/s3access/_dev/test/pipeline/test-s3-server-access.log b/packages/aws/data_stream/s3access/_dev/test/pipeline/test-s3-server-access.log
@@ -9,3 +9,4 @@
 67797214d75628047d9c76b18a78cded1a4b069b71f2a9d5a53649c38da8770b flow-log-test [14/Jul/2021:18:57:31 +0000] - svc:delivery.logs.amazonaws.com MVGXZXEVN3IG9S24 REST.PUT.OBJECT AWSLogs/000000000000/vpcflowlogs/us-gov-east-1/2021/07/13/000000000000_vpcflowlogs_us-gov-east-1_fl-_20210713T1855Z_f12aa632.log.gz "PUT /AWSLogs/000000000000/vpcflowlogs/us-gov-east-1/2021/07/13/000000000000_vpcflowlogs_us-gov-east-1_fl-0e7c13bf00cf15bfe_20210713T1855Z_f12aa632.log.gz HTTP/1.1" - - - 773 103 13 "-" "-" - 02SxwfXpO5UysN0GsKGa3uGDQ6E/W7+Hwo/luRH8p1VEexULoe66RCM+nja0dEq2JqLrtgjocvVRRkVt4= SigV4 ECDHE-RSA-AES128-GCM-SHA256 AuthHeader flow-log-test.s3.us-gov-west-1.amazonaws.com TLSv1.2 arn:aws:s3:us-west-1:123456789012:accesspoint/example-AP -
 b854390a51155554b82ce2759564a1135bce83133d004f4d2001f157e13985d7 flow-log-test [25/Mar/2025:19:28:02 +0000] - AmazonS3 366DB3C4B325AB11 S3.EXPIRE.OBJECT 0/chum/_vars/logtests/PlannerModule/5f6ea3b7da96ab304a77225d5b2b2a55e54b74e4ddfdf14b9b1d853d77515b88_9febba22f08b11ef8cf6020058a9efab/2024/12/30/164700/kitt_189/_spcu_sride__state_svx__feature__flags.sst "-" - - - 317 - - "-" "-" qsEq9bDa2VyxyZ4cz0c7oBnF67VYTTij DMlPb9al4CvVBck150CgpEIIYgtSI3HC/atetNVYwPtHZffW6jfpg+BrffhbT9/B - - - - - - -
 b854390a51155554b82ce2759564a1135bce83133d004f4d2001f157e13985d7 flow-log-test [25/Mar/2025:19:28:02 +0000] - AmazonS3 366DB3C4B325AB11 S3.EXPIRE.OBJECT 0/chum/_vars/logtests/PlannerModule/5f6ea3b7da96ab304a77225d5b2b2a55e54b74e4ddfdf14b9b1d853d77515b88_9febba22f08b11ef8cf6020058a9efab/2024/12/30/164700/kitt_189/_spcu_sride__state_svx__feature__flags.sst "-" - - - 317 - - "-" "-" qsEq9bDa2VyxyZ4cz0c7oBnF67VYTTij DMlPb9al4CvVBck150CgpEIIYgtSI3HC/atetNVYwPtHZffW6jfpg+BrffhbT9/B - - - - - - Yes
+b854390a51155554b82ce2759564a1135bce83133d004f4d2001f157e13985d7 flow-log-test [25/Mar/2025:19:28:02 +0000] - AmazonS3 366DB3C4B325AB11 S3.PUT.OBJECT test/special(chars*'!%)/75*75/log.txt "-" - - - 317 - - "-" "-" qsEq9bDa2VyxyZ4cz0c7oBnF67VYTTij DMlPb9al4CvVBck150CgpEIIYgtSI3HC/atetNVYwPtHZffW6jfpg+BrffhbT9/B - - - - - - Yes
diff --git a/packages/aws/data_stream/s3access/_dev/test/pipeline/test-s3-server-access.log-expected.json b/packages/aws/data_stream/s3access/_dev/test/pipeline/test-s3-server-access.log-expected.json
@@ -1005,6 +1005,56 @@
             "tags": [
                 "preserve_original_event"
             ]
+        },
+        {
+            "@timestamp": "2025-03-25T19:28:02.000Z",
+            "aws": {
+                "s3access": {
+                    "aclrequired": "Yes",
+                    "bucket": "flow-log-test",
+                    "bucket_owner": "b854390a51155554b82ce2759564a1135bce83133d004f4d2001f157e13985d7",
+                    "host_id": "DMlPb9al4CvVBck150CgpEIIYgtSI3HC/atetNVYwPtHZffW6jfpg+BrffhbT9/B",
+                    "key": "test/special(chars*'!%)/75*75/log.txt",
+                    "object_size": 317,
+                    "operation": "S3.PUT.OBJECT",
+                    "request_id": "366DB3C4B325AB11",
+                    "request_uri": "-",
+                    "requester": "AmazonS3",
+                    "version_id": "qsEq9bDa2VyxyZ4cz0c7oBnF67VYTTij"
+                }
+            },
+            "client": {
+                "user": {
+                    "id": "AmazonS3"
+                }
+            },
+            "cloud": {
+                "provider": "aws"
+            },
+            "ecs": {
+                "version": "8.11.0"
+            },
+            "event": {
+                "action": "S3.PUT.OBJECT",
+                "category": [
+                    "web"
+                ],
+                "id": "366DB3C4B325AB11",
+                "kind": "event",
+                "original": "b854390a51155554b82ce2759564a1135bce83133d004f4d2001f157e13985d7 flow-log-test [25/Mar/2025:19:28:02 +0000] - AmazonS3 366DB3C4B325AB11 S3.PUT.OBJECT test/special(chars*'!%)/75*75/log.txt \"-\" - - - 317 - - \"-\" \"-\" qsEq9bDa2VyxyZ4cz0c7oBnF67VYTTij DMlPb9al4CvVBck150CgpEIIYgtSI3HC/atetNVYwPtHZffW6jfpg+BrffhbT9/B - - - - - - Yes",
+                "outcome": "success",
+                "type": [
+                    "access"
+                ]
+            },
+            "related": {
+                "user": [
+                    "b854390a51155554b82ce2759564a1135bce83133d004f4d2001f157e13985d7"
+                ]
+            },
+            "tags": [
+                "preserve_original_event"
+            ]
         }
     ]
 }
diff --git a/packages/aws/data_stream/s3access/elasticsearch/ingest_pipeline/default.yml b/packages/aws/data_stream/s3access/elasticsearch/ingest_pipeline/default.yml
@@ -38,7 +38,7 @@ processors:
         S3REQUESTER: "[a-zA-Z0-9\\/_\\.\\-%:@]+"
         S3REQUESTID: "[a-zA-Z0-9]+"
         S3OPERATION: "%{WORD}.%{WORD}.%{WORD}"
-        S3KEY: "[a-zA-Z0-9\\/_\\.\\-%+]+"
+        S3KEY: "[a-zA-Z0-9\\/\\_\\!\\-\\.\\*\\'\\(\\)\\%\\+]+"
         S3ID: "[a-zA-Z0-9\\/_\\.\\-%+=]+"
         S3VERSION: "[a-zA-Z0-9.]+"
   - grok:
diff --git a/packages/aws/manifest.yml b/packages/aws/manifest.yml
@@ -1,7 +1,7 @@
 format_version: 3.3.2
 name: aws
 title: AWS
-version: "5.3.1"
+version: "5.3.2"
 description: Collect logs and metrics from Amazon Web Services (AWS) with Elastic Agent.
 type: integration
 categories:
