[Filebeat][AWS] Fix vpcflow pipeline exception: Cannot invoke "Object.getClass()" because "receiver" is null (#24167)

adriansr · adriansr · commit 4dab85cc9a99 · 2021-02-22T23:55:13.000+01:00
The pipeline failed with an obscure error.message: `Cannot invoke "Object.getClass()" because "receiver" is null` when the ingested message didn't contain `aws.vpcflow.*` fiels. Filebeat generates documents that lack those fields when parsing lines from a .log file (not .json) that doesn't conform to the expected formats. This happened for empty lines in particular. (cherry picked from commit f4b7a25)
diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc
@@ -167,6 +167,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Fix event.kind for system/syslog pipeline {issue}20365[20365] {pull}20390[20390]
 - Fix event.type for zeek/ssl and duplicate event.category for zeek/connection {pull}20696[20696]
 - Fix Okta default date formatting. {issue}24018[24018] {pull}24025[24025]
+- Fix aws/vpcflow generating errors for empty logs or unidentified formats. {pull}24167[24167]
 
 *Heartbeat*
 
diff --git a/x-pack/filebeat/module/aws/vpcflow/ingest/pipeline.yml b/x-pack/filebeat/module/aws/vpcflow/ingest/pipeline.yml
@@ -119,7 +119,7 @@ processors:
       ignore_empty_value: true
 
   - set:
-      if: "ctx.aws.vpcflow.instance_id != '-'"
+      if: "ctx.aws?.vpcflow?.instance_id != null && ctx.aws.vpcflow.instance_id != '-'"
       field: cloud.instance.id
       value: "{{aws.vpcflow.instance_id}}"
       ignore_empty_value: true
diff --git a/x-pack/filebeat/module/aws/vpcflow/test/bad.log b/x-pack/filebeat/module/aws/vpcflow/test/bad.log
@@ -0,0 +1 @@
+Phony unsupported log format.
diff --git a/x-pack/filebeat/module/aws/vpcflow/test/bad.log-expected.json b/x-pack/filebeat/module/aws/vpcflow/test/bad.log-expected.json
@@ -0,0 +1,18 @@
+[
+    {
+        "cloud.provider": "aws",
+        "event.category": "network_traffic",
+        "event.dataset": "aws.vpcflow",
+        "event.kind": "event",
+        "event.module": "aws",
+        "event.original": "Phony unsupported log format.",
+        "event.type": "flow",
+        "fileset.name": "vpcflow",
+        "input.type": "log",
+        "log.offset": 0,
+        "service.type": "aws",
+        "tags": [
+            "forwarded"
+        ]
+    }
+]
