This error received when processing events from an Ubiquiti Firewall Log. I'm pretty sure this problem was not present in 7.10.1, but have not been able to trace it back to where it started.
This causes data to be missing from the Kibana Ubiquiti Firewall Overview ECS dashboard.
{
"_index": "filebeat-7.12.0-2021.03.24-000001",
"_type": "_doc",
"_id": "VXbMiHgBQf4FhkA2s-IK",
"_version": 1,
"_score": null,
"fields": {
"rule.id": [
"2000"
],
"event.category": [
"network"
],
"iptables.id": [
0
],
"host.hostname": [
"net-rt-usg"
],
"iptables.input_device": [
"eth0"
],
"suricata.eve.src_port": [
443
],
"traefik.access.geoip.location": [
{
"coordinates": [
30.5233,
50.45
],
"type": "Point"
}
],
"iptables.tos": [
8
],
"destination.mac": [
"74:8a:20:42:d2:d4"
],
"service.type": [
"iptables"
],
"hostname": [
"net-rt-usg"
],
"iptables.precedence_bits": [
32
],
"iptables.ubiquiti.rule_number": [
"2000"
],
"iptables.length": [
60
],
"source.ip": [
"37.19.198.61"
],
"agent.name": [
"srv-filebeat"
],
"network.community_id": [
"1:0V4q42mcC43EU24xc89rPXriMxY="
],
"host.name": [
"net-rt-usg"
],
"event.kind": [
"event"
],
"log.original": [
"[WAN_IN-2000-D]IN=eth0 OUT=eth1 MAC=74:8a:20:42:d2:d4:3e:41:04:04:a4:f4:08:00 SRC=37.19.198.61 DST=192.168.1.115 LEN=60 TOS=0x08 PREC=0x20 TTL=54 ID=0 DF PROTO=TCP SPT=443 DPT=37097 WINDOW=65160 RES=0x00 ACK SYN URGP=0 "
],
"event.severity": [
4
],
"suricata.eve.proto": [
"tcp"
],
"rule.name": [
"WAN_IN"
],
"traefik.access.geoip.country_iso_code": [
"UA"
],
"fileset.name": [
"log"
],
"iptables.ubiquiti.rule_set": [
"WAN_IN"
],
"input.type": [
"syslog"
],
"iptables.tcp.flags": [
"ACK",
"SYN"
],
"agent.hostname": [
"srv-filebeat"
],
"tags": [
"iptables"
],
"iptables.fragment_flags": [
"DF"
],
"agent.id": [
"dd4a0411-0ba3-42d8-9b7d-0261b2c6244f"
],
"source.port": [
443
],
"ecs.version": [
"1.8.0"
],
"log.source.address": [
"192.168.1.1:44303"
],
"agent.version": [
"7.12.0"
],
"iptables.tcp.window": [
65160
],
"iptables.output_device": [
"eth1"
],
"source.as.number": [
31343
],
"suricata.eve.src_ip": [
"37.19.198.61"
],
"destination.port": [
37097
],
"process.program": [
"kernel"
],
"syslog.facility": [
0
],
"source.geo.location": [
{
"coordinates": [
40.5233,
60.45
],
"type": "Point"
}
],
"suricata.eve.alert.severity": [
4
],
"agent.type": [
"filebeat"
],
"source.mac": [
"3e:41:04:04:a4:f4"
],
"iptables.tcp.reserved_bits": [
0
],
"event.module": [
"iptables"
],
"related.ip": [
"37.19.198.61",
"192.168.1.115"
],
"source.geo.country_iso_code": [
"UA"
],
"network.type": [
"ipv4"
],
"event.timezone": [
"-04:00"
],
"source.as.organization.name.text": [
"Intertelecom Ltd"
],
"syslog.priority": [
4
],
"source.geo.continent_name": [
"Europe"
],
"source.as.organization.name": [
"Intertelecom Ltd"
],
"iptables.ether_type": [
2048
],
"traefik.access.geoip.continent_name": [
"Europe"
],
"destination.ip": [
"192.168.1.115"
],
"network.transport": [
"tcp"
],
"suricata.eve.dest_ip": [
"192.168.1.115"
],
"event.ingested": [
"2021-03-31T15:00:19.848Z"
],
"event.action": [
"drop"
],
"iptables.ttl": [
54
],
"@timestamp": [
"2021-03-31T15:00:19.000Z"
],
"syslog.severity_label": [
"Warning"
],
"suricata.eve.dest_port": [
37097
],
"error.message": [
"field [raw_date] not present as part of path [iptables.raw_date]"
],
"event.type": [
"denied",
"connection"
],
"agent.ephemeral_id": [
"f4936f35-3ad9-4606-ab6e-edc02ebd9657"
],
"syslog.facility_label": [
"kernel"
],
"source.geo.country_name": [
"Ukraine"
],
"event.dataset": [
"iptables.log"
]
},
"sort": [
1617202819000
]
}
This error received when processing events from an Ubiquiti Firewall Log. I'm pretty sure this problem was not present in 7.10.1, but have not been able to trace it back to where it started.
This causes data to be missing from the Kibana Ubiquiti Firewall Overview ECS dashboard.
Steps to Reproduce:
field [raw_date] not present as part of path [iptables.raw_date]Configuration
Event
{ "_index": "filebeat-7.12.0-2021.03.24-000001", "_type": "_doc", "_id": "VXbMiHgBQf4FhkA2s-IK", "_version": 1, "_score": null, "fields": { "rule.id": [ "2000" ], "event.category": [ "network" ], "iptables.id": [ 0 ], "host.hostname": [ "net-rt-usg" ], "iptables.input_device": [ "eth0" ], "suricata.eve.src_port": [ 443 ], "traefik.access.geoip.location": [ { "coordinates": [ 30.5233, 50.45 ], "type": "Point" } ], "iptables.tos": [ 8 ], "destination.mac": [ "74:8a:20:42:d2:d4" ], "service.type": [ "iptables" ], "hostname": [ "net-rt-usg" ], "iptables.precedence_bits": [ 32 ], "iptables.ubiquiti.rule_number": [ "2000" ], "iptables.length": [ 60 ], "source.ip": [ "37.19.198.61" ], "agent.name": [ "srv-filebeat" ], "network.community_id": [ "1:0V4q42mcC43EU24xc89rPXriMxY=" ], "host.name": [ "net-rt-usg" ], "event.kind": [ "event" ], "log.original": [ "[WAN_IN-2000-D]IN=eth0 OUT=eth1 MAC=74:8a:20:42:d2:d4:3e:41:04:04:a4:f4:08:00 SRC=37.19.198.61 DST=192.168.1.115 LEN=60 TOS=0x08 PREC=0x20 TTL=54 ID=0 DF PROTO=TCP SPT=443 DPT=37097 WINDOW=65160 RES=0x00 ACK SYN URGP=0 " ], "event.severity": [ 4 ], "suricata.eve.proto": [ "tcp" ], "rule.name": [ "WAN_IN" ], "traefik.access.geoip.country_iso_code": [ "UA" ], "fileset.name": [ "log" ], "iptables.ubiquiti.rule_set": [ "WAN_IN" ], "input.type": [ "syslog" ], "iptables.tcp.flags": [ "ACK", "SYN" ], "agent.hostname": [ "srv-filebeat" ], "tags": [ "iptables" ], "iptables.fragment_flags": [ "DF" ], "agent.id": [ "dd4a0411-0ba3-42d8-9b7d-0261b2c6244f" ], "source.port": [ 443 ], "ecs.version": [ "1.8.0" ], "log.source.address": [ "192.168.1.1:44303" ], "agent.version": [ "7.12.0" ], "iptables.tcp.window": [ 65160 ], "iptables.output_device": [ "eth1" ], "source.as.number": [ 31343 ], "suricata.eve.src_ip": [ "37.19.198.61" ], "destination.port": [ 37097 ], "process.program": [ "kernel" ], "syslog.facility": [ 0 ], "source.geo.location": [ { "coordinates": [ 40.5233, 60.45 ], "type": "Point" } ], "suricata.eve.alert.severity": [ 4 ], "agent.type": [ "filebeat" ], "source.mac": [ "3e:41:04:04:a4:f4" ], "iptables.tcp.reserved_bits": [ 0 ], "event.module": [ "iptables" ], "related.ip": [ "37.19.198.61", "192.168.1.115" ], "source.geo.country_iso_code": [ "UA" ], "network.type": [ "ipv4" ], "event.timezone": [ "-04:00" ], "source.as.organization.name.text": [ "Intertelecom Ltd" ], "syslog.priority": [ 4 ], "source.geo.continent_name": [ "Europe" ], "source.as.organization.name": [ "Intertelecom Ltd" ], "iptables.ether_type": [ 2048 ], "traefik.access.geoip.continent_name": [ "Europe" ], "destination.ip": [ "192.168.1.115" ], "network.transport": [ "tcp" ], "suricata.eve.dest_ip": [ "192.168.1.115" ], "event.ingested": [ "2021-03-31T15:00:19.848Z" ], "event.action": [ "drop" ], "iptables.ttl": [ 54 ], "@timestamp": [ "2021-03-31T15:00:19.000Z" ], "syslog.severity_label": [ "Warning" ], "suricata.eve.dest_port": [ 37097 ], "error.message": [ "field [raw_date] not present as part of path [iptables.raw_date]" ], "event.type": [ "denied", "connection" ], "agent.ephemeral_id": [ "f4936f35-3ad9-4606-ab6e-edc02ebd9657" ], "syslog.facility_label": [ "kernel" ], "source.geo.country_name": [ "Ukraine" ], "event.dataset": [ "iptables.log" ] }, "sort": [ 1617202819000 ] }