While preparing packages for the ECS updates I found that duplicated field definitions now caused indeterminate outcomes from elastic-package build. This prevented tests from passing while doing the updates and so I fixed the packages that were failing making use of new elastic-package behaviour for vetting v2 packages for duplicate field definitions. At the time, elastic-package did not continue to vet packages beyond failures in the manifest checks, which meant that it was necessary to fix those before being able to investigate duplications.
elastic-package now proceeds to check duplications even when there are manifest vet failures (here and here ), and it has become clear that the packages that I found in the original pass based on test failures did not find all cases (for example google_workspace fixed here ).
So here is a complete list of non-deprecated SEI packages that have duplicated field definitions found using the script at the footer of this issue. The script requires elastic-package v0.65.0 and was run on the tree at 28fecbd from the packages directory.
akamai akamai,auditd,barracuda,bluecoat,box_events,carbon_black_cloud,cef: remove duplicated fields #4399
"siem": client.geo.country_iso_code ecs.yml, ecs.yml
auditd akamai,auditd,barracuda,bluecoat,box_events,carbon_black_cloud,cef: remove duplicated fields #4399
"log": auditd.log.a0 fields.yml, fields.yml
"log": container.name agent.yml, ecs.yml
"log": host.architecture agent.yml, ecs.yml
"log": user.effective.group.id ecs.yml, package-fields.yml
"log": user.effective.group.name ecs.yml, package-fields.yml
"log": user.effective.id ecs.yml, package-fields.yml
"log": user.effective.name ecs.yml, package-fields.yml
aws [AWS] Remove duplicate fields from agent.yml and use ecs.yml for ECS fields #4657 (non-SEI)
"billing": cloud.account.id agent.yml, ecs.yml
"billing": cloud.availability_zone agent.yml, ecs.yml
"billing": cloud.instance.id agent.yml, ecs.yml
"billing": cloud.machine.type agent.yml, ecs.yml
"billing": cloud.provider agent.yml, ecs.yml
"billing": cloud.region agent.yml, ecs.yml
"cloudtrail": cloud.account.id agent.yml, ecs.yml
"cloudtrail": cloud.region agent.yml, ecs.yml
"cloudwatch_metrics": aws.dimensions.* fields.yml, package-fields.yml
"cloudwatch_metrics": cloud.account.id agent.yml, ecs.yml
"cloudwatch_metrics": cloud.availability_zone agent.yml, ecs.yml
"cloudwatch_metrics": cloud.instance.id agent.yml, ecs.yml
"cloudwatch_metrics": cloud.machine.type agent.yml, ecs.yml
"cloudwatch_metrics": cloud.provider agent.yml, ecs.yml
"cloudwatch_metrics": cloud.region agent.yml, ecs.yml
"dynamodb": cloud.account.id agent.yml, ecs.yml
"dynamodb": cloud.availability_zone agent.yml, ecs.yml
"dynamodb": cloud.instance.id agent.yml, ecs.yml
"dynamodb": cloud.machine.type agent.yml, ecs.yml
"dynamodb": cloud.provider agent.yml, ecs.yml
"dynamodb": cloud.region agent.yml, ecs.yml
"ebs": cloud.account.id agent.yml, ecs.yml
"ebs": cloud.availability_zone agent.yml, ecs.yml
"ebs": cloud.instance.id agent.yml, ecs.yml
"ebs": cloud.machine.type agent.yml, ecs.yml
"ebs": cloud.provider agent.yml, ecs.yml
"ebs": cloud.region agent.yml, ecs.yml
"ecs_metrics": cloud.account.id agent.yml, ecs.yml
"ecs_metrics": cloud.availability_zone agent.yml, ecs.yml
"ecs_metrics": cloud.instance.id agent.yml, ecs.yml
"ecs_metrics": cloud.machine.type agent.yml, ecs.yml
"ecs_metrics": cloud.provider agent.yml, ecs.yml
"ecs_metrics": cloud.region agent.yml, ecs.yml
"elb_logs": cloud.provider agent.yml, ecs.yml
"elb_metrics": cloud.account.id agent.yml, ecs.yml
"elb_metrics": cloud.availability_zone agent.yml, ecs.yml
"elb_metrics": cloud.instance.id agent.yml, ecs.yml
"elb_metrics": cloud.machine.type agent.yml, ecs.yml
"elb_metrics": cloud.provider agent.yml, ecs.yml
"elb_metrics": cloud.region agent.yml, ecs.yml
"firewall_logs": cloud.account.id agent.yml, ecs.yml
"firewall_logs": cloud.region agent.yml, ecs.yml
"firewall_metrics": cloud.account.id agent.yml, ecs.yml
"firewall_metrics": cloud.availability_zone agent.yml, ecs.yml
"firewall_metrics": cloud.instance.id agent.yml, ecs.yml
"firewall_metrics": cloud.machine.type agent.yml, ecs.yml
"firewall_metrics": cloud.provider agent.yml, ecs.yml
"firewall_metrics": cloud.region agent.yml, ecs.yml
"lambda": cloud.account.id agent.yml, ecs.yml
"lambda": cloud.availability_zone agent.yml, ecs.yml
"lambda": cloud.instance.id agent.yml, ecs.yml
"lambda": cloud.machine.type agent.yml, ecs.yml
"lambda": cloud.provider agent.yml, ecs.yml
"lambda": cloud.region agent.yml, ecs.yml
"natgateway": cloud.account.id agent.yml, ecs.yml
"natgateway": cloud.availability_zone agent.yml, ecs.yml
"natgateway": cloud.instance.id agent.yml, ecs.yml
"natgateway": cloud.machine.type agent.yml, ecs.yml
"natgateway": cloud.provider agent.yml, ecs.yml
"natgateway": cloud.region agent.yml, ecs.yml
"rds": cloud.account.id agent.yml, ecs.yml
"rds": cloud.availability_zone agent.yml, ecs.yml
"rds": cloud.instance.id agent.yml, ecs.yml
"rds": cloud.machine.type agent.yml, ecs.yml
"rds": cloud.provider agent.yml, ecs.yml
"rds": cloud.region agent.yml, ecs.yml
"s3_daily_storage": cloud.account.id agent.yml, ecs.yml
"s3_daily_storage": cloud.availability_zone agent.yml, ecs.yml
"s3_daily_storage": cloud.instance.id agent.yml, ecs.yml
"s3_daily_storage": cloud.machine.type agent.yml, ecs.yml
"s3_daily_storage": cloud.provider agent.yml, ecs.yml
"s3_daily_storage": cloud.region agent.yml, ecs.yml
"s3_request": cloud.account.id agent.yml, ecs.yml
"s3_request": cloud.availability_zone agent.yml, ecs.yml
"s3_request": cloud.instance.id agent.yml, ecs.yml
"s3_request": cloud.machine.type agent.yml, ecs.yml
"s3_request": cloud.provider agent.yml, ecs.yml
"s3_request": cloud.region agent.yml, ecs.yml
"s3_storage_lens": cloud.account.id agent.yml, ecs.yml
"s3_storage_lens": cloud.availability_zone agent.yml, ecs.yml
"s3_storage_lens": cloud.instance.id agent.yml, ecs.yml
"s3_storage_lens": cloud.machine.type agent.yml, ecs.yml
"s3_storage_lens": cloud.provider agent.yml, ecs.yml
"s3_storage_lens": cloud.region agent.yml, ecs.yml
"s3access": cloud.provider agent.yml, ecs.yml
"sns": cloud.account.id agent.yml, ecs.yml
"sns": cloud.availability_zone agent.yml, ecs.yml
"sns": cloud.instance.id agent.yml, ecs.yml
"sns": cloud.machine.type agent.yml, ecs.yml
"sns": cloud.provider agent.yml, ecs.yml
"sns": cloud.region agent.yml, ecs.yml
"sqs": cloud.account.id agent.yml, ecs.yml
"sqs": cloud.availability_zone agent.yml, ecs.yml
"sqs": cloud.instance.id agent.yml, ecs.yml
"sqs": cloud.machine.type agent.yml, ecs.yml
"sqs": cloud.provider agent.yml, ecs.yml
"sqs": cloud.region agent.yml, ecs.yml
"transitgateway": cloud.account.id agent.yml, ecs.yml
"transitgateway": cloud.availability_zone agent.yml, ecs.yml
"transitgateway": cloud.instance.id agent.yml, ecs.yml
"transitgateway": cloud.machine.type agent.yml, ecs.yml
"transitgateway": cloud.provider agent.yml, ecs.yml
"transitgateway": cloud.region agent.yml, ecs.yml
"usage": cloud.account.id agent.yml, ecs.yml
"usage": cloud.availability_zone agent.yml, ecs.yml
"usage": cloud.instance.id agent.yml, ecs.yml
"usage": cloud.machine.type agent.yml, ecs.yml
"usage": cloud.provider agent.yml, ecs.yml
"usage": cloud.region agent.yml, ecs.yml
"vpcflow": cloud.account.id agent.yml, ecs.yml
"vpcflow": cloud.instance.id agent.yml, ecs.yml
"vpcflow": cloud.provider agent.yml, ecs.yml
"vpcflow": source.as.organization.name ecs.yml, ecs.yml
"vpn": cloud.account.id agent.yml, ecs.yml
"vpn": cloud.availability_zone agent.yml, ecs.yml
"vpn": cloud.instance.id agent.yml, ecs.yml
"vpn": cloud.machine.type agent.yml, ecs.yml
"vpn": cloud.provider agent.yml, ecs.yml
"vpn": cloud.region agent.yml, ecs.yml
"waf": cloud.provider agent.yml, ecs.yml
barracuda akamai,auditd,barracuda,bluecoat,box_events,carbon_black_cloud,cef: remove duplicated fields #4399
"spamfirewall": @timestamp base-fields.yml, ecs.yml
"spamfirewall": tags base-fields.yml, ecs.yml
"waf": @timestamp base-fields.yml, ecs.yml
"waf": tags base-fields.yml, ecs.yml
bluecoat akamai,auditd,barracuda,bluecoat,box_events,carbon_black_cloud,cef: remove duplicated fields #4399
"director": @timestamp base-fields.yml, ecs.yml
"director": tags base-fields.yml, ecs.yml
box_events akamai,auditd,barracuda,bluecoat,box_events,carbon_black_cloud,cef: remove duplicated fields #4399
"events": box.source.id fields.yml, fields.yml
carbon_black_cloud akamai,auditd,barracuda,bluecoat,box_events,carbon_black_cloud,cef: remove duplicated fields #4399
"alert": host.hostname agent.yml, ecs.yml
"alert": host.id agent.yml, ecs.yml
"alert": host.ip agent.yml, ecs.yml
"alert": host.name agent.yml, ecs.yml
"alert": host.os.version agent.yml, ecs.yml
"asset_vulnerability_summary": host.hostname agent.yml, ecs.yml
"asset_vulnerability_summary": host.id agent.yml, ecs.yml
"asset_vulnerability_summary": host.name agent.yml, ecs.yml
"asset_vulnerability_summary": host.os.name agent.yml, ecs.yml
"asset_vulnerability_summary": host.os.version agent.yml, ecs.yml
"endpoint_event": host.hostname agent.yml, ecs.yml
"endpoint_event": host.id agent.yml, ecs.yml
"endpoint_event": host.ip agent.yml, ecs.yml
"endpoint_event": host.name agent.yml, ecs.yml
"endpoint_event": host.os.family agent.yml, ecs.yml
"endpoint_event": host.os.name agent.yml, ecs.yml
"watchlist_hit": host.hostname agent.yml, ecs.yml
"watchlist_hit": host.id agent.yml, ecs.yml
"watchlist_hit": host.ip agent.yml, ecs.yml
"watchlist_hit": host.name agent.yml, ecs.yml
cef akamai,auditd,barracuda,bluecoat,box_events,carbon_black_cloud,cef: remove duplicated fields #4399
"log": cef.extensions.deviceCustomIPv6Address2Label fields.yml, fields.yml
"log": cef.extensions.deviceCustomIPv6Address2 fields.yml, fields.yml
"log": cef.extensions.deviceCustomIPv6Address3Label fields.yml, fields.yml
"log": cef.extensions.deviceCustomIPv6Address3 fields.yml, fields.yml
cisco_asa cisco_*: remove duplicate fields #4400
"log": event.created ecs.yml, ecs.yml
"log": server.domain ecs.yml, ecs.yml
cisco_ftd cisco_*: remove duplicate fields #4400
"log": event.created ecs.yml, ecs.yml
"log": server.domain ecs.yml, ecs.yml
cisco_ios cisco_*: remove duplicate fields #4400
"log": event.created ecs.yml, ecs.yml
cisco_ise cisco_*: remove duplicate fields #4400
"log": cisco_ise.log.state fields.yml, fields.yml
"log": host.hostname agent.yml, ecs.yml
"log": host.ip agent.yml, ecs.yml
cisco_meraki cisco_*: remove duplicate fields #4400
"events": container.id agent.yml, base-fields.yml
"events": network.direction ecs.yml, ecs.yml
"events": network.protocol ecs.yml, ecs.yml
"log": container.id agent.yml, base-fields.yml
"log": network.direction ecs.yml, ecs.yml
"log": network.protocol ecs.yml, ecs.yml
cisco_nexus cisco_*: remove duplicate fields #4400
"log": container.id agent.yml, base-fields.yml
"log": tags base-fields.yml, ecs.yml
cisco_secure_email_gateway cisco_*: remove duplicate fields #4400
"log": input.type agent.yml, fields.yml
cisco_secure_endpoint cisco_*: remove duplicate fields #4400
"event": container.id agent.yml, base-fields.yml
"event": event.code ecs.yml, ecs.yml
citrix_waf citrix_waf,cloudflare,cyberark_pta,cylance,darktrace: remove duplicated fields #4401
"log": @timestamp base-fields.yml, ecs.yml
"log": event.created ecs.yml, ecs.yml
"log": server.domain ecs.yml, ecs.yml
cloudflare citrix_waf,cloudflare,cyberark_pta,cylance,darktrace: remove duplicated fields #4401
"logpull": client.geo.country_iso_code ecs.yml, ecs.yml
cyberark_pta citrix_waf,cloudflare,cyberark_pta,cylance,darktrace: remove duplicated fields #4401
"events": cef.extensions.deviceCustomIPv6Address2Label cef.yml, cef.yml
"events": cef.extensions.deviceCustomIPv6Address2 cef.yml, cef.yml
"events": cef.extensions.deviceCustomIPv6Address3Label cef.yml, cef.yml
"events": cef.extensions.deviceCustomIPv6Address3 cef.yml, cef.yml
cylance citrix_waf,cloudflare,cyberark_pta,cylance,darktrace: remove duplicated fields #4401
"protect": @timestamp base-fields.yml, ecs.yml
"protect": tags base-fields.yml, ecs.yml
darktrace citrix_waf,cloudflare,cyberark_pta,cylance,darktrace: remove duplicated fields #4401
"ai_analyst_alert": host.hostname agent.yml, ecs.yml
"ai_analyst_alert": host.id agent.yml, ecs.yml
"ai_analyst_alert": host.ip agent.yml, ecs.yml
"ai_analyst_alert": host.name agent.yml, ecs.yml
"model_breach_alert": host.hostname agent.yml, ecs.yml
"model_breach_alert": host.id agent.yml, ecs.yml
"model_breach_alert": host.ip agent.yml, ecs.yml
"model_breach_alert": host.type agent.yml, ecs.yml
"system_status_alert": host.hostname agent.yml, ecs.yml
"system_status_alert": host.ip agent.yml, ecs.yml
f5 f5,fim,fireeye,fortinet_*: remove duplicate fields #4407
"bigipafm": @timestamp base-fields.yml, ecs.yml
"bigipafm": tags base-fields.yml, ecs.yml
"bigipapm": @timestamp base-fields.yml, ecs.yml
"bigipapm": tags base-fields.yml, ecs.yml
fim f5,fim,fireeye,fortinet_*: remove duplicate fields #4407
"event": container.name agent.yml, ecs.yml
"event": host.architecture agent.yml, ecs.yml
fireeye f5,fim,fireeye,fortinet_*: remove duplicate fields #4407
"nx": destination.port ecs.yml, ecs.yml
"nx": host.ip agent.yml, ecs.yml
fortinet_forticlient f5,fim,fireeye,fortinet_*: remove duplicate fields #4407
"log": container.id agent.yml, base-fields.yml
"log": tags base-fields.yml, ecs.yml
fortinet_fortigate f5,fim,fireeye,fortinet_*: remove duplicate fields #4407
"log": container.id agent.yml, ecs.yml
fortinet_fortimail f5,fim,fireeye,fortinet_*: remove duplicate fields #4407
"log": container.id agent.yml, base-fields.yml
"log": tags base-fields.yml, ecs.yml
fortinet_fortimanager f5,fim,fireeye,fortinet_*: remove duplicate fields #4407
"log": container.id agent.yml, base-fields.yml
"log": tags base-fields.yml, ecs.yml
gcp gcp,google_workspace: remove duplicate fields #4397
"redis": cloud.account.id agent.yml, ecs.yml
"redis": cloud.availability_zone agent.yml, ecs.yml
"redis": cloud.instance.id agent.yml, ecs.yml
"redis": cloud.machine.type agent.yml, ecs.yml
"redis": cloud.provider agent.yml, ecs.yml
"redis": cloud.region agent.yml, ecs.yml
google_workspace gcp,google_workspace: remove duplicate fields #4397
"admin": container.name agent.yml, ecs.yml
"drive": container.name agent.yml, ecs.yml
"groups": container.name agent.yml, ecs.yml
"login": container.name agent.yml, ecs.yml
"saml": container.name agent.yml, ecs.yml
"user_accounts": container.name agent.yml, ecs.yml
hid_bravura_monitor hid_bravura_monitor,imperva,infoblox_*,zscalar_*: remove duplicate fields #4610
"log": event.created ecs.yml, ecs.yml
"log": log.offset agent.yml, base-fields.yml
"log": server.domain ecs.yml, ecs.yml
"winlog": event.module base-fields.yml, ecs.yml
"winlog": host.name agent.yml, ecs.yml
imperva hid_bravura_monitor,imperva,infoblox_*,zscalar_*: remove duplicate fields #4610
"securesphere": @timestamp base-fields.yml, ecs.yml
"securesphere": tags base-fields.yml, ecs.yml
infoblox_bloxone_ddi hid_bravura_monitor,imperva,infoblox_*,zscalar_*: remove duplicate fields #4610
"dhcp_lease": host.hostname agent.yml, ecs.yml
"dhcp_lease": host.name agent.yml, ecs.yml
infoblox_nios hid_bravura_monitor,imperva,infoblox_*,zscalar_*: remove duplicate fields #4610
"log": host.ip agent.yml, ecs.yml
juniper_junos juniper_*,m365_defender,microsoft_defender_endpoint,mimecast,modsecurity: remove duplicate fields #4611
"log": container.id agent.yml, base-fields.yml
"log": tags base-fields.yml, ecs.yml
juniper_netscreen juniper_*,m365_defender,microsoft_defender_endpoint,mimecast,modsecurity: remove duplicate fields #4611
"log": container.id agent.yml, base-fields.yml
"log": tags base-fields.yml, ecs.yml
juniper_srx juniper_*,m365_defender,microsoft_defender_endpoint,mimecast,modsecurity: remove duplicate fields #4611
"log": container.id agent.yml, ecs.yml
"log": container.image.name agent.yml, ecs.yml
"log": container.labels agent.yml, ecs.yml
"log": container.name agent.yml, ecs.yml
m365_defender juniper_*,m365_defender,microsoft_defender_endpoint,mimecast,modsecurity: remove duplicate fields #4611
"log": url.full ecs.yml, ecs.yml
microsoft_defender_endpoint juniper_*,m365_defender,microsoft_defender_endpoint,mimecast,modsecurity: remove duplicate fields #4611
"log": container.id agent.yml, ecs.yml
"log": container.image.name agent.yml, ecs.yml
"log": container.labels agent.yml, ecs.yml
"log": container.name agent.yml, ecs.yml
mimecast juniper_*,m365_defender,microsoft_defender_endpoint,mimecast,modsecurity: remove duplicate fields #4611
"siem_logs": email.attachments.file.name ecs.yml, ecs.yml
"siem_logs": event.action ecs.yml, ecs.yml
"ttp_ap_logs": email.attachments.file.mime_type ecs.yml, ecs.yml
"ttp_ap_logs": event.action ecs.yml, ecs.yml
"ttp_ip_logs": event.action ecs.yml, ecs.yml
"ttp_url_logs": event.action ecs.yml, ecs.yml
modsecurity juniper_*,m365_defender,microsoft_defender_endpoint,mimecast,modsecurity: remove duplicate fields #4611
"auditlog": host.ip agent.yml, ecs.yml
netflow netflow,netscout,netskope,o365,okta: remove duplicate fields #4632
"log": container.id agent.yml, ecs.yml
"log": container.image.name agent.yml, ecs.yml
"log": container.labels agent.yml, ecs.yml
"log": container.name agent.yml, ecs.yml
"log": host.os.kernel agent.yml, ecs.yml
"log": host.os.platform agent.yml, ecs.yml
"log": host.os.version agent.yml, ecs.yml
"log": host.type agent.yml, ecs.yml
netscout netflow,netscout,netskope,o365,okta: remove duplicate fields #4632
"sightline": @timestamp base-fields.yml, ecs.yml
"sightline": tags base-fields.yml, ecs.yml
netskope netflow,netscout,netskope,o365,okta: remove duplicate fields #4632
"alerts": cloud.account.id agent.yml, ecs.yml
"alerts": destination.ip ecs.yml, ecs.yml
"alerts": host.hostname agent.yml, ecs.yml
"alerts": host.os.name agent.yml, ecs.yml
"alerts": source.ip ecs.yml, ecs.yml
"events": client.bytes ecs.yml, ecs.yml
"events": client.packets ecs.yml, ecs.yml
"events": cloud.region agent.yml, ecs.yml
"events": host.hostname agent.yml, ecs.yml
o365 netflow,netscout,netskope,o365,okta: remove duplicate fields #4632
"audit": container.id agent.yml, ecs.yml
"audit": host.id agent.yml, ecs.yml
"audit": host.name agent.yml, ecs.yml
okta netflow,netscout,netskope,o365,okta: remove duplicate fields #4632
"system": container.id agent.yml, ecs.yml
panw panw*,pulse_connect_secure: remove duplicate fields #4633
"panos": container.id agent.yml, ecs.yml
"panos": host.id agent.yml, ecs.yml
"panos": host.ip agent.yml, ecs.yml
"panos": host.mac agent.yml, ecs.yml
"panos": host.name agent.yml, ecs.yml
"panos": host.os.family agent.yml, ecs.yml
"panos": source.user.name ecs.yml, ecs.yml
panw_cortex_xdr panw*,pulse_connect_secure: remove duplicate fields #4633
"alerts": host.domain agent.yml, ecs.yml
"alerts": host.hostname agent.yml, ecs.yml
"alerts": host.id agent.yml, ecs.yml
"alerts": host.ip agent.yml, ecs.yml
"alerts": host.mac agent.yml, ecs.yml
"alerts": host.os.version agent.yml, ecs.yml
pulse_connect_secure panw*,pulse_connect_secure: remove duplicate fields #4633
"log": source.as.number ecs.yml, ecs.yml
"log": source.as.organization.name ecs.yml, ecs.yml
"log": source.geo.continent_name ecs.yml, ecs.yml
"log": source.geo.country_iso_code ecs.yml, ecs.yml
"log": source.geo.country_name ecs.yml, ecs.yml
radware radware,slack,sophos,squid,suricata,symantec,ti_*,tomcat: remove duplicate fields #4642
"defensepro": @timestamp base-fields.yml, ecs.yml
"defensepro": tags base-fields.yml, ecs.yml
slack radware,slack,sophos,squid,suricata,symantec,ti_*,tomcat: remove duplicate fields #4642
"audit": event.category ecs.yml, ecs.yml
"audit": slack.audit.entity.timestamp fields.yml, fields.yml
"audit": user_agent.device.name ecs.yml, ecs.yml
"audit": user_agent.name ecs.yml, ecs.yml
"audit": user_agent.original ecs.yml, ecs.yml
"audit": user_agent.os.name ecs.yml, ecs.yml
sophos radware,slack,sophos,squid,suricata,symantec,ti_*,tomcat: remove duplicate fields #4642
"utm": tags base-fields.yml, ecs.yml
"xg": sophos.xg.syslog_server_name fields.yml, fields.yml
squid radware,slack,sophos,squid,suricata,symantec,ti_*,tomcat: remove duplicate fields #4642
"log": @timestamp base-fields.yml, ecs.yml
"log": tags base-fields.yml, ecs.yml
"log": user_agent.original ecs.yml, ecs.yml
suricata radware,slack,sophos,squid,suricata,symantec,ti_*,tomcat: remove duplicate fields #4642
"eve": @timestamp base-fields.yml, ecs.yml
"eve": host.ip agent.yml, ecs.yml
symantec_endpoint radware,slack,sophos,squid,suricata,symantec,ti_*,tomcat: remove duplicate fields #4642
"log": container.id agent.yml, ecs.yml
ti_cybersixgill radware,slack,sophos,squid,suricata,symantec,ti_*,tomcat: remove duplicate fields #4642
"threat": message ecs.yml, ecs.yml
ti_threatq radware,slack,sophos,squid,suricata,symantec,ti_*,tomcat: remove duplicate fields #4642
"threat": threat.feed.name base-fields.yml, ecs.yml
tomcat radware,slack,sophos,squid,suricata,symantec,ti_*,tomcat: remove duplicate fields #4642
"log": @timestamp base-fields.yml, ecs.yml
"log": tags ecs.yml, ecs.yml
zscaler_zia hid_bravura_monitor,imperva,infoblox_*,zscalar_*: remove duplicate fields #4610
"firewall": host.hostname agent.yml, ecs.yml
zscaler_zpa hid_bravura_monitor,imperva,infoblox_*,zscalar_*: remove duplicate fields #4610
"browser_access": client.geo.country_iso_code ecs.yml, ecs.yml
for p in *; do
grep 'elastic/security-external-integrations' ${p}/manifest.yml >/dev/null || continue
grep '^description: Deprecated' ${p}/manifest.yml >/dev/null && continue
gsed -i -e 's/^format_version: 1.0.0/format_version: 2.0.0/' -e '/^license: .*/d' ${p}/manifest.yml;
(
cd $p
m="$(elastic-package build 2>&1 | grep 'defined multiple')"
if [ "$m" != "" ]; then
echo "- [ ] $p"
echo $m \
| gsed -r 's|^ +[0-9]+\. field "(.*)" is defined multiple times for data stream (.*), found in:| \2: `\1`|g' \
| gsed -r 's|/[^ ]+/data_stream/[^ ]+/fields/||g' \
| sort
fi
)
git reset --hard >/dev/null
done
While preparing packages for the ECS updates I found that duplicated field definitions now caused indeterminate outcomes from
elastic-package build. This prevented tests from passing while doing the updates and so I fixed the packages that were failing making use of new elastic-package behaviour for vetting v2 packages for duplicate field definitions. At the time, elastic-package did not continue to vet packages beyond failures in the manifest checks, which meant that it was necessary to fix those before being able to investigate duplications.elastic-package now proceeds to check duplications even when there are manifest vet failures (here and here), and it has become clear that the packages that I found in the original pass based on test failures did not find all cases (for example google_workspace fixed here).
So here is a complete list of non-deprecated SEI packages that have duplicated field definitions found using the script at the footer of this issue. The script requires elastic-package v0.65.0 and was run on the tree at 28fecbd from the packages directory.
"siem":
client.geo.country_iso_codeecs.yml, ecs.yml"log":
auditd.log.a0fields.yml, fields.yml"log":
container.nameagent.yml, ecs.yml"log":
host.architectureagent.yml, ecs.yml"log":
user.effective.group.idecs.yml, package-fields.yml"log":
user.effective.group.nameecs.yml, package-fields.yml"log":
user.effective.idecs.yml, package-fields.yml"log":
user.effective.nameecs.yml, package-fields.yml"billing":
cloud.account.idagent.yml, ecs.yml"billing":
cloud.availability_zoneagent.yml, ecs.yml"billing":
cloud.instance.idagent.yml, ecs.yml"billing":
cloud.machine.typeagent.yml, ecs.yml"billing":
cloud.provideragent.yml, ecs.yml"billing":
cloud.regionagent.yml, ecs.yml"cloudtrail":
cloud.account.idagent.yml, ecs.yml"cloudtrail":
cloud.regionagent.yml, ecs.yml"cloudwatch_metrics":
aws.dimensions.*fields.yml, package-fields.yml"cloudwatch_metrics":
cloud.account.idagent.yml, ecs.yml"cloudwatch_metrics":
cloud.availability_zoneagent.yml, ecs.yml"cloudwatch_metrics":
cloud.instance.idagent.yml, ecs.yml"cloudwatch_metrics":
cloud.machine.typeagent.yml, ecs.yml"cloudwatch_metrics":
cloud.provideragent.yml, ecs.yml"cloudwatch_metrics":
cloud.regionagent.yml, ecs.yml"dynamodb":
cloud.account.idagent.yml, ecs.yml"dynamodb":
cloud.availability_zoneagent.yml, ecs.yml"dynamodb":
cloud.instance.idagent.yml, ecs.yml"dynamodb":
cloud.machine.typeagent.yml, ecs.yml"dynamodb":
cloud.provideragent.yml, ecs.yml"dynamodb":
cloud.regionagent.yml, ecs.yml"ebs":
cloud.account.idagent.yml, ecs.yml"ebs":
cloud.availability_zoneagent.yml, ecs.yml"ebs":
cloud.instance.idagent.yml, ecs.yml"ebs":
cloud.machine.typeagent.yml, ecs.yml"ebs":
cloud.provideragent.yml, ecs.yml"ebs":
cloud.regionagent.yml, ecs.yml"ecs_metrics":
cloud.account.idagent.yml, ecs.yml"ecs_metrics":
cloud.availability_zoneagent.yml, ecs.yml"ecs_metrics":
cloud.instance.idagent.yml, ecs.yml"ecs_metrics":
cloud.machine.typeagent.yml, ecs.yml"ecs_metrics":
cloud.provideragent.yml, ecs.yml"ecs_metrics":
cloud.regionagent.yml, ecs.yml"elb_logs":
cloud.provideragent.yml, ecs.yml"elb_metrics":
cloud.account.idagent.yml, ecs.yml"elb_metrics":
cloud.availability_zoneagent.yml, ecs.yml"elb_metrics":
cloud.instance.idagent.yml, ecs.yml"elb_metrics":
cloud.machine.typeagent.yml, ecs.yml"elb_metrics":
cloud.provideragent.yml, ecs.yml"elb_metrics":
cloud.regionagent.yml, ecs.yml"firewall_logs":
cloud.account.idagent.yml, ecs.yml"firewall_logs":
cloud.regionagent.yml, ecs.yml"firewall_metrics":
cloud.account.idagent.yml, ecs.yml"firewall_metrics":
cloud.availability_zoneagent.yml, ecs.yml"firewall_metrics":
cloud.instance.idagent.yml, ecs.yml"firewall_metrics":
cloud.machine.typeagent.yml, ecs.yml"firewall_metrics":
cloud.provideragent.yml, ecs.yml"firewall_metrics":
cloud.regionagent.yml, ecs.yml"lambda":
cloud.account.idagent.yml, ecs.yml"lambda":
cloud.availability_zoneagent.yml, ecs.yml"lambda":
cloud.instance.idagent.yml, ecs.yml"lambda":
cloud.machine.typeagent.yml, ecs.yml"lambda":
cloud.provideragent.yml, ecs.yml"lambda":
cloud.regionagent.yml, ecs.yml"natgateway":
cloud.account.idagent.yml, ecs.yml"natgateway":
cloud.availability_zoneagent.yml, ecs.yml"natgateway":
cloud.instance.idagent.yml, ecs.yml"natgateway":
cloud.machine.typeagent.yml, ecs.yml"natgateway":
cloud.provideragent.yml, ecs.yml"natgateway":
cloud.regionagent.yml, ecs.yml"rds":
cloud.account.idagent.yml, ecs.yml"rds":
cloud.availability_zoneagent.yml, ecs.yml"rds":
cloud.instance.idagent.yml, ecs.yml"rds":
cloud.machine.typeagent.yml, ecs.yml"rds":
cloud.provideragent.yml, ecs.yml"rds":
cloud.regionagent.yml, ecs.yml"s3_daily_storage":
cloud.account.idagent.yml, ecs.yml"s3_daily_storage":
cloud.availability_zoneagent.yml, ecs.yml"s3_daily_storage":
cloud.instance.idagent.yml, ecs.yml"s3_daily_storage":
cloud.machine.typeagent.yml, ecs.yml"s3_daily_storage":
cloud.provideragent.yml, ecs.yml"s3_daily_storage":
cloud.regionagent.yml, ecs.yml"s3_request":
cloud.account.idagent.yml, ecs.yml"s3_request":
cloud.availability_zoneagent.yml, ecs.yml"s3_request":
cloud.instance.idagent.yml, ecs.yml"s3_request":
cloud.machine.typeagent.yml, ecs.yml"s3_request":
cloud.provideragent.yml, ecs.yml"s3_request":
cloud.regionagent.yml, ecs.yml"s3_storage_lens":
cloud.account.idagent.yml, ecs.yml"s3_storage_lens":
cloud.availability_zoneagent.yml, ecs.yml"s3_storage_lens":
cloud.instance.idagent.yml, ecs.yml"s3_storage_lens":
cloud.machine.typeagent.yml, ecs.yml"s3_storage_lens":
cloud.provideragent.yml, ecs.yml"s3_storage_lens":
cloud.regionagent.yml, ecs.yml"s3access":
cloud.provideragent.yml, ecs.yml"sns":
cloud.account.idagent.yml, ecs.yml"sns":
cloud.availability_zoneagent.yml, ecs.yml"sns":
cloud.instance.idagent.yml, ecs.yml"sns":
cloud.machine.typeagent.yml, ecs.yml"sns":
cloud.provideragent.yml, ecs.yml"sns":
cloud.regionagent.yml, ecs.yml"sqs":
cloud.account.idagent.yml, ecs.yml"sqs":
cloud.availability_zoneagent.yml, ecs.yml"sqs":
cloud.instance.idagent.yml, ecs.yml"sqs":
cloud.machine.typeagent.yml, ecs.yml"sqs":
cloud.provideragent.yml, ecs.yml"sqs":
cloud.regionagent.yml, ecs.yml"transitgateway":
cloud.account.idagent.yml, ecs.yml"transitgateway":
cloud.availability_zoneagent.yml, ecs.yml"transitgateway":
cloud.instance.idagent.yml, ecs.yml"transitgateway":
cloud.machine.typeagent.yml, ecs.yml"transitgateway":
cloud.provideragent.yml, ecs.yml"transitgateway":
cloud.regionagent.yml, ecs.yml"usage":
cloud.account.idagent.yml, ecs.yml"usage":
cloud.availability_zoneagent.yml, ecs.yml"usage":
cloud.instance.idagent.yml, ecs.yml"usage":
cloud.machine.typeagent.yml, ecs.yml"usage":
cloud.provideragent.yml, ecs.yml"usage":
cloud.regionagent.yml, ecs.yml"vpcflow":
cloud.account.idagent.yml, ecs.yml"vpcflow":
cloud.instance.idagent.yml, ecs.yml"vpcflow":
cloud.provideragent.yml, ecs.yml"vpcflow":
source.as.organization.nameecs.yml, ecs.yml"vpn":
cloud.account.idagent.yml, ecs.yml"vpn":
cloud.availability_zoneagent.yml, ecs.yml"vpn":
cloud.instance.idagent.yml, ecs.yml"vpn":
cloud.machine.typeagent.yml, ecs.yml"vpn":
cloud.provideragent.yml, ecs.yml"vpn":
cloud.regionagent.yml, ecs.yml"waf":
cloud.provideragent.yml, ecs.yml"spamfirewall":
@timestampbase-fields.yml, ecs.yml"spamfirewall":
tagsbase-fields.yml, ecs.yml"waf":
@timestampbase-fields.yml, ecs.yml"waf":
tagsbase-fields.yml, ecs.yml"director":
@timestampbase-fields.yml, ecs.yml"director":
tagsbase-fields.yml, ecs.yml"events":
box.source.idfields.yml, fields.yml"alert":
host.hostnameagent.yml, ecs.yml"alert":
host.idagent.yml, ecs.yml"alert":
host.ipagent.yml, ecs.yml"alert":
host.nameagent.yml, ecs.yml"alert":
host.os.versionagent.yml, ecs.yml"asset_vulnerability_summary":
host.hostnameagent.yml, ecs.yml"asset_vulnerability_summary":
host.idagent.yml, ecs.yml"asset_vulnerability_summary":
host.nameagent.yml, ecs.yml"asset_vulnerability_summary":
host.os.nameagent.yml, ecs.yml"asset_vulnerability_summary":
host.os.versionagent.yml, ecs.yml"endpoint_event":
host.hostnameagent.yml, ecs.yml"endpoint_event":
host.idagent.yml, ecs.yml"endpoint_event":
host.ipagent.yml, ecs.yml"endpoint_event":
host.nameagent.yml, ecs.yml"endpoint_event":
host.os.familyagent.yml, ecs.yml"endpoint_event":
host.os.nameagent.yml, ecs.yml"watchlist_hit":
host.hostnameagent.yml, ecs.yml"watchlist_hit":
host.idagent.yml, ecs.yml"watchlist_hit":
host.ipagent.yml, ecs.yml"watchlist_hit":
host.nameagent.yml, ecs.yml"log":
cef.extensions.deviceCustomIPv6Address2Labelfields.yml, fields.yml"log":
cef.extensions.deviceCustomIPv6Address2fields.yml, fields.yml"log":
cef.extensions.deviceCustomIPv6Address3Labelfields.yml, fields.yml"log":
cef.extensions.deviceCustomIPv6Address3fields.yml, fields.yml"log":
event.createdecs.yml, ecs.yml"log":
server.domainecs.yml, ecs.yml"log":
event.createdecs.yml, ecs.yml"log":
server.domainecs.yml, ecs.yml"log":
event.createdecs.yml, ecs.yml"log":
cisco_ise.log.statefields.yml, fields.yml"log":
host.hostnameagent.yml, ecs.yml"log":
host.ipagent.yml, ecs.yml"events":
container.idagent.yml, base-fields.yml"events":
network.directionecs.yml, ecs.yml"events":
network.protocolecs.yml, ecs.yml"log":
container.idagent.yml, base-fields.yml"log":
network.directionecs.yml, ecs.yml"log":
network.protocolecs.yml, ecs.yml"log":
container.idagent.yml, base-fields.yml"log":
tagsbase-fields.yml, ecs.yml"log":
input.typeagent.yml, fields.yml"event":
container.idagent.yml, base-fields.yml"event":
event.codeecs.yml, ecs.yml"log":
@timestampbase-fields.yml, ecs.yml"log":
event.createdecs.yml, ecs.yml"log":
server.domainecs.yml, ecs.yml"logpull":
client.geo.country_iso_codeecs.yml, ecs.yml"events":
cef.extensions.deviceCustomIPv6Address2Labelcef.yml, cef.yml"events":
cef.extensions.deviceCustomIPv6Address2cef.yml, cef.yml"events":
cef.extensions.deviceCustomIPv6Address3Labelcef.yml, cef.yml"events":
cef.extensions.deviceCustomIPv6Address3cef.yml, cef.yml"protect":
@timestampbase-fields.yml, ecs.yml"protect":
tagsbase-fields.yml, ecs.yml"ai_analyst_alert":
host.hostnameagent.yml, ecs.yml"ai_analyst_alert":
host.idagent.yml, ecs.yml"ai_analyst_alert":
host.ipagent.yml, ecs.yml"ai_analyst_alert":
host.nameagent.yml, ecs.yml"model_breach_alert":
host.hostnameagent.yml, ecs.yml"model_breach_alert":
host.idagent.yml, ecs.yml"model_breach_alert":
host.ipagent.yml, ecs.yml"model_breach_alert":
host.typeagent.yml, ecs.yml"system_status_alert":
host.hostnameagent.yml, ecs.yml"system_status_alert":
host.ipagent.yml, ecs.yml"bigipafm":
@timestampbase-fields.yml, ecs.yml"bigipafm":
tagsbase-fields.yml, ecs.yml"bigipapm":
@timestampbase-fields.yml, ecs.yml"bigipapm":
tagsbase-fields.yml, ecs.yml"event":
container.nameagent.yml, ecs.yml"event":
host.architectureagent.yml, ecs.yml"nx":
destination.portecs.yml, ecs.yml"nx":
host.ipagent.yml, ecs.yml"log":
container.idagent.yml, base-fields.yml"log":
tagsbase-fields.yml, ecs.yml"log":
container.idagent.yml, ecs.yml"log":
container.idagent.yml, base-fields.yml"log":
tagsbase-fields.yml, ecs.yml"log":
container.idagent.yml, base-fields.yml"log":
tagsbase-fields.yml, ecs.yml"redis":
cloud.account.idagent.yml, ecs.yml"redis":
cloud.availability_zoneagent.yml, ecs.yml"redis":
cloud.instance.idagent.yml, ecs.yml"redis":
cloud.machine.typeagent.yml, ecs.yml"redis":
cloud.provideragent.yml, ecs.yml"redis":
cloud.regionagent.yml, ecs.yml"admin":
container.nameagent.yml, ecs.yml"drive":
container.nameagent.yml, ecs.yml"groups":
container.nameagent.yml, ecs.yml"login":
container.nameagent.yml, ecs.yml"saml":
container.nameagent.yml, ecs.yml"user_accounts":
container.nameagent.yml, ecs.yml"log":
event.createdecs.yml, ecs.yml"log":
log.offsetagent.yml, base-fields.yml"log":
server.domainecs.yml, ecs.yml"winlog":
event.modulebase-fields.yml, ecs.yml"winlog":
host.nameagent.yml, ecs.yml"securesphere":
@timestampbase-fields.yml, ecs.yml"securesphere":
tagsbase-fields.yml, ecs.yml"dhcp_lease":
host.hostnameagent.yml, ecs.yml"dhcp_lease":
host.nameagent.yml, ecs.yml"log":
host.ipagent.yml, ecs.yml"log":
container.idagent.yml, base-fields.yml"log":
tagsbase-fields.yml, ecs.yml"log":
container.idagent.yml, base-fields.yml"log":
tagsbase-fields.yml, ecs.yml"log":
container.idagent.yml, ecs.yml"log":
container.image.nameagent.yml, ecs.yml"log":
container.labelsagent.yml, ecs.yml"log":
container.nameagent.yml, ecs.yml"log":
url.fullecs.yml, ecs.yml"log":
container.idagent.yml, ecs.yml"log":
container.image.nameagent.yml, ecs.yml"log":
container.labelsagent.yml, ecs.yml"log":
container.nameagent.yml, ecs.yml"siem_logs":
email.attachments.file.nameecs.yml, ecs.yml"siem_logs":
event.actionecs.yml, ecs.yml"ttp_ap_logs":
email.attachments.file.mime_typeecs.yml, ecs.yml"ttp_ap_logs":
event.actionecs.yml, ecs.yml"ttp_ip_logs":
event.actionecs.yml, ecs.yml"ttp_url_logs":
event.actionecs.yml, ecs.yml"auditlog":
host.ipagent.yml, ecs.yml"log":
container.idagent.yml, ecs.yml"log":
container.image.nameagent.yml, ecs.yml"log":
container.labelsagent.yml, ecs.yml"log":
container.nameagent.yml, ecs.yml"log":
host.os.kernelagent.yml, ecs.yml"log":
host.os.platformagent.yml, ecs.yml"log":
host.os.versionagent.yml, ecs.yml"log":
host.typeagent.yml, ecs.yml"sightline":
@timestampbase-fields.yml, ecs.yml"sightline":
tagsbase-fields.yml, ecs.yml"alerts":
cloud.account.idagent.yml, ecs.yml"alerts":
destination.ipecs.yml, ecs.yml"alerts":
host.hostnameagent.yml, ecs.yml"alerts":
host.os.nameagent.yml, ecs.yml"alerts":
source.ipecs.yml, ecs.yml"events":
client.bytesecs.yml, ecs.yml"events":
client.packetsecs.yml, ecs.yml"events":
cloud.regionagent.yml, ecs.yml"events":
host.hostnameagent.yml, ecs.yml"audit":
container.idagent.yml, ecs.yml"audit":
host.idagent.yml, ecs.yml"audit":
host.nameagent.yml, ecs.yml"system":
container.idagent.yml, ecs.yml"panos":
container.idagent.yml, ecs.yml"panos":
host.idagent.yml, ecs.yml"panos":
host.ipagent.yml, ecs.yml"panos":
host.macagent.yml, ecs.yml"panos":
host.nameagent.yml, ecs.yml"panos":
host.os.familyagent.yml, ecs.yml"panos":
source.user.nameecs.yml, ecs.yml"alerts":
host.domainagent.yml, ecs.yml"alerts":
host.hostnameagent.yml, ecs.yml"alerts":
host.idagent.yml, ecs.yml"alerts":
host.ipagent.yml, ecs.yml"alerts":
host.macagent.yml, ecs.yml"alerts":
host.os.versionagent.yml, ecs.yml"log":
source.as.numberecs.yml, ecs.yml"log":
source.as.organization.nameecs.yml, ecs.yml"log":
source.geo.continent_nameecs.yml, ecs.yml"log":
source.geo.country_iso_codeecs.yml, ecs.yml"log":
source.geo.country_nameecs.yml, ecs.yml"defensepro":
@timestampbase-fields.yml, ecs.yml"defensepro":
tagsbase-fields.yml, ecs.yml"audit":
event.categoryecs.yml, ecs.yml"audit":
slack.audit.entity.timestampfields.yml, fields.yml"audit":
user_agent.device.nameecs.yml, ecs.yml"audit":
user_agent.nameecs.yml, ecs.yml"audit":
user_agent.originalecs.yml, ecs.yml"audit":
user_agent.os.nameecs.yml, ecs.yml"utm":
tagsbase-fields.yml, ecs.yml"xg":
sophos.xg.syslog_server_namefields.yml, fields.yml"log":
@timestampbase-fields.yml, ecs.yml"log":
tagsbase-fields.yml, ecs.yml"log":
user_agent.originalecs.yml, ecs.yml"eve":
@timestampbase-fields.yml, ecs.yml"eve":
host.ipagent.yml, ecs.yml"log":
container.idagent.yml, ecs.yml"threat":
messageecs.yml, ecs.yml"threat":
threat.feed.namebase-fields.yml, ecs.yml"log":
@timestampbase-fields.yml, ecs.yml"log":
tagsecs.yml, ecs.yml"firewall":
host.hostnameagent.yml, ecs.yml"browser_access":
client.geo.country_iso_codeecs.yml, ecs.yml