You signed in with another tab or window. Reload to refresh your session.You signed out in another tab or window. Reload to refresh your session.You switched accounts on another tab or window. Reload to refresh your session.Dismiss alert
Currently there is no way to distinguish between some documents from AWS Usage. If we enable TSDB with the dimensions set as of now, they will not be enough and we will end up losing data. However, there are no keyword fields available to differentiate between the these set of documents. Example:
This issue might be hard to reproduce. When testing, I got the output: Out of 40000 documents from the index .ds-metrics-aws.usage-default-2023.06.29-000001, 429 of them were discarded., which means that this is happening with just 1% of the documents.
Currently there is no way to distinguish between some documents from AWS Usage. If we enable TSDB with the dimensions set as of now, they will not be enough and we will end up losing data. However, there are no keyword fields available to differentiate between the these set of documents. Example:
Document 1
{ "_index": ".ds-metrics-aws.usage-default-2023.06.29-000001", "_id": "VaIZB4kBLpMqNjezszQ9", "_version": 1, "_score": 0, "_source": { "cloud": { "provider": "aws", "region": "sa-east-1", "account": { "name": "elastic-observability", "id": "627286350134" } }, "agent": { "name": "kind-control-plane", "id": "178edbcb-2132-497d-b6da-e8c7d8095a90", "type": "metricbeat", "ephemeral_id": "e63bc826-7b49-4d1d-85f0-40340b77461d", "version": "8.8.0" }, "@timestamp": "2023-06-29T12:20:00.000Z", "ecs": { "version": "8.0.0" }, "data_stream": { "namespace": "default", "type": "metrics", "dataset": "aws.usage" }, "service": { "type": "aws" }, "elastic_agent": { "id": "178edbcb-2132-497d-b6da-e8c7d8095a90", "version": "8.8.0", "snapshot": true }, "host": { "hostname": "kind-control-plane", "os": { "kernel": "5.15.49-linuxkit", "codename": "focal", "name": "Ubuntu", "type": "linux", "family": "debian", "version": "20.04.6 LTS (Focal Fossa)", "platform": "ubuntu" }, "containerized": false, "ip": [ "10.244.0.1", "10.244.0.1", "10.244.0.1", "172.18.0.2", "fc00:f853:ccd:e793::2", "fe80::42:acff:fe12:2", "172.19.0.4" ], "name": "kind-control-plane", "id": "0aab3a64904042bdb1c956d6fe2fa4f1", "mac": [ "02-42-AC-12-00-02", "02-42-AC-13-00-04", "06-DD-17-EE-41-97", "22-F1-EB-33-1A-13", "66-56-4C-AB-83-C0" ], "architecture": "x86_64" }, "metricset": { "period": 60000, "name": "cloudwatch" }, "aws": { "usage": { "metrics": { "CallCount": { "sum": 28 } } }, "cloudwatch": { "namespace": "AWS/Usage" }, "dimensions": { "Type": "API", "Resource": "ListMetrics", "Service": "CloudWatch", "Class": "None" } }, "event": { "duration": 9649888084, "agent_id_status": "verified", "ingested": "2023-06-29T12:21:12Z", "module": "aws", "dataset": "aws.usage" } }, "fields": { "elastic_agent.version": [ "8.8.0" ], "host.os.name.text": [ "Ubuntu" ], "host.hostname": [ "kind-control-plane" ], "host.mac": [ "02-42-AC-12-00-02", "02-42-AC-13-00-04", "06-DD-17-EE-41-97", "22-F1-EB-33-1A-13", "66-56-4C-AB-83-C0" ], "service.type": [ "aws" ], "host.ip": [ "10.244.0.1", "10.244.0.1", "10.244.0.1", "172.18.0.2", "fc00:f853:ccd:e793::2", "fe80::42:acff:fe12:2", "172.19.0.4" ], "agent.type": [ "metricbeat" ], "aws.dimensions.Class": [ "None" ], "event.module": [ "aws" ], "host.os.version": [ "20.04.6 LTS (Focal Fossa)" ], "host.os.kernel": [ "5.15.49-linuxkit" ], "host.os.name": [ "Ubuntu" ], "aws.cloudwatch.namespace": [ "AWS/Usage" ], "agent.name": [ "kind-control-plane" ], "elastic_agent.snapshot": [ true ], "host.name": [ "kind-control-plane" ], "event.agent_id_status": [ "verified" ], "aws.dimensions.Service": [ "CloudWatch" ], "host.id": [ "0aab3a64904042bdb1c956d6fe2fa4f1" ], "aws.usage.metrics.CallCount.sum": [ 28 ], "cloud.region": [ "sa-east-1" ], "host.os.type": [ "linux" ], "cloud.account.name": [ "elastic-observability" ], "elastic_agent.id": [ "178edbcb-2132-497d-b6da-e8c7d8095a90" ], "data_stream.namespace": [ "default" ], "metricset.period": [ 60000 ], "aws.dimensions.Type": [ "API" ], "host.os.codename": [ "focal" ], "data_stream.type": [ "metrics" ], "event.duration": [ 9649888084 ], "host.architecture": [ "x86_64" ], "metricset.name": [ "cloudwatch" ], "cloud.provider": [ "aws" ], "event.ingested": [ "2023-06-29T12:21:12.000Z" ], "@timestamp": [ "2023-06-29T12:20:00.000Z" ], "agent.id": [ "178edbcb-2132-497d-b6da-e8c7d8095a90" ], "host.containerized": [ false ], "ecs.version": [ "8.0.0" ], "host.os.platform": [ "ubuntu" ], "cloud.account.id": [ "627286350134" ], "data_stream.dataset": [ "aws.usage" ], "agent.ephemeral_id": [ "e63bc826-7b49-4d1d-85f0-40340b77461d" ], "agent.version": [ "8.8.0" ], "aws.dimensions.Resource": [ "ListMetrics" ], "host.os.family": [ "debian" ], "event.dataset": [ "aws.usage" ] } }Document 2
{ "_index": ".ds-metrics-aws.usage-default-2023.06.29-000001", "_id": "aaIaB4kBLpMqNjezKDWL", "_version": 1, "_score": 0, "_source": { "cloud": { "provider": "aws", "region": "sa-east-1", "account": { "name": "elastic-observability", "id": "627286350134" } }, "agent": { "name": "kind-control-plane", "id": "178edbcb-2132-497d-b6da-e8c7d8095a90", "type": "metricbeat", "ephemeral_id": "e63bc826-7b49-4d1d-85f0-40340b77461d", "version": "8.8.0" }, "@timestamp": "2023-06-29T12:20:00.000Z", "ecs": { "version": "8.0.0" }, "service": { "type": "aws" }, "data_stream": { "namespace": "default", "type": "metrics", "dataset": "aws.usage" }, "host": { "hostname": "kind-control-plane", "os": { "kernel": "5.15.49-linuxkit", "codename": "focal", "name": "Ubuntu", "type": "linux", "family": "debian", "version": "20.04.6 LTS (Focal Fossa)", "platform": "ubuntu" }, "ip": [ "10.244.0.1", "10.244.0.1", "10.244.0.1", "172.18.0.2", "fc00:f853:ccd:e793::2", "fe80::42:acff:fe12:2", "172.19.0.4" ], "containerized": false, "name": "kind-control-plane", "id": "0aab3a64904042bdb1c956d6fe2fa4f1", "mac": [ "02-42-AC-12-00-02", "02-42-AC-13-00-04", "06-DD-17-EE-41-97", "22-F1-EB-33-1A-13", "66-56-4C-AB-83-C0" ], "architecture": "x86_64" }, "elastic_agent": { "id": "178edbcb-2132-497d-b6da-e8c7d8095a90", "version": "8.8.0", "snapshot": true }, "metricset": { "period": 60000, "name": "cloudwatch" }, "aws": { "usage": { "metrics": { "CallCount": { "sum": 40 } } }, "cloudwatch": { "namespace": "AWS/Usage" }, "dimensions": { "Type": "API", "Resource": "ListMetrics", "Service": "CloudWatch", "Class": "None" } }, "event": { "duration": 9720431083, "agent_id_status": "verified", "ingested": "2023-06-29T12:21:42Z", "module": "aws", "dataset": "aws.usage" } }, "fields": { "elastic_agent.version": [ "8.8.0" ], "host.os.name.text": [ "Ubuntu" ], "host.hostname": [ "kind-control-plane" ], "host.mac": [ "02-42-AC-12-00-02", "02-42-AC-13-00-04", "06-DD-17-EE-41-97", "22-F1-EB-33-1A-13", "66-56-4C-AB-83-C0" ], "service.type": [ "aws" ], "host.ip": [ "10.244.0.1", "10.244.0.1", "10.244.0.1", "172.18.0.2", "fc00:f853:ccd:e793::2", "fe80::42:acff:fe12:2", "172.19.0.4" ], "agent.type": [ "metricbeat" ], "aws.dimensions.Class": [ "None" ], "event.module": [ "aws" ], "host.os.version": [ "20.04.6 LTS (Focal Fossa)" ], "host.os.kernel": [ "5.15.49-linuxkit" ], "host.os.name": [ "Ubuntu" ], "aws.cloudwatch.namespace": [ "AWS/Usage" ], "agent.name": [ "kind-control-plane" ], "elastic_agent.snapshot": [ true ], "host.name": [ "kind-control-plane" ], "event.agent_id_status": [ "verified" ], "aws.dimensions.Service": [ "CloudWatch" ], "host.id": [ "0aab3a64904042bdb1c956d6fe2fa4f1" ], "aws.usage.metrics.CallCount.sum": [ 40 ], "cloud.region": [ "sa-east-1" ], "host.os.type": [ "linux" ], "cloud.account.name": [ "elastic-observability" ], "elastic_agent.id": [ "178edbcb-2132-497d-b6da-e8c7d8095a90" ], "data_stream.namespace": [ "default" ], "metricset.period": [ 60000 ], "aws.dimensions.Type": [ "API" ], "host.os.codename": [ "focal" ], "data_stream.type": [ "metrics" ], "event.duration": [ 9720431083 ], "host.architecture": [ "x86_64" ], "metricset.name": [ "cloudwatch" ], "cloud.provider": [ "aws" ], "event.ingested": [ "2023-06-29T12:21:42.000Z" ], "@timestamp": [ "2023-06-29T12:20:00.000Z" ], "agent.id": [ "178edbcb-2132-497d-b6da-e8c7d8095a90" ], "host.containerized": [ false ], "ecs.version": [ "8.0.0" ], "host.os.platform": [ "ubuntu" ], "cloud.account.id": [ "627286350134" ], "data_stream.dataset": [ "aws.usage" ], "agent.ephemeral_id": [ "e63bc826-7b49-4d1d-85f0-40340b77461d" ], "agent.version": [ "8.8.0" ], "aws.dimensions.Resource": [ "ListMetrics" ], "host.os.family": [ "debian" ], "event.dataset": [ "aws.usage" ] } }This issue might be hard to reproduce. When testing, I got the output:
Out of 40000 documents from the index .ds-metrics-aws.usage-default-2023.06.29-000001, 429 of them were discarded., which means that this is happening with just 1% of the documents.