[SIEM] [Maps] Network Map fails to load data with failed request to '/internal/search/es'

In testing https://github.com/elastic/kibana/pull/61165, it was noticed that the SIEM Network Map (Map Embeddable) was failing to load data. The same behavior was then verified against master (e202fe7aa31721a4c319aa414cc4e6739b9bf000), albeit slightly different (sometimes returning a `403` instead of `400`).

This can be verified internally by on `siem-dev` here: https://kibana.siem.estc.dev/app/siem#/network/flows

##### `/internal/search/es` -- `400` (consistent)

<img src="https://user-images.githubusercontent.com/2946766/78283181-aea59080-74da-11ea-9048-4f59bf66dd1e.png" width=500 />
<details><summary>Request paylod</summary>



```json
{
 "params": {
 "ignoreThrottled": true,
 "preference": 1585846087508,
 "index": "auditbeat-*",
 "body": {
 "docvalue_fields": ["source.geo.location"],
 "size": 10000,
 "_source": false,
 "stored_fields": ["source.geo.location"],
 "script_fields": {},
 "query": {
 "bool": {
 "must": [],
 "filter": [
 { "match_all": {} },
 { "match_all": {} },
 {
 "range": {
 "@timestamp": {
 "gte": "2020-04-02T16:34:32.538Z",
 "lte": "2020-04-02T16:49:32.538Z",
 "format": "strict_date_optional_time"
 }
 }
 }
 ],
 "should": [],
 "must_not": []
 }
 }
 },
 "rest_total_hits_as_int": true,
 "ignore_unavailable": true,
 "ignore_throttled": true,
 "timeout": "30000ms"
 },
 "serverStrategy": "es"
}
```


</details>

<details><summary>Response payload</summary>


```json
{
 "statusCode": 400,
 "error": "Bad Request",
 "message": "Bad Request",
 "attributes": { "error": "Bad Request" }
}

```


</details>



##### `/internal/search/es` -- `403` (sporadic)

<img src="https://user-images.githubusercontent.com/2946766/78284075-0d1f3e80-74dc-11ea-80fd-f26b32314879.png" width=500 />

<details><summary>Request payload</summary>


``` json
{
 "params": {
 "ignoreThrottled": true,
 "preference": 1585849411730,
 "index": "filebeat-*",
 "body": {
 "size": 0,
 "aggs": {
 "destSplit": {
 "terms": {
 "script": {
 "source": "doc['destination.geo.location'].value.toString()",
 "lang": "painless"
 },
 "order": { "_count": "desc" },
 "size": 100
 },
 "aggs": {
 "sourceGrid": {
 "geotile_grid": {
 "field": "source.geo.location",
 "precision": 6,
 "size": 500
 },
 "aggs": {
 "sourceCentroid": {
 "geo_centroid": { "field": "source.geo.location" }
 },
 "sum_of_source.bytes": { "sum": { "field": "source.bytes" } },
 "sum_of_destination.bytes": {
 "sum": { "field": "destination.bytes" }
 }
 }
 }
 }
 }
 },
 "stored_fields": ["*"],
 "script_fields": {},
 "docvalue_fields": [
 { "field": "@timestamp", "format": "date_time" },
 {
 "field": "azure.auditlogs.properties.activity_datetime",
 "format": "date_time"
 },
 { "field": "azure.enqueued_time", "format": "date_time" },
 { "field": "cef.extensions.agentReceiptTime", "format": "date_time" },
 { "field": "cef.extensions.deviceCustomDate1", "format": "date_time" },
 { "field": "cef.extensions.deviceCustomDate2", "format": "date_time" },
 { "field": "cef.extensions.deviceReceiptTime", "format": "date_time" },
 { "field": "cef.extensions.endTime", "format": "date_time" },
 { "field": "cef.extensions.fileCreateTime", "format": "date_time" },
 {
 "field": "cef.extensions.fileModificationTime",
 "format": "date_time"
 },
 { "field": "cef.extensions.flexDate1", "format": "date_time" },
 { "field": "cef.extensions.managerReceiptTime", "format": "date_time" },
 { "field": "cef.extensions.oldFileCreateTime", "format": "date_time" },
 {
 "field": "cef.extensions.oldFileModificationTime",
 "format": "date_time"
 },
 { "field": "cef.extensions.startTime", "format": "date_time" },
 { "field": "event.created", "format": "date_time" },
 { "field": "event.end", "format": "date_time" },
 { "field": "event.ingested", "format": "date_time" },
 { "field": "event.start", "format": "date_time" },
 { "field": "file.accessed", "format": "date_time" },
 { "field": "file.created", "format": "date_time" },
 { "field": "file.ctime", "format": "date_time" },
 { "field": "file.mtime", "format": "date_time" },
 { "field": "kafka.block_timestamp", "format": "date_time" },
 { "field": "misp.campaign.first_seen", "format": "date_time" },
 { "field": "misp.campaign.last_seen", "format": "date_time" },
 { "field": "misp.intrusion_set.first_seen", "format": "date_time" },
 { "field": "misp.intrusion_set.last_seen", "format": "date_time" },
 { "field": "misp.observed_data.first_observed", "format": "date_time" },
 { "field": "misp.observed_data.last_observed", "format": "date_time" },
 { "field": "misp.report.published", "format": "date_time" },
 { "field": "misp.threat_indicator.valid_from", "format": "date_time" },
 { "field": "misp.threat_indicator.valid_until", "format": "date_time" },
 {
 "field": "netflow.collection_time_milliseconds",
 "format": "date_time"
 },
 { "field": "netflow.flow_end_microseconds", "format": "date_time" },
 { "field": "netflow.flow_end_milliseconds", "format": "date_time" },
 { "field": "netflow.flow_end_nanoseconds", "format": "date_time" },
 { "field": "netflow.flow_end_seconds", "format": "date_time" },
 { "field": "netflow.flow_start_microseconds", "format": "date_time" },
 { "field": "netflow.flow_start_milliseconds", "format": "date_time" },
 { "field": "netflow.flow_start_nanoseconds", "format": "date_time" },
 { "field": "netflow.flow_start_seconds", "format": "date_time" },
 { "field": "netflow.max_export_seconds", "format": "date_time" },
 { "field": "netflow.max_flow_end_microseconds", "format": "date_time" },
 { "field": "netflow.max_flow_end_milliseconds", "format": "date_time" },
 { "field": "netflow.max_flow_end_nanoseconds", "format": "date_time" },
 { "field": "netflow.max_flow_end_seconds", "format": "date_time" },
 { "field": "netflow.min_export_seconds", "format": "date_time" },
 {
 "field": "netflow.min_flow_start_microseconds",
 "format": "date_time"
 },
 {
 "field": "netflow.min_flow_start_milliseconds",
 "format": "date_time"
 },
 {
 "field": "netflow.min_flow_start_nanoseconds",
 "format": "date_time"
 },
 { "field": "netflow.min_flow_start_seconds", "format": "date_time" },
 {
 "field": "netflow.monitoring_interval_end_milli_seconds",
 "format": "date_time"
 },
 {
 "field": "netflow.monitoring_interval_start_milli_seconds",
 "format": "date_time"
 },
 {
 "field": "netflow.observation_time_microseconds",
 "format": "date_time"
 },
 {
 "field": "netflow.observation_time_milliseconds",
 "format": "date_time"
 },
 {
 "field": "netflow.observation_time_nanoseconds",
 "format": "date_time"
 },
 { "field": "netflow.observation_time_seconds", "format": "date_time" },
 {
 "field": "netflow.system_init_time_milliseconds",
 "format": "date_time"
 },
 { "field": "package.installed", "format": "date_time" },
 { "field": "process.parent.start", "format": "date_time" },
 { "field": "process.start", "format": "date_time" },
 { "field": "suricata.eve.flow.end", "format": "date_time" },
 { "field": "suricata.eve.flow.start", "format": "date_time" },
 { "field": "suricata.eve.timestamp", "format": "date_time" },
 { "field": "suricata.eve.tls.notafter", "format": "date_time" },
 { "field": "suricata.eve.tls.notbefore", "format": "date_time" },
 { "field": "tls.client.not_after", "format": "date_time" },
 { "field": "tls.client.not_before", "format": "date_time" },
 { "field": "tls.server.not_after", "format": "date_time" },
 { "field": "tls.server.not_before", "format": "date_time" },
 { "field": "zeek.kerberos.valid.from", "format": "date_time" },
 { "field": "zeek.kerberos.valid.until", "format": "date_time" },
 { "field": "zeek.ocsp.revoke.time", "format": "date_time" },
 { "field": "zeek.ocsp.update.next", "format": "date_time" },
 { "field": "zeek.ocsp.update.this", "format": "date_time" },
 { "field": "zeek.pe.compile_time", "format": "date_time" },
 { "field": "zeek.smb_files.times.accessed", "format": "date_time" },
 { "field": "zeek.smb_files.times.changed", "format": "date_time" },
 { "field": "zeek.smb_files.times.created", "format": "date_time" },
 { "field": "zeek.smb_files.times.modified", "format": "date_time" },
 { "field": "zeek.smtp.date", "format": "date_time" },
 { "field": "zeek.snmp.up_since", "format": "date_time" },
 { "field": "zeek.x509.certificate.valid.from", "format": "date_time" },
 { "field": "zeek.x509.certificate.valid.until", "format": "date_time" }
 ],
 "_source": { "excludes": [] },
 "query": {
 "bool": {
 "must": [],
 "filter": [
 { "match_all": {} },
 { "match_all": {} },
 {
 "geo_bounding_box": {
 "destination.geo.location": {
 "top_left": [-140.625, 48.9225],
 "bottom_right": [-28.125, 21.94305]
 }
 }
 },
 {
 "range": {
 "@timestamp": {
 "gte": "2020-04-01T17:43:34.626Z",
 "lte": "2020-04-02T17:43:34.626Z",
 "format": "strict_date_optional_time"
 }
 }
 }
 ],
 "should": [],
 "must_not": []
 }
 }
 },
 "rest_total_hits_as_int": true,
 "ignore_unavailable": true,
 "ignore_throttled": true,
 "timeout": "30000ms"
 },
 "serverStrategy": "es"
}

```


</details>


<details><summary>Response payload</summary>


``` json
{
 "statusCode": 403,
 "error": "Forbidden",
 "message": "[security_exception] action [indices:data/write/bulk[s]] is unauthorized for user [_async_search]",
 "attributes": {
 "error": {
 "root_cause": [
 {
 "type": "security_exception",
 "reason": "action [indices:data/write/bulk[s]] is unauthorized for user [_async_search]"
 }
 ],
 "type": "security_exception",
 "reason": "action [indices:data/write/bulk[s]] is unauthorized for user [_async_search]"
 }
 }
}

```


</details>

Provide feedback

Saved searches

Use saved searches to filter your results more quickly

[SIEM] [Maps] Network Map fails to load data with failed request to '/internal/search/es' #62356

`/internal/search/es` -- `400` (consistent)

`/internal/search/es` -- `403` (sporadic)

Metadata

Assignees

Labels

Type

Fields

Projects

Milestone

Relationships

Development

[SIEM] [Maps] Network Map fails to load data with failed request to '/internal/search/es' #62356

Description

/internal/search/es -- 400 (consistent)

/internal/search/es -- 403 (sporadic)

Metadata

Metadata

Assignees

Labels

Type

Fields

Projects

Milestone

Relationships

Development

Issue actions

`/internal/search/es` -- `400` (consistent)

`/internal/search/es` -- `403` (sporadic)