-
Notifications
You must be signed in to change notification settings - Fork 190
[BUG] Aggregation pushdown return unexpected result when having inner filter #3996
Copy link
Copy link
Closed
Labels
PPLPiped processing languagePiped processing languagePriority-HighbugSomething isn't workingSomething isn't workingcalcitecalcite migration releatedcalcite migration releatedpushdownpushdown related issuespushdown related issues
Description
What is the bug?
Aggregation script pushdown return unexpected result when reference field (body) is text type.
How can one reproduce the bug?
###
POST {{baseUrl}}/_plugins/_ppl/
Content-Type: application/x-ndjson
{
"query": "source=logs-otel-001| parse body 'HTTP/1.1\" (?<httpstatus>\\d+)' | eval status2xx=if(httpstatus>='200' and httpstatus<'300', 1, 0), status3xx=if(httpstatus>='300' and httpstatus<'400', 1, 0), status4xx=if(httpstatus>='400' and httpstatus<'500', 1, 0), status5xx=if(httpstatus>='500' and httpstatus<'600', 1, 0), statusOther=if(httpstatus>='600', 1, 0) | stats count() as `Request Count`, sum(status2xx) as `HTTP 2xx`, sum(status3xx) as `HTTP 3xx`, sum(status4xx) as `HTTP 4xx`, sum(status5xx) as `HTTP 5xx`, sum(statusOther) as `Other` "
}
{
"datarows": [
[
1,
1,
1,
1,
1,
1
]
],
"total": 1,
"size": 1
}
What is the expected behavior?
Expected resutls should be as same as plugins.calcite.pushdown.enabled=false.
If field is text type, we should read from _source instead of _docValue.
"datarows": [
[
1,
1,
0,
0,
0,
0
]
],
What is your host/environment?
main branch
Do you have any screenshots?
n/a
Do you have any additional context?
PUT {{baseUrl}}/logs-otel-001/
Content-Type: application/x-ndjson
{"mappings":{"dynamic_templates":[{"long_resource_attributes":{"path_match":"resource.attributes.*","match_mapping_type":"long","mapping":{"type":"long"}}},{"double_resource_attributes":{"path_match":"resource.attributes.*","match_mapping_type":"double","mapping":{"type":"double"}}},{"string_resource_attributes":{"path_match":"resource.attributes.*","match_mapping_type":"string","mapping":{"ignore_above":256,"type":"keyword"}}},{"long_scope_attributes":{"path_match":"instrumentationScope.attributes.*","match_mapping_type":"long","mapping":{"type":"long"}}},{"double_scope_attributes":{"path_match":"instrumentationScope.attributes.*","match_mapping_type":"double","mapping":{"type":"double"}}},{"string_scope_attributes":{"path_match":"instrumentationScope.attributes.*","match_mapping_type":"string","mapping":{"ignore_above":256,"type":"keyword"}}},{"long_attributes":{"path_match":"attributes.*","match_mapping_type":"long","mapping":{"type":"long"}}},{"double_attributes":{"path_match":"attributes.*","match_mapping_type":"double","mapping":{"type":"double"}}},{"string_attributes":{"path_match":"attributes.*","match_mapping_type":"string","mapping":{"ignore_above":256,"type":"keyword"}}}],"date_detection":false,"properties":{"@timestamp":{"type":"date_nanos"},"attributes":{"properties":{"@OrderResult":{"type":"keyword","ignore_above":256},"amount":{"properties":{"currencyCode":{"type":"keyword","ignore_above":256},"nanos":{"type":"long"},"units":{"properties":{"high":{"type":"long"},"low":{"type":"long"},"unsigned":{"type":"boolean"}}}}},"app":{"properties":{"order":{"properties":{"amount":{"type":"double"},"id":{"type":"keyword","ignore_above":256},"items":{"properties":{"count":{"type":"long"}}}}},"product":{"properties":{"id":{"type":"keyword","ignore_above":256},"name":{"type":"keyword","ignore_above":256}}},"shipping":{"properties":{"amount":{"type":"double"},"tracking":{"properties":{"id":{"type":"keyword","ignore_above":256}}}}}}},"cardType":{"type":"keyword","ignore_above":256},"code":{"properties":{"file":{"properties":{"path":{"type":"keyword","ignore_above":256}}},"function":{"properties":{"name":{"type":"keyword","ignore_above":256}}},"line":{"properties":{"number":{"type":"long"}}}}},"context":{"properties":{"total":{"type":"double"}}},"destination":{"properties":{"address":{"type":"keyword","ignore_above":256}}},"event":{"properties":{"name":{"type":"keyword","ignore_above":256}}},"exception":{"properties":{"message":{"type":"keyword","ignore_above":256},"stacktrace":{"type":"keyword","ignore_above":256},"type":{"type":"keyword","ignore_above":256}}},"lastFourDigits":{"type":"keyword","ignore_above":256},"loyalty_level":{"type":"keyword","ignore_above":256},"name":{"type":"keyword","ignore_above":256},"otelServiceName":{"type":"keyword","ignore_above":256},"otelSpanID":{"type":"keyword","ignore_above":256},"otelTraceID":{"type":"keyword","ignore_above":256},"otelTraceSampled":{"type":"boolean"},"productId":{"type":"keyword","ignore_above":256},"products":{"type":"long"},"quantity":{"type":"long"},"quote":{"properties":{"cents":{"type":"long"},"dollars":{"type":"long"}}},"quote_service_addr":{"type":"keyword","ignore_above":256},"request":{"properties":{"amount":{"properties":{"currencyCode":{"type":"keyword","ignore_above":256},"nanos":{"type":"long"},"units":{"properties":{"high":{"type":"long"},"low":{"type":"long"},"unsigned":{"type":"boolean"}}}}},"creditCard":{"properties":{"creditCardCvv":{"type":"long"},"creditCardExpirationMonth":{"type":"long"},"creditCardExpirationYear":{"type":"long"},"creditCardNumber":{"type":"keyword","ignore_above":256}}}}},"server":{"properties":{"address":{"type":"keyword","ignore_above":256}}},"source":{"properties":{"address":{"type":"keyword","ignore_above":256}}},"tracking_id":{"type":"keyword","ignore_above":256},"transactionId":{"type":"keyword","ignore_above":256},"transaction_id":{"type":"keyword","ignore_above":256},"upstream":{"properties":{"cluster":{"type":"keyword","ignore_above":256},"host":{"type":"keyword","ignore_above":256}}},"url":{"properties":{"full":{"type":"keyword","ignore_above":256},"path":{"type":"keyword","ignore_above":256},"query":{"type":"keyword","ignore_above":256},"template":{"type":"keyword","ignore_above":256}}},"userId":{"type":"keyword","ignore_above":256},"user_agent":{"properties":{"original":{"type":"keyword","ignore_above":256}}},"user_currency":{"type":"keyword","ignore_above":256},"user_id":{"type":"keyword","ignore_above":256}}},"body":{"type":"text"},"droppedAttributesCount":{"type":"integer"},"flags":{"type":"long"},"instrumentationScope":{"properties":{"droppedAttributesCount":{"type":"integer"},"name":{"type":"keyword","ignore_above":128},"schemaUrl":{"type":"keyword","ignore_above":256},"version":{"type":"keyword","ignore_above":64}}},"observedTime":{"type":"object"},"observedTimestamp":{"type":"text","fields":{"keyword":{"type":"keyword","ignore_above":256}}},"resource":{"properties":{"attributes":{"properties":{"cluster_name":{"type":"keyword","ignore_above":256},"container":{"properties":{"id":{"type":"keyword","ignore_above":256}}},"host":{"properties":{"arch":{"type":"keyword","ignore_above":256},"name":{"type":"keyword","ignore_above":256}}},"log_name":{"type":"keyword","ignore_above":256},"node_name":{"type":"keyword","ignore_above":256},"os":{"properties":{"build_id":{"type":"keyword","ignore_above":256},"description":{"type":"keyword","ignore_above":256},"name":{"type":"keyword","ignore_above":256},"type":{"type":"keyword","ignore_above":256},"version":{"type":"keyword","ignore_above":256}}},"process":{"properties":{"command":{"type":"keyword","ignore_above":256},"command_args":{"type":"keyword","ignore_above":256},"command_line":{"type":"keyword","ignore_above":256},"executable":{"properties":{"name":{"type":"keyword","ignore_above":256},"path":{"type":"keyword","ignore_above":256}}},"owner":{"type":"keyword","ignore_above":256},"pid":{"type":"long"},"runtime":{"properties":{"description":{"type":"keyword","ignore_above":256},"name":{"type":"keyword","ignore_above":256},"version":{"type":"keyword","ignore_above":256}}}}},"service":{"properties":{"instance":{"properties":{"id":{"type":"keyword","ignore_above":256}}},"name":{"type":"keyword","ignore_above":256},"namespace":{"type":"keyword","ignore_above":256},"version":{"type":"keyword","ignore_above":256}}},"telemetry":{"properties":{"distro":{"properties":{"name":{"type":"keyword","ignore_above":256},"version":{"type":"keyword","ignore_above":256}}},"sdk":{"properties":{"language":{"type":"keyword","ignore_above":256},"name":{"type":"keyword","ignore_above":256},"version":{"type":"keyword","ignore_above":256}}}}},"zone_name":{"type":"keyword","ignore_above":256}}},"droppedAttributesCount":{"type":"integer"},"schemaUrl":{"type":"keyword","ignore_above":256}}},"schemaUrl":{"type":"text","fields":{"keyword":{"type":"keyword","ignore_above":256}}},"severity":{"properties":{"number":{"type":"integer"},"text":{"type":"keyword","ignore_above":32}}},"severityNumber":{"type":"long"},"severityText":{"type":"text","fields":{"keyword":{"type":"keyword","ignore_above":256}}},"spanId":{"type":"keyword","ignore_above":16},"time":{"type":"date_nanos"},"traceId":{"type":"keyword","ignore_above":32}}}}
###
PUT {{baseUrl}}/logs-otel-001/_doc/1
Content-Type: application/x-ndjson
{
"traceId": "a7cfc2ecf55c11a39c94d8d67f519def",
"instrumentationScope": {
"droppedAttributesCount": 0
},
"resource": {
"schemaUrl": "",
"attributes": {
"cluster_name": "",
"service.name": "frontend-proxy",
"log_name": "otel_envoy_access_log",
"node_name": "",
"zone_name": ""
},
"droppedAttributesCount": 0
},
"flags": 0,
"severityNumber": 0,
"body": "[2025-07-31T05:44:45.164Z] \"GET /api/data HTTP/1.1\" 200 - via_upstream - \"-\" 0 221 2 2 \"-\" \"python-requests/2.32.4\" \"80a2a234-bbb2-9bf3-bbc6-ba7554aee8b6\" \"frontend-proxy:8080\" \"172.18.0.25:8080\" frontend 172.18.0.27:46596 172.18.0.27:8080 172.18.0.26:53294 - -",
"schemaUrl": "",
"spanId": "d7926ff268c9c60d",
"severityText": "",
"attributes": {
"server.address": "172.18.0.27:8080",
"upstream.cluster": "frontend",
"user_agent.original": "python-requests/2.32.4",
"url.query": "-",
"url.path": "/api/data",
"url.template": "-",
"upstream.host": "172.18.0.25:8080",
"event.name": "proxy.access",
"destination.address": "172.18.0.25",
"source.address": "172.18.0.26",
"url.full": "http://frontend-proxy:8080/api/data"
},
"time": "2025-07-31T05:44:45.164182Z",
"droppedAttributesCount": 0,
"observedTimestamp": "1970-01-01T00:00:00Z"
}
Reactions are currently unavailable
Metadata
Metadata
Assignees
Labels
PPLPiped processing languagePiped processing languagePriority-HighbugSomething isn't workingSomething isn't workingcalcitecalcite migration releatedcalcite migration releatedpushdownpushdown related issuespushdown related issues
Type
Projects
Status
Done