Skip to content

[BUG] Aggregation pushdown return unexpected result when having inner filter #3996

@penghuo

Description

@penghuo

What is the bug?
Aggregation script pushdown return unexpected result when reference field (body) is text type.

How can one reproduce the bug?

###
POST {{baseUrl}}/_plugins/_ppl/
Content-Type: application/x-ndjson

{
  "query": "source=logs-otel-001| parse body 'HTTP/1.1\" (?<httpstatus>\\d+)' | eval status2xx=if(httpstatus>='200' and httpstatus<'300', 1, 0), status3xx=if(httpstatus>='300' and httpstatus<'400', 1, 0), status4xx=if(httpstatus>='400' and httpstatus<'500', 1, 0), status5xx=if(httpstatus>='500' and httpstatus<'600', 1, 0), statusOther=if(httpstatus>='600', 1, 0) | stats count() as `Request Count`, sum(status2xx) as `HTTP 2xx`, sum(status3xx) as `HTTP 3xx`, sum(status4xx) as `HTTP 4xx`, sum(status5xx) as `HTTP 5xx`, sum(statusOther) as `Other` "
}

{
  "datarows": [
    [
      1,
      1,
      1,
      1,
      1,
      1
    ]
  ],
  "total": 1,
  "size": 1
}

What is the expected behavior?
Expected resutls should be as same as plugins.calcite.pushdown.enabled=false.
If field is text type, we should read from _source instead of _docValue.

  "datarows": [
    [
      1,
      1,
      0,
      0,
      0,
      0
    ]
  ],

What is your host/environment?
main branch

Do you have any screenshots?
n/a

Do you have any additional context?

PUT {{baseUrl}}/logs-otel-001/
Content-Type: application/x-ndjson

{"mappings":{"dynamic_templates":[{"long_resource_attributes":{"path_match":"resource.attributes.*","match_mapping_type":"long","mapping":{"type":"long"}}},{"double_resource_attributes":{"path_match":"resource.attributes.*","match_mapping_type":"double","mapping":{"type":"double"}}},{"string_resource_attributes":{"path_match":"resource.attributes.*","match_mapping_type":"string","mapping":{"ignore_above":256,"type":"keyword"}}},{"long_scope_attributes":{"path_match":"instrumentationScope.attributes.*","match_mapping_type":"long","mapping":{"type":"long"}}},{"double_scope_attributes":{"path_match":"instrumentationScope.attributes.*","match_mapping_type":"double","mapping":{"type":"double"}}},{"string_scope_attributes":{"path_match":"instrumentationScope.attributes.*","match_mapping_type":"string","mapping":{"ignore_above":256,"type":"keyword"}}},{"long_attributes":{"path_match":"attributes.*","match_mapping_type":"long","mapping":{"type":"long"}}},{"double_attributes":{"path_match":"attributes.*","match_mapping_type":"double","mapping":{"type":"double"}}},{"string_attributes":{"path_match":"attributes.*","match_mapping_type":"string","mapping":{"ignore_above":256,"type":"keyword"}}}],"date_detection":false,"properties":{"@timestamp":{"type":"date_nanos"},"attributes":{"properties":{"@OrderResult":{"type":"keyword","ignore_above":256},"amount":{"properties":{"currencyCode":{"type":"keyword","ignore_above":256},"nanos":{"type":"long"},"units":{"properties":{"high":{"type":"long"},"low":{"type":"long"},"unsigned":{"type":"boolean"}}}}},"app":{"properties":{"order":{"properties":{"amount":{"type":"double"},"id":{"type":"keyword","ignore_above":256},"items":{"properties":{"count":{"type":"long"}}}}},"product":{"properties":{"id":{"type":"keyword","ignore_above":256},"name":{"type":"keyword","ignore_above":256}}},"shipping":{"properties":{"amount":{"type":"double"},"tracking":{"properties":{"id":{"type":"keyword","ignore_above":256}}}}}}},"cardType":{"type":"keyword","ignore_above":256},"code":{"properties":{"file":{"properties":{"path":{"type":"keyword","ignore_above":256}}},"function":{"properties":{"name":{"type":"keyword","ignore_above":256}}},"line":{"properties":{"number":{"type":"long"}}}}},"context":{"properties":{"total":{"type":"double"}}},"destination":{"properties":{"address":{"type":"keyword","ignore_above":256}}},"event":{"properties":{"name":{"type":"keyword","ignore_above":256}}},"exception":{"properties":{"message":{"type":"keyword","ignore_above":256},"stacktrace":{"type":"keyword","ignore_above":256},"type":{"type":"keyword","ignore_above":256}}},"lastFourDigits":{"type":"keyword","ignore_above":256},"loyalty_level":{"type":"keyword","ignore_above":256},"name":{"type":"keyword","ignore_above":256},"otelServiceName":{"type":"keyword","ignore_above":256},"otelSpanID":{"type":"keyword","ignore_above":256},"otelTraceID":{"type":"keyword","ignore_above":256},"otelTraceSampled":{"type":"boolean"},"productId":{"type":"keyword","ignore_above":256},"products":{"type":"long"},"quantity":{"type":"long"},"quote":{"properties":{"cents":{"type":"long"},"dollars":{"type":"long"}}},"quote_service_addr":{"type":"keyword","ignore_above":256},"request":{"properties":{"amount":{"properties":{"currencyCode":{"type":"keyword","ignore_above":256},"nanos":{"type":"long"},"units":{"properties":{"high":{"type":"long"},"low":{"type":"long"},"unsigned":{"type":"boolean"}}}}},"creditCard":{"properties":{"creditCardCvv":{"type":"long"},"creditCardExpirationMonth":{"type":"long"},"creditCardExpirationYear":{"type":"long"},"creditCardNumber":{"type":"keyword","ignore_above":256}}}}},"server":{"properties":{"address":{"type":"keyword","ignore_above":256}}},"source":{"properties":{"address":{"type":"keyword","ignore_above":256}}},"tracking_id":{"type":"keyword","ignore_above":256},"transactionId":{"type":"keyword","ignore_above":256},"transaction_id":{"type":"keyword","ignore_above":256},"upstream":{"properties":{"cluster":{"type":"keyword","ignore_above":256},"host":{"type":"keyword","ignore_above":256}}},"url":{"properties":{"full":{"type":"keyword","ignore_above":256},"path":{"type":"keyword","ignore_above":256},"query":{"type":"keyword","ignore_above":256},"template":{"type":"keyword","ignore_above":256}}},"userId":{"type":"keyword","ignore_above":256},"user_agent":{"properties":{"original":{"type":"keyword","ignore_above":256}}},"user_currency":{"type":"keyword","ignore_above":256},"user_id":{"type":"keyword","ignore_above":256}}},"body":{"type":"text"},"droppedAttributesCount":{"type":"integer"},"flags":{"type":"long"},"instrumentationScope":{"properties":{"droppedAttributesCount":{"type":"integer"},"name":{"type":"keyword","ignore_above":128},"schemaUrl":{"type":"keyword","ignore_above":256},"version":{"type":"keyword","ignore_above":64}}},"observedTime":{"type":"object"},"observedTimestamp":{"type":"text","fields":{"keyword":{"type":"keyword","ignore_above":256}}},"resource":{"properties":{"attributes":{"properties":{"cluster_name":{"type":"keyword","ignore_above":256},"container":{"properties":{"id":{"type":"keyword","ignore_above":256}}},"host":{"properties":{"arch":{"type":"keyword","ignore_above":256},"name":{"type":"keyword","ignore_above":256}}},"log_name":{"type":"keyword","ignore_above":256},"node_name":{"type":"keyword","ignore_above":256},"os":{"properties":{"build_id":{"type":"keyword","ignore_above":256},"description":{"type":"keyword","ignore_above":256},"name":{"type":"keyword","ignore_above":256},"type":{"type":"keyword","ignore_above":256},"version":{"type":"keyword","ignore_above":256}}},"process":{"properties":{"command":{"type":"keyword","ignore_above":256},"command_args":{"type":"keyword","ignore_above":256},"command_line":{"type":"keyword","ignore_above":256},"executable":{"properties":{"name":{"type":"keyword","ignore_above":256},"path":{"type":"keyword","ignore_above":256}}},"owner":{"type":"keyword","ignore_above":256},"pid":{"type":"long"},"runtime":{"properties":{"description":{"type":"keyword","ignore_above":256},"name":{"type":"keyword","ignore_above":256},"version":{"type":"keyword","ignore_above":256}}}}},"service":{"properties":{"instance":{"properties":{"id":{"type":"keyword","ignore_above":256}}},"name":{"type":"keyword","ignore_above":256},"namespace":{"type":"keyword","ignore_above":256},"version":{"type":"keyword","ignore_above":256}}},"telemetry":{"properties":{"distro":{"properties":{"name":{"type":"keyword","ignore_above":256},"version":{"type":"keyword","ignore_above":256}}},"sdk":{"properties":{"language":{"type":"keyword","ignore_above":256},"name":{"type":"keyword","ignore_above":256},"version":{"type":"keyword","ignore_above":256}}}}},"zone_name":{"type":"keyword","ignore_above":256}}},"droppedAttributesCount":{"type":"integer"},"schemaUrl":{"type":"keyword","ignore_above":256}}},"schemaUrl":{"type":"text","fields":{"keyword":{"type":"keyword","ignore_above":256}}},"severity":{"properties":{"number":{"type":"integer"},"text":{"type":"keyword","ignore_above":32}}},"severityNumber":{"type":"long"},"severityText":{"type":"text","fields":{"keyword":{"type":"keyword","ignore_above":256}}},"spanId":{"type":"keyword","ignore_above":16},"time":{"type":"date_nanos"},"traceId":{"type":"keyword","ignore_above":32}}}}

###
PUT {{baseUrl}}/logs-otel-001/_doc/1
Content-Type: application/x-ndjson

{

    "traceId": "a7cfc2ecf55c11a39c94d8d67f519def",
    "instrumentationScope": {
      "droppedAttributesCount": 0
    },
    "resource": {
      "schemaUrl": "",
      "attributes": {
        "cluster_name": "",
        "service.name": "frontend-proxy",
        "log_name": "otel_envoy_access_log",
        "node_name": "",
        "zone_name": ""
      },
      "droppedAttributesCount": 0
    },
    "flags": 0,
    "severityNumber": 0,
    "body": "[2025-07-31T05:44:45.164Z] \"GET /api/data HTTP/1.1\" 200 - via_upstream - \"-\" 0 221 2 2 \"-\" \"python-requests/2.32.4\" \"80a2a234-bbb2-9bf3-bbc6-ba7554aee8b6\" \"frontend-proxy:8080\" \"172.18.0.25:8080\" frontend 172.18.0.27:46596 172.18.0.27:8080 172.18.0.26:53294 - -",
    "schemaUrl": "",
    "spanId": "d7926ff268c9c60d",
    "severityText": "",
    "attributes": {
      "server.address": "172.18.0.27:8080",
      "upstream.cluster": "frontend",
      "user_agent.original": "python-requests/2.32.4",
      "url.query": "-",
      "url.path": "/api/data",
      "url.template": "-",
      "upstream.host": "172.18.0.25:8080",
      "event.name": "proxy.access",
      "destination.address": "172.18.0.25",
      "source.address": "172.18.0.26",
      "url.full": "http://frontend-proxy:8080/api/data"
    },
    "time": "2025-07-31T05:44:45.164182Z",
    "droppedAttributesCount": 0,
    "observedTimestamp": "1970-01-01T00:00:00Z"
}

Metadata

Metadata

Assignees

Labels

PPLPiped processing languagePriority-HighbugSomething isn't workingcalcitecalcite migration releatedpushdownpushdown related issues

Type

No type

Projects

Status

Done

Milestone

No milestone

Relationships

None yet

Development

No branches or pull requests

Issue actions