Skip to content

difference between OTLP intake and OTel bridge #8067

Closed
Closed
difference between OTLP intake and OTel bridge#8067
Assignees
stuartnelson3
Labels
blockedbug
@SylvainJuge

Description

@SylvainJuge
SylvainJuge
opened on May 6, 2022

APM Server version (apm-server version): 8.x, tested with 8.2

Description of the problem including expected versus actual behavior:

Steps to reproduce:

When capturing an OTel span with the OpenTelemetry SDK, setting the http.status_code to 200 is enough to make the OTel span to be stored as a transaction with the fields http.response.status_code set to 200 and `transaction.

Java code to reproduce (nothing fancy, just a plain usage of the OTel API), identical in both cases, the sent OTel attributes remain identical.

Tracer tracer = GlobalOpenTelemetry.get().getTracer("co.elastic.apm:opbeans");

Span span = tracer.spanBuilder("fake HTTP transaction")
        .startSpan()
        .setAttribute("http.method", "GET")
        .setAttribute("http.status_code", 200);

try {
    sleep(100);
} catch (InterruptedException e) {
    // ignored
}

span.end();

With the OTel SDK:

  • event.outcome = success
  • transaction.result = HTTP 2xx
  • http.response.status_code = 200

with the OTel bridge:

  • event.outcome is set to unknown
  • other fields are missing

After investigating a bit how the transaction/span attributes are inferred from OTel attributes, I think it comes from the fact that the current implementation of APM Server calling TranslateTransaction and TranslateSpan give lower priority to the OTel fields when they should actually have higher priority.

JSON of transaction with the OTel SDK : works as expected 
{
"_index": ".ds-traces-apm-default-2022.05.06-000004",
"_id": "-tpdmoAB8N83zBpI6pgv",
"_version": 1,
"_score": 1,
"_source": {
  "agent": {
    "name": "opentelemetry/java",
    "version": "1.11.0"
  },
  "data_stream.namespace": "default",
  "processor": {
    "name": "transaction",
    "event": "transaction"
  },
  "data_stream.type": "traces",
  "url": {
    "scheme": "http",
    "full": "http:"
  },
  "observer": {
    "hostname": "Sylvain-x390",
    "id": "628c5520-eb81-4182-8474-86d00bbbea4a",
    "type": "apm-server",
    "ephemeral_id": "9858849a-2f9c-4458-a12b-6e3959450095",
    "version": "8.2.0"
  },
  "trace": {
    "id": "bee4190a09e4165ada7f23c472f80d38"
  },
  "@timestamp": "2022-05-06T17:14:41.497Z",
  "ecs": {
    "version": "1.12.0"
  },
  "service": {
    "environment": "local-dev",
    "framework": {
      "name": "co.elastic.apm:opbeans"
    },
    "name": "opbeans",
    "language": {
      "name": "java"
    }
  },
  "data_stream.dataset": "apm",
  "http": {
    "request": {
      "method": "GET"
    },
    "response": {
      "status_code": 200
    }
  },
  "event": {
    "agent_id_status": "missing",
    "ingested": "2022-05-06T17:14:47Z",
    "outcome": "success"
  },
  "transaction": {
    "result": "HTTP 2xx",
    "duration": {
      "us": 102872
    },
    "name": "fake HTTP transaction",
    "id": "c861fb7b8ed3133b",
    "type": "request",
    "sampled": true
  },
  "timestamp": {
    "us": 1651857281497000
  }
},
"fields": {
  "transaction.name.text": [
    "fake HTTP transaction"
  ],
  "url.scheme": [
    "http"
  ],
  "transaction.result": [
    "HTTP 2xx"
  ],
  "service.language.name": [
    "java"
  ],
  "transaction.sampled": [
    true
  ],
  "transaction.id": [
    "c861fb7b8ed3133b"
  ],
  "trace.id": [
    "bee4190a09e4165ada7f23c472f80d38"
  ],
  "http.request.method": [
    "GET"
  ],
  "processor.event": [
    "transaction"
  ],
  "agent.name": [
    "opentelemetry/java"
  ],
  "event.agent_id_status": [
    "missing"
  ],
  "http.response.status_code": [
    200
  ],
  "url.full": [
    "http:"
  ],
  "event.outcome": [
    "success"
  ],
  "service.environment": [
    "local-dev"
  ],
  "service.name": [
    "opbeans"
  ],
  "service.framework.name": [
    "co.elastic.apm:opbeans"
  ],
  "data_stream.namespace": [
    "default"
  ],
  "processor.name": [
    "transaction"
  ],
  "transaction.duration.us": [
    102872
  ],
  "observer.hostname": [
    "Sylvain-x390"
  ],
  "data_stream.type": [
    "traces"
  ],
  "transaction.type": [
    "request"
  ],
  "event.ingested": [
    "2022-05-06T17:14:47.000Z"
  ],
  "observer.id": [
    "628c5520-eb81-4182-8474-86d00bbbea4a"
  ],
  "timestamp.us": [
    1651857281497000
  ],
  "@timestamp": [
    "2022-05-06T17:14:41.497Z"
  ],
  "observer.ephemeral_id": [
    "9858849a-2f9c-4458-a12b-6e3959450095"
  ],
  "observer.version": [
    "8.2.0"
  ],
  "observer.type": [
    "apm-server"
  ],
  "ecs.version": [
    "1.12.0"
  ],
  "data_stream.dataset": [
    "apm"
  ],
  "transaction.name": [
    "fake HTTP transaction"
  ],
  "agent.version": [
    "1.11.0"
  ]
}
}
JSON of a transaction captured with Java agent through OTel bridge 
{
"_index": ".ds-traces-apm-default-2022.05.06-000004",
"_id": "69pamoAB8N83zBpIV3iY",
"_version": 1,
"_score": 1,
"_source": {
  "agent": {
    "name": "java",
    "ephemeral_id": "db91d7d9-d337-4dbe-b70d-c83f252b20d0",
    "version": "1.30.2-SNAPSHOT.null"
  },
  "process": {
    "pid": 1669813,
    "title": "/home/sylvain/.sdkman/candidates/java/11.0.8.hs-adpt/bin/java",
    "ppid": 1669727
  },
  "data_stream.namespace": "default",
  "processor": {
    "name": "transaction",
    "event": "transaction"
  },
  "data_stream.type": "traces",
  "observer": {
    "hostname": "Sylvain-x390",
    "id": "628c5520-eb81-4182-8474-86d00bbbea4a",
    "type": "apm-server",
    "ephemeral_id": "9858849a-2f9c-4458-a12b-6e3959450095",
    "version": "8.2.0"
  },
  "trace": {
    "id": "40e28a7812f558120af9a7372a2d9092"
  },
  "@timestamp": "2022-05-06T17:10:42.458Z",
  "ecs": {
    "version": "1.12.0"
  },
  "service": {
    "node": {
      "name": "Sylvain-x390"
    },
    "environment": "local-dev",
    "framework": {
      "name": "OpenTelemetry",
      "version": "1.11.0"
    },
    "name": "opbeans",
    "runtime": {
      "name": "Java",
      "version": "11.0.8"
    },
    "language": {
      "name": "Java",
      "version": "11.0.8"
    },
    "version": "0.0.1-SNAPSHOT"
  },
  "data_stream.dataset": "apm",
  "host": {
    "hostname": "Sylvain-x390",
    "os": {
      "platform": "Linux"
    },
    "ip": [
      "127.0.0.1"
    ],
    "name": "Sylvain-x390",
    "architecture": "amd64"
  },
  "event": {
    "agent_id_status": "missing",
    "ingested": "2022-05-06T17:10:53Z",
    "outcome": "unknown"
  },
  "transaction": {
    "duration": {
      "us": 113244
    },
    "name": "fake HTTP transaction",
    "span_count": {
      "dropped": 0,
      "started": 0
    },
    "id": "9318a2109629379d",
    "type": "unknown",
    "sampled": true
  },
  "timestamp": {
    "us": 1651857042458023
  }
},
"fields": {
  "transaction.name.text": [
    "fake HTTP transaction"
  ],
  "service.framework.version": [
    "1.11.0"
  ],
  "service.node.name": [
    "Sylvain-x390"
  ],
  "host.hostname": [
    "Sylvain-x390"
  ],
  "process.pid": [
    1669813
  ],
  "service.language.name": [
    "Java"
  ],
  "transaction.sampled": [
    true
  ],
  "transaction.id": [
    "9318a2109629379d"
  ],
  "host.ip": [
    "127.0.0.1"
  ],
  "trace.id": [
    "40e28a7812f558120af9a7372a2d9092"
  ],
  "transaction.span_count.dropped": [
    0
  ],
  "processor.event": [
    "transaction"
  ],
  "agent.name": [
    "java"
  ],
  "host.name": [
    "Sylvain-x390"
  ],
  "event.agent_id_status": [
    "missing"
  ],
  "event.outcome": [
    "unknown"
  ],
  "service.environment": [
    "local-dev"
  ],
  "service.name": [
    "opbeans"
  ],
  "service.framework.name": [
    "OpenTelemetry"
  ],
  "process.ppid": [
    1669727
  ],
  "data_stream.namespace": [
    "default"
  ],
  "processor.name": [
    "transaction"
  ],
  "service.runtime.name": [
    "Java"
  ],
  "transaction.duration.us": [
    113244
  ],
  "service.runtime.version": [
    "11.0.8"
  ],
  "observer.hostname": [
    "Sylvain-x390"
  ],
  "data_stream.type": [
    "traces"
  ],
  "transaction.type": [
    "unknown"
  ],
  "host.architecture": [
    "amd64"
  ],
  "transaction.span_count.started": [
    0
  ],
  "event.ingested": [
    "2022-05-06T17:10:53.000Z"
  ],
  "observer.id": [
    "628c5520-eb81-4182-8474-86d00bbbea4a"
  ],
  "timestamp.us": [
    1651857042458023
  ],
  "@timestamp": [
    "2022-05-06T17:10:42.458Z"
  ],
  "service.version": [
    "0.0.1-SNAPSHOT"
  ],
  "observer.ephemeral_id": [
    "9858849a-2f9c-4458-a12b-6e3959450095"
  ],
  "observer.version": [
    "8.2.0"
  ],
  "host.os.platform": [
    "Linux"
  ],
  "observer.type": [
    "apm-server"
  ],
  "ecs.version": [
    "1.12.0"
  ],
  "data_stream.dataset": [
    "apm"
  ],
  "service.language.version": [
    "11.0.8"
  ],
  "agent.ephemeral_id": [
    "db91d7d9-d337-4dbe-b70d-c83f252b20d0"
  ],
  "transaction.name": [
    "fake HTTP transaction"
  ],
  "agent.version": [
    "1.30.2-SNAPSHOT.null"
  ],
  "process.title": [
    "/home/sylvain/.sdkman/candidates/java/11.0.8.hs-adpt/bin/java"
  ]
}
}

Metadata

Metadata

Assignees

Labels

blockedbug

Type

No type

Fields

No fields configured for issues without a type.

Projects

No projects

Milestone

No milestone

Relationships

None yet

Development

No branches or pull requests

Issue actions