Skip to content

Opentelemetry returning 404's as errors in the tracing #29615

@deep-c

Description

@deep-c

Version: Deno 2.3.5

Is a status code of 2 correct for responses that have status 404?

I am running an app at work and have tracing data ingested into AWS X-Ray via ADOT collector on ecs in AWS. I get the same issue using the lgtm stack on my local but the status codes are different..

Here is a sample trace from grafana tempo on my local:

{
    "batches": [
        {
            "resource": {
                "attributes": [
                    {
                        "key": "telemetry.sdk.version",
                        "value": {
                            "stringValue": "2.3.5-0.27.1"
                        }
                    },
                    {
                        "key": "telemetry.sdk.name",
                        "value": {
                            "stringValue": "deno-opentelemetry"
                        }
                    },
                    {
                        "key": "process.runtime.name",
                        "value": {
                            "stringValue": "deno"
                        }
                    },
                    {
                        "key": "process.runtime.version",
                        "value": {
                            "stringValue": "2.3.5"
                        }
                    },
                    {
                        "key": "telemetry.sdk.language",
                        "value": {
                            "stringValue": "deno-rust"
                        }
                    },
                    {
                        "key": "service.name",
                        "value": {
                            "stringValue": "sepang"
                        }
                    }
                ],
                "droppedAttributesCount": 0
            },
            "instrumentationLibrarySpans": [
                {
                    "spans": [
                        {
                            "traceId": "93645fd5355d13e870e9d331f8874ed0",
                            "spanId": "c54b4a0790f279d4",
                            "parentSpanId": "a2245a046853abbd",
                            "traceState": "",
                            "name": "POST",
                            "kind": "SPAN_KIND_CLIENT",
                            "startTimeUnixNano": 1749106657498131700,
                            "endTimeUnixNano": 1749106657501726200,
                            "attributes": [
                                {
                                    "key": "http.request.method",
                                    "value": {
                                        "stringValue": "POST"
                                    }
                                },
                                {
                                    "key": "url.full",
                                    "value": {
                                        "stringValue": "http://phoenix.racetracks.docker:9455/token/validate"
                                    }
                                },
                                {
                                    "key": "url.scheme",
                                    "value": {
                                        "stringValue": "http"
                                    }
                                },
                                {
                                    "key": "url.path",
                                    "value": {
                                        "stringValue": "/token/validate"
                                    }
                                },
                                {
                                    "key": "url.query",
                                    "value": {
                                        "stringValue": ""
                                    }
                                },
                                {
                                    "key": "http.response.status_code",
                                    "value": {
                                        "stringValue": "200"
                                    }
                                }
                            ],
                            "droppedAttributesCount": 0,
                            "droppedEventsCount": 0,
                            "droppedLinksCount": 0,
                            "status": {
                                "code": 0,
                                "message": ""
                            }
                        }
                    ],
                    "instrumentationLibrary": {
                        "name": "deno",
                        "version": "2.3.5"
                    }
                }
            ]
        },
        {
            "resource": {
                "attributes": [
                    {
                        "key": "telemetry.sdk.version",
                        "value": {
                            "stringValue": "2.3.5-0.27.1"
                        }
                    },
                    {
                        "key": "telemetry.sdk.name",
                        "value": {
                            "stringValue": "deno-opentelemetry"
                        }
                    },
                    {
                        "key": "process.runtime.name",
                        "value": {
                            "stringValue": "deno"
                        }
                    },
                    {
                        "key": "process.runtime.version",
                        "value": {
                            "stringValue": "2.3.5"
                        }
                    },
                    {
                        "key": "telemetry.sdk.language",
                        "value": {
                            "stringValue": "deno-rust"
                        }
                    },
                    {
                        "key": "service.name",
                        "value": {
                            "stringValue": "sepang"
                        }
                    }
                ],
                "droppedAttributesCount": 0
            },
            "instrumentationLibrarySpans": [
                {
                    "spans": [
                        {
                            "traceId": "93645fd5355d13e870e9d331f8874ed0",
                            "spanId": "aa3151782eabc932",
                            "parentSpanId": "a2245a046853abbd",
                            "traceState": "",
                            "name": "POST",
                            "kind": "SPAN_KIND_CLIENT",
                            "startTimeUnixNano": 1749106657550789600,
                            "endTimeUnixNano": 1749106657619380500,
                            "attributes": [
                                {
                                    "key": "http.request.method",
                                    "value": {
                                        "stringValue": "POST"
                                    }
                                },
                                {
                                    "key": "url.full",
                                    "value": {
                                        "stringValue": "https://dynamodb.ap-southeast-2.amazonaws.com/"
                                    }
                                },
                                {
                                    "key": "url.scheme",
                                    "value": {
                                        "stringValue": "https"
                                    }
                                },
                                {
                                    "key": "url.path",
                                    "value": {
                                        "stringValue": "/"
                                    }
                                },
                                {
                                    "key": "url.query",
                                    "value": {
                                        "stringValue": ""
                                    }
                                },
                                {
                                    "key": "http.response.status_code",
                                    "value": {
                                        "intValue": 200
                                    }
                                }
                            ],
                            "droppedAttributesCount": 0,
                            "droppedEventsCount": 0,
                            "droppedLinksCount": 0,
                            "status": {
                                "code": 0,
                                "message": ""
                            }
                        },
                        {
                            "traceId": "93645fd5355d13e870e9d331f8874ed0",
                            "spanId": "a2245a046853abbd",
                            "parentSpanId": "",
                            "traceState": "",
                            "name": "GET",
                            "kind": "SPAN_KIND_SERVER",
                            "startTimeUnixNano": 1749106657497477000,
                            "endTimeUnixNano": 1749106657623805000,
                            "attributes": [
                                {
                                    "key": "http.request.method",
                                    "value": {
                                        "stringValue": "GET"
                                    }
                                },
                                {
                                    "key": "url.full",
                                    "value": {
                                        "stringValue": "http://sepang.racetracks.docker/api/v1/users/PNI09jSo3rby9fb/bookmarks/page/157330"
                                    }
                                },
                                {
                                    "key": "url.scheme",
                                    "value": {
                                        "stringValue": "http"
                                    }
                                },
                                {
                                    "key": "url.path",
                                    "value": {
                                        "stringValue": "/api/v1/users/PNI09jSo3rby9fb/bookmarks/page/157330"
                                    }
                                },
                                {
                                    "key": "url.query",
                                    "value": {
                                        "stringValue": ""
                                    }
                                },
                                {
                                    "key": "http.response.status_code",
                                    "value": {
                                        "stringValue": "404"
                                    }
                                },
                                {
                                    "key": "error.type",
                                    "value": {
                                        "stringValue": "404"
                                    }
                                }
                            ],
                            "droppedAttributesCount": 0,
                            "droppedEventsCount": 0,
                            "droppedLinksCount": 0,
                            "status": {
                                "code": 2,
                                "message": ""
                            }
                        }
                    ],
                    "instrumentationLibrary": {
                        "name": "deno",
                        "version": "2.3.5"
                    }
                }
            ]
        }
    ]
}

Here is the trace data from AWS X-Ray:

{
    "Id": "1-f4c18d3d-71a43b6189af4f3f6f8e5f34",
    "Duration": 0.029,
    "LimitExceeded": false,
    "Segments": [
        {
            "Id": "4411b78663f243a2",
            "Document": {
                "id": "4411b78663f243a2",
                "name": "sepang",
                "start_time": 1749102843.3889108,
                "trace_id": "1-f4c18d3d-71a43b6189af4f3f6f8e5f34",
                "end_time": 1749102843.4182088,
                "fault": true,
                "error": false,
                "throttle": false,
                "http": {
                    "request": {
                        "url": "http://sepang.newsnow.io/api/v1/users/PNIfZ9SY1rgp18t/bookmarks/page/166509",
                        "method": "GET"
                    },
                    "response": {
                        "status": 0,
                        "content_length": 0
                    }
                },
                "aws": {
                    "xray": {
                        "auto_instrumentation": false,
                        "sdk_version": "2.3.5-0.27.1",
                        "sdk": "deno-opentelemetry for deno-rust"
                    }
                },
                "metadata": {
                    "default": {
                        "otel.resource.telemetry.sdk.name": "deno-opentelemetry",
                        "error.type": "404",
                        "otel.resource.process.runtime.version": "2.3.5",
                        "otel.resource.process.runtime.name": "deno",
                        "otel.resource.service.name": "sepang",
                        "otel.resource.telemetry.sdk.language": "deno-rust",
                        "otel.resource.telemetry.sdk.version": "2.3.5-0.27.1"
                    }
                },
                "subsegments": [
                    {
                        "id": "0613dc383e7f3d2a",
                        "name": "POST",
                        "start_time": 1749102843.3892868,
                        "end_time": 1749102843.3940136,
                        "fault": false,
                        "error": false,
                        "throttle": false,
                        "http": {
                            "request": {
                                "url": "https://phoenix.newsnow.io/token/validate",
                                "method": "POST"
                            },
                            "response": {
                                "status": 0,
                                "content_length": 0
                            }
                        },
                        "aws": {
                            "xray": {
                                "auto_instrumentation": false,
                                "sdk_version": "2.3.5-0.27.1",
                                "sdk": "deno-opentelemetry for deno-rust"
                            }
                        },
                        "namespace": "remote"
                    },
                    {
                        "id": "78527afba0bb1c81",
                        "name": "POST",
                        "start_time": 1749102843.3982284,
                        "end_time": 1749102843.4173532,
                        "fault": false,
                        "error": false,
                        "throttle": false,
                        "http": {
                            "request": {
                                "url": "https://dynamodb.ap-southeast-2.amazonaws.com/",
                                "method": "POST"
                            },
                            "response": {
                                "status": 0,
                                "content_length": 0
                            }
                        },
                        "aws": {
                            "xray": {
                                "auto_instrumentation": false,
                                "sdk_version": "2.3.5-0.27.1",
                                "sdk": "deno-opentelemetry for deno-rust"
                            }
                        },
                        "namespace": "remote"
                    }
                ]
            }
        },
        {
            "Id": "0a28f0451c7a5cd6",
            "Document": {
                "id": "0a28f0451c7a5cd6",
                "name": "POST",
                "start_time": 1749102843.3982284,
                "trace_id": "1-f4c18d3d-71a43b6189af4f3f6f8e5f34",
                "end_time": 1749102843.4173532,
                "parent_id": "78527afba0bb1c81",
                "inferred": true,
                "http": {
                    "request": {
                        "url": "https://dynamodb.ap-southeast-2.amazonaws.com/",
                        "method": "POST"
                    },
                    "response": {
                        "status": 0,
                        "content_length": 0
                    }
                }
            }
        },
        {
            "Id": "0b4a1b1e3fc5589f",
            "Document": {
                "id": "0b4a1b1e3fc5589f",
                "name": "POST",
                "start_time": 1749102843.3892868,
                "trace_id": "1-f4c18d3d-71a43b6189af4f3f6f8e5f34",
                "end_time": 1749102843.3940136,
                "parent_id": "0613dc383e7f3d2a",
                "inferred": true,
                "http": {
                    "request": {
                        "url": "https://phoenix.newsnow.io/token/validate",
                        "method": "POST"
                    },
                    "response": {
                        "status": 0,
                        "content_length": 0
                    }
                }
            }
        }
    ]
}

Both return an error in their respective services (fault in AWS, error in Grafana) but I cannot understand why. Ive tried to set span.setStatus({ code: SpanStatusCode.OK }); but it does not change the status code either. This is affecting our ability to put proper alarms in place

Any help would be appreciated, thanks

Metadata

Metadata

Assignees

No one assigned

    Labels

    bugSomething isn't working correctlyotel

    Type

    No type

    Projects

    No projects

    Milestone

    No milestone

    Relationships

    None yet

    Development

    No branches or pull requests

    Issue actions