Skip to content

Commit 5921aa2

Browse files
NadavLeviclaude
NadavLevi
and
claude
authored
perf(chainlib): send response bodies as bytes via fiber.Ctx.Send (#2270)
addHeadersAndSendString called fiber.Ctx.SendString, which copies the string into fasthttp's response bytebufferpool. All callers in the relay path already hold the body as []byte (reply.Data) or as JSON marshal output (also []byte) and were round-tripping through string() at the call site. That string() conversion is a full per-request copy on top of the unavoidable copy into fasthttp's buffer — visible in pprof as 96 GB inuse under bytebufferpool.(*ByteBuffer).WriteString on the eth router. Rename to addHeadersAndSendBytes, take []byte, and use fiber.Ctx.Send. Update every caller (jsonRPC, tendermintRPC, rest) to drop the string() round-trip on success paths where reply.Data is already []byte; error paths keep []byte(s) on the small error strings since those aren't on the hot path. Add focused tests covering body + metadata headers, the empty-body case, and a binary payload round-trip to guard against a regression that reintroduces the string() conversion. Co-authored-by: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
1 parent 3f1baeb commit 5921aa2

5 files changed

Lines changed: 93 additions & 31 deletions

File tree

‎protocol/chainlib/common.go‎

Lines changed: 10 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -212,18 +212,10 @@ func convertToUTXOResponse(msg *rpcclient.JsonrpcMessage) *rpcclient.BTCResponse
212212
}
213213
}
214214

215-
func addHeadersAndSendString(c *fiber.Ctx, metaData []pairingtypes.Metadata, data string) error {
216-
for _, value := range metaData {
217-
c.Set(value.Name, value.Value)
218-
}
219-
220-
return c.SendString(data)
221-
}
222-
223-
// addHeadersAndSendBytes is the []byte counterpart of addHeadersAndSendString.
224-
// Used on the ETH/Cosmos hot path so the upstream reply body flows straight to
225-
// fiber.Ctx.Send without a string(response) detour — which would otherwise
226-
// cancel out the zero-copy passthrough in checkUTXOResponseAndFixReply.
215+
// addHeadersAndSendBytes writes response metadata headers and sends the raw body via
216+
// fiber.Ctx.Send to avoid the []byte → string → []byte round-trip that fiber.Ctx.SendString
217+
// imposes at every call site. Hot-path response bodies (already []byte from the relay
218+
// pipeline) no longer pay an extra string-conversion allocation per request.
227219
func addHeadersAndSendBytes(c *fiber.Ctx, metaData []pairingtypes.Metadata, data []byte) error {
228220
for _, value := range metaData {
229221
c.Set(value.Name, value.Value)
@@ -232,15 +224,18 @@ func addHeadersAndSendBytes(c *fiber.Ctx, metaData []pairingtypes.Metadata, data
232224
return c.Send(data)
233225
}
234226

235-
func convertToJsonError(errorMsg string) string {
227+
// convertToJsonError returns a JSON-encoded `{"error": errorMsg}` body as raw bytes.
228+
// Returning []byte (rather than string) lets every error-path caller hand the
229+
// result straight to addHeadersAndSendBytes / fiber.Ctx.Send with no conversion.
230+
func convertToJsonError(errorMsg string) []byte {
236231
jsonResponse, err := json.Marshal(fiber.Map{
237232
"error": errorMsg,
238233
})
239234
if err != nil {
240-
return `{"error": "Failed to marshal error response to json"}`
235+
return []byte(`{"error": "Failed to marshal error response to json"}`)
241236
}
242237

243-
return string(jsonResponse)
238+
return jsonResponse
244239
}
245240

246241
func addAttributeToError(key, value, errorMessage string) string {

‎protocol/chainlib/common_test.go‎

Lines changed: 70 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@ import (
1515
"github.com/lavanet/lava/v5/protocol/chainlib/chainproxy"
1616
"github.com/lavanet/lava/v5/protocol/chainlib/chainproxy/rpcclient"
1717
"github.com/lavanet/lava/v5/protocol/common"
18+
pairingtypes "github.com/lavanet/lava/v5/x/pairing/types"
1819
spectypes "github.com/lavanet/lava/v5/x/spec/types"
1920
"github.com/stretchr/testify/assert"
2021
"github.com/stretchr/testify/require"
@@ -129,7 +130,7 @@ func TestConvertToJsonError(t *testing.T) {
129130
t.Parallel()
130131

131132
result := convertToJsonError(testCase.errorMsg)
132-
if result != testCase.expected {
133+
if string(result) != testCase.expected {
133134
t.Errorf("Expected result to be %s, but got %s", testCase.expected, result)
134135
}
135136
})
@@ -550,6 +551,74 @@ func TestApplyResponseCompression(t *testing.T) {
550551
}
551552
}
552553

554+
func TestAddHeadersAndSendBytes(t *testing.T) {
555+
t.Run("writes_body_bytes_and_metadata_headers", func(t *testing.T) {
556+
payload := []byte(`{"jsonrpc":"2.0","id":1,"result":"0xdeadbeef"}`)
557+
meta := []pairingtypes.Metadata{
558+
{Name: "X-Test-Trace-Id", Value: "abc-123"},
559+
{Name: "Content-Type", Value: "application/json"},
560+
}
561+
562+
app := fiber.New()
563+
app.Get("/", func(c *fiber.Ctx) error {
564+
return addHeadersAndSendBytes(c, meta, payload)
565+
})
566+
567+
req := httptest.NewRequest(fiber.MethodGet, "/", nil)
568+
resp, err := app.Test(req)
569+
require.NoError(t, err)
570+
defer resp.Body.Close()
571+
572+
body, err := io.ReadAll(resp.Body)
573+
require.NoError(t, err)
574+
require.Equal(t, fiber.StatusOK, resp.StatusCode)
575+
require.Equal(t, payload, body, "response body must match the []byte payload exactly")
576+
require.Equal(t, "abc-123", resp.Header.Get("X-Test-Trace-Id"))
577+
require.Equal(t, "application/json", resp.Header.Get("Content-Type"))
578+
})
579+
580+
t.Run("empty_body_is_allowed", func(t *testing.T) {
581+
app := fiber.New()
582+
app.Get("/", func(c *fiber.Ctx) error {
583+
return addHeadersAndSendBytes(c, nil, nil)
584+
})
585+
586+
req := httptest.NewRequest(fiber.MethodGet, "/", nil)
587+
resp, err := app.Test(req)
588+
require.NoError(t, err)
589+
defer resp.Body.Close()
590+
591+
body, err := io.ReadAll(resp.Body)
592+
require.NoError(t, err)
593+
require.Equal(t, fiber.StatusOK, resp.StatusCode)
594+
require.Empty(t, body)
595+
})
596+
597+
t.Run("preserves_binary_payload_bytes", func(t *testing.T) {
598+
// The whole point of the []byte signature is that non-UTF8 / binary content
599+
// isn't mangled by a []byte → string → []byte round-trip. Verify with a
600+
// payload that includes every byte value, including embedded NULs.
601+
payload := make([]byte, 256)
602+
for i := range payload {
603+
payload[i] = byte(i)
604+
}
605+
606+
app := fiber.New()
607+
app.Get("/", func(c *fiber.Ctx) error {
608+
return addHeadersAndSendBytes(c, nil, payload)
609+
})
610+
611+
req := httptest.NewRequest(fiber.MethodGet, "/", nil)
612+
resp, err := app.Test(req)
613+
require.NoError(t, err)
614+
defer resp.Body.Close()
615+
616+
body, err := io.ReadAll(resp.Body)
617+
require.NoError(t, err)
618+
require.Equal(t, payload, body, "binary payload must round-trip byte-for-byte")
619+
})
620+
}
621+
553622
func TestCompareRequestedBlockInBatch(t *testing.T) {
554623
playbook := []struct {
555624
latest int64

‎protocol/chainlib/jsonRPC.go‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -473,13 +473,13 @@ func (apil *JsonRPCChainListener) Serve(ctx context.Context, cmdFlags common.Con
473473
if common.APINotSupportedError.Is(err) {
474474
// Convert error to JSON string and add headers
475475
errorResponse, _ := json.Marshal(common.JsonRpcMethodNotFoundError)
476-
return addHeadersAndSendString(fiberCtx, reply.GetMetadata(), string(errorResponse))
476+
return addHeadersAndSendBytes(fiberCtx, reply.GetMetadata(), errorResponse)
477477
}
478478

479479
if _, ok := err.(*json.SyntaxError); ok {
480480
// Convert error to JSON string and add headers
481481
errorResponse, _ := json.Marshal(common.JsonRpcParseError)
482-
return addHeadersAndSendString(fiberCtx, reply.GetMetadata(), string(errorResponse))
482+
return addHeadersAndSendBytes(fiberCtx, reply.GetMetadata(), errorResponse)
483483
}
484484

485485
// Get unique GUID response
@@ -498,7 +498,7 @@ func (apil *JsonRPCChainListener) Serve(ctx context.Context, cmdFlags common.Con
498498
// Construct json response
499499
response := convertToJsonError(errMasking)
500500
// Return error json response
501-
return addHeadersAndSendString(fiberCtx, reply.GetMetadata(), response)
501+
return addHeadersAndSendBytes(fiberCtx, reply.GetMetadata(), response)
502502
}
503503

504504
response := checkUTXOResponseAndFixReply(chainID, reply.Data)

‎protocol/chainlib/rest.go‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -322,15 +322,15 @@ func (apil *RestChainListener) Serve(ctx context.Context, cmdFlags common.Consum
322322
response := convertToJsonError(errMasking)
323323

324324
// Return error json response
325-
return addHeadersAndSendString(fiberCtx, reply.GetMetadata(), response)
325+
return addHeadersAndSendBytes(fiberCtx, reply.GetMetadata(), response)
326326
}
327327
// Log request and response
328328
apil.logger.LogRequestAndResponse("http in/out", false, http.MethodPost, path, requestBody, string(reply.Data), msgSeed, time.Since(startTime), nil)
329329
if relayResult.GetStatusCode() != 0 {
330330
fiberCtx.Status(relayResult.StatusCode)
331331
}
332332
// Return json response and add metric for after provider processing
333-
err = addHeadersAndSendString(fiberCtx, reply.GetMetadata(), string(reply.Data))
333+
err = addHeadersAndSendBytes(fiberCtx, reply.GetMetadata(), reply.Data)
334334
return err
335335
}
336336

@@ -397,7 +397,7 @@ func (apil *RestChainListener) Serve(ctx context.Context, cmdFlags common.Consum
397397
response := convertToJsonError(errMasking)
398398

399399
// Return error json response
400-
return addHeadersAndSendString(fiberCtx, reply.GetMetadata(), response)
400+
return addHeadersAndSendBytes(fiberCtx, reply.GetMetadata(), response)
401401
}
402402
if relayResult.GetStatusCode() != 0 {
403403
fiberCtx.Status(relayResult.StatusCode)
@@ -406,7 +406,7 @@ func (apil *RestChainListener) Serve(ctx context.Context, cmdFlags common.Consum
406406
apil.logger.LogRequestAndResponse("http in/out", false, http.MethodGet, path, "", string(reply.Data), msgSeed, time.Since(startTime), nil)
407407

408408
// Return json response
409-
err = addHeadersAndSendString(fiberCtx, reply.GetMetadata(), string(reply.Data))
409+
err = addHeadersAndSendBytes(fiberCtx, reply.GetMetadata(), reply.Data)
410410
return err
411411
}
412412

‎protocol/chainlib/tendermintRPC.go‎

Lines changed: 6 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -484,7 +484,7 @@ func (apil *TendermintRpcChainListener) Serve(ctx context.Context, cmdFlags comm
484484
if common.APINotSupportedError.Is(err) {
485485
// Convert error to JSON string and add headers
486486
errorResponse, _ := json.Marshal(common.JsonRpcMethodNotFoundError)
487-
return addHeadersAndSendString(fiberCtx, reply.GetMetadata(), string(errorResponse))
487+
return addHeadersAndSendBytes(fiberCtx, reply.GetMetadata(), errorResponse)
488488
}
489489

490490
// Get unique GUID response
@@ -503,16 +503,15 @@ func (apil *TendermintRpcChainListener) Serve(ctx context.Context, cmdFlags comm
503503
// Construct json response
504504
response := rpcInterfaceMessages.ConvertToTendermintError(errMasking, fiberCtx.Body())
505505
// Return error json response
506-
return addHeadersAndSendString(fiberCtx, reply.GetMetadata(), response)
506+
return addHeadersAndSendBytes(fiberCtx, reply.GetMetadata(), []byte(response))
507507
}
508508
// Log request and response
509509
apil.logger.LogRequestAndResponse("tendermint http in/out", false, "POST", fiberCtx.Request().URI().String(), msg, string(reply.Data), msgSeed, time.Since(startTime), nil)
510510
if relayResult.GetStatusCode() != 0 {
511511
fiberCtx.Status(relayResult.StatusCode)
512512
}
513-
response := string(reply.Data)
514513
// Return json response
515-
err = addHeadersAndSendString(fiberCtx, reply.GetMetadata(), response)
514+
err = addHeadersAndSendBytes(fiberCtx, reply.GetMetadata(), reply.Data)
516515
return err
517516
}
518517

@@ -568,16 +567,15 @@ func (apil *TendermintRpcChainListener) Serve(ctx context.Context, cmdFlags comm
568567
response := convertToJsonError(errMasking)
569568

570569
// Return error json response
571-
return addHeadersAndSendString(fiberCtx, reply.GetMetadata(), response)
570+
return addHeadersAndSendBytes(fiberCtx, reply.GetMetadata(), response)
572571
}
573-
response := string(reply.Data)
574572
// Log request and response
575-
apil.logger.LogRequestAndResponse("tendermint http in/out", false, "GET", fiberCtx.Request().URI().String(), "", response, msgSeed, time.Since(startTime), nil)
573+
apil.logger.LogRequestAndResponse("tendermint http in/out", false, "GET", fiberCtx.Request().URI().String(), "", string(reply.Data), msgSeed, time.Since(startTime), nil)
576574
if relayResult.GetStatusCode() != 0 {
577575
fiberCtx.Status(relayResult.StatusCode)
578576
}
579577
// Return json response
580-
err = addHeadersAndSendString(fiberCtx, reply.GetMetadata(), response)
578+
err = addHeadersAndSendBytes(fiberCtx, reply.GetMetadata(), reply.Data)
581579
return err
582580
}
583581

0 commit comments

Comments
 (0)