Skip to content

Commit 3e6aa16

Browse files
SantiagoDePoloniaSantiagoDePolonia
authored
refactor: decoupling and speeding up (#163)
* perf(core): optimize JSON hot path preservation and extraction * perf(responsecache): reuse snapshot bodies and bound cache writes * perf(server): reduce request snapshot overhead * test(perf): tighten hot path guard thresholds * refactor(core): complete unknown field migration * refactor(test): move selector baselines out of runtime * fix(hotpath): honor duplicate top-level selector keys * test(perf): log measured guard results * perf(hotpath): restore gjson selector peeking * docs(hotpath): document gjson selector tradeoff * fix(core): harden json field handling and cache guards * test(perf): tighten hot path guard thresholds * fix(review): address follow-up comments
1 parent 76d4993 commit 3e6aa16

37 files changed

Lines changed: 1880 additions & 318 deletions

‎.gitignore‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -28,3 +28,6 @@
2828

2929
# Claude Code local settings (user-specific)
3030
/.claude/settings.local.json
31+
32+
# Local git worktrees
33+
/.worktrees/

‎go.mod‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@ require (
1717
github.com/testcontainers/testcontainers-go v0.41.0
1818
github.com/testcontainers/testcontainers-go/modules/mongodb v0.41.0
1919
github.com/testcontainers/testcontainers-go/modules/postgres v0.41.0
20+
github.com/tidwall/gjson v1.18.0
2021
go.mongodb.org/mongo-driver/v2 v2.5.0
2122
golang.org/x/term v0.41.0
2223
gopkg.in/yaml.v3 v3.0.1
@@ -93,6 +94,8 @@ require (
9394
github.com/sv-tools/openapi v0.4.0 // indirect
9495
github.com/swaggo/files/v2 v2.0.2 // indirect
9596
github.com/swaggo/swag/v2 v2.0.0-rc5 // indirect
97+
github.com/tidwall/match v1.1.1 // indirect
98+
github.com/tidwall/pretty v1.2.0 // indirect
9699
github.com/tklauser/go-sysconf v0.3.16 // indirect
97100
github.com/tklauser/numcpus v0.11.0 // indirect
98101
github.com/urfave/cli/v2 v2.27.5 // indirect

‎go.sum‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -209,6 +209,12 @@ github.com/testcontainers/testcontainers-go/modules/mongodb v0.41.0 h1:z5sroe+jX
209209
github.com/testcontainers/testcontainers-go/modules/mongodb v0.41.0/go.mod h1:pb+JZ21ixA1TlyUkhJclS/hCuZCnAkgnh+iB+pTHnSk=
210210
github.com/testcontainers/testcontainers-go/modules/postgres v0.41.0 h1:AOtFXssrDlLm84A2sTTR/AhvJiYbrIuCO59d+Ro9Tb0=
211211
github.com/testcontainers/testcontainers-go/modules/postgres v0.41.0/go.mod h1:k2a09UKhgSp6vNpliIY0QSgm4Hi7GXVTzWvWgUemu/8=
212+
github.com/tidwall/gjson v1.18.0 h1:FIDeeyB800efLX89e5a8Y0BNH+LOngJyGrIWxG2FKQY=
213+
github.com/tidwall/gjson v1.18.0/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk=
214+
github.com/tidwall/match v1.1.1 h1:+Ho715JplO36QYgwN9PGYNhgZvoUSc9X2c80KVTi+GA=
215+
github.com/tidwall/match v1.1.1/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM=
216+
github.com/tidwall/pretty v1.2.0 h1:RWIZEg2iJ8/g6fDDYzMpobmaoGh5OLl4AXtGUGPcqCs=
217+
github.com/tidwall/pretty v1.2.0/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU=
212218
github.com/tklauser/go-sysconf v0.3.16 h1:frioLaCQSsF5Cy1jgRBrzr6t502KIIwQ0MArYICU0nA=
213219
github.com/tklauser/go-sysconf v0.3.16/go.mod h1:/qNL9xxDhc7tx3HSRsLWNnuzbVfh3e7gh/BmM179nYI=
214220
github.com/tklauser/numcpus v0.11.0 h1:nSTwhKH5e1dMNsCdVBukSZrURJRoHbSEQjdEbY+9RXw=

‎internal/core/batch.go‎

Lines changed: 11 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -13,12 +13,12 @@ import "encoding/json"
1313
// Gateway extension:
1414
// - requests (inline payloads for providers that support native inline batch bodies)
1515
type BatchRequest struct {
16-
InputFileID string `json:"input_file_id,omitempty"`
17-
Endpoint string `json:"endpoint,omitempty"`
18-
CompletionWindow string `json:"completion_window,omitempty"`
19-
Metadata map[string]string `json:"metadata,omitempty"`
20-
Requests []BatchRequestItem `json:"requests,omitempty"`
21-
ExtraFields map[string]json.RawMessage `json:"-" swaggerignore:"true"`
16+
InputFileID string `json:"input_file_id,omitempty"`
17+
Endpoint string `json:"endpoint,omitempty"`
18+
CompletionWindow string `json:"completion_window,omitempty"`
19+
Metadata map[string]string `json:"metadata,omitempty"`
20+
Requests []BatchRequestItem `json:"requests,omitempty"`
21+
ExtraFields UnknownJSONFields `json:"-" swaggerignore:"true"`
2222
}
2323

2424
const (
@@ -60,11 +60,11 @@ func (req *BatchRouteInfo) ensureParsedLimit() error {
6060

6161
// BatchRequestItem represents one sub-request in an inline batch.
6262
type BatchRequestItem struct {
63-
CustomID string `json:"custom_id,omitempty"`
64-
Method string `json:"method,omitempty"`
65-
URL string `json:"url"`
66-
Body json.RawMessage `json:"body" swaggertype:"object"`
67-
ExtraFields map[string]json.RawMessage `json:"-" swaggerignore:"true"`
63+
CustomID string `json:"custom_id,omitempty"`
64+
Method string `json:"method,omitempty"`
65+
URL string `json:"url"`
66+
Body json.RawMessage `json:"body" swaggertype:"object"`
67+
ExtraFields UnknownJSONFields `json:"-" swaggerignore:"true"`
6868
}
6969

7070
// BatchResponse uses OpenAI-compatible batch fields and includes provider mapping plus optional cached results.

‎internal/core/batch_json_test.go‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -25,11 +25,11 @@ func TestBatchRequestJSON_PreservesUnknownFields(t *testing.T) {
2525
t.Fatalf("json.Unmarshal() error = %v", err)
2626
}
2727

28-
if req.ExtraFields["x_top"] == nil {
28+
if req.ExtraFields.Lookup("x_top") == nil {
2929
t.Fatalf("x_top missing from ExtraFields: %+v", req.ExtraFields)
3030
}
3131
var topExtra map[string]any
32-
if err := json.Unmarshal(req.ExtraFields["x_top"], &topExtra); err != nil {
32+
if err := json.Unmarshal(req.ExtraFields.Lookup("x_top"), &topExtra); err != nil {
3333
t.Fatalf("failed to decode x_top: %v", err)
3434
}
3535
if topExtra["trace"] != "batch-1" || topExtra["mode"] != "strict" {
@@ -38,11 +38,11 @@ func TestBatchRequestJSON_PreservesUnknownFields(t *testing.T) {
3838
if len(req.Requests) != 1 {
3939
t.Fatalf("len(Requests) = %d, want 1", len(req.Requests))
4040
}
41-
if req.Requests[0].ExtraFields["x_item_flag"] == nil {
41+
if req.Requests[0].ExtraFields.Lookup("x_item_flag") == nil {
4242
t.Fatalf("x_item_flag missing from Requests[0].ExtraFields: %+v", req.Requests[0].ExtraFields)
4343
}
4444
var itemExtra map[string]any
45-
if err := json.Unmarshal(req.Requests[0].ExtraFields["x_item_flag"], &itemExtra); err != nil {
45+
if err := json.Unmarshal(req.Requests[0].ExtraFields.Lookup("x_item_flag"), &itemExtra); err != nil {
4646
t.Fatalf("failed to decode x_item_flag: %v", err)
4747
}
4848
if itemExtra["enabled"] != true || itemExtra["label"] != "batch-item" {

‎internal/core/batch_preparation.go‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -310,7 +310,7 @@ func cloneBatchRequest(req *BatchRequest) *BatchRequest {
310310
Endpoint: req.Endpoint,
311311
CompletionWindow: req.CompletionWindow,
312312
Metadata: cloneBatchStringMap(req.Metadata),
313-
ExtraFields: CloneRawJSONMap(req.ExtraFields),
313+
ExtraFields: CloneUnknownJSONFields(req.ExtraFields),
314314
}
315315
if req.Requests != nil {
316316
cloned.Requests = make([]BatchRequestItem, len(req.Requests))
@@ -327,7 +327,7 @@ func cloneBatchRequestItem(item BatchRequestItem) BatchRequestItem {
327327
Method: item.Method,
328328
URL: item.URL,
329329
Body: CloneRawJSON(item.Body),
330-
ExtraFields: CloneRawJSONMap(item.ExtraFields),
330+
ExtraFields: CloneUnknownJSONFields(item.ExtraFields),
331331
}
332332
}
333333

‎internal/core/batch_preparation_test.go‎

Lines changed: 14 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -19,14 +19,14 @@ func TestCloneBatchRequestDeepCopiesNestedFields(t *testing.T) {
1919
Method: "POST",
2020
URL: "/v1/chat/completions",
2121
Body: json.RawMessage(`{"model":"smart","messages":[{"role":"user","content":"hi"}]}`),
22-
ExtraFields: map[string]json.RawMessage{
22+
ExtraFields: UnknownJSONFieldsFromMap(map[string]json.RawMessage{
2323
"x_item": json.RawMessage(`{"trace":true}`),
24-
},
24+
}),
2525
},
2626
},
27-
ExtraFields: map[string]json.RawMessage{
27+
ExtraFields: UnknownJSONFieldsFromMap(map[string]json.RawMessage{
2828
"x_top": json.RawMessage(`{"debug":true}`),
29-
},
29+
}),
3030
}
3131

3232
cloned := cloneBatchRequest(original)
@@ -37,9 +37,15 @@ func TestCloneBatchRequestDeepCopiesNestedFields(t *testing.T) {
3737
cloned.Metadata["provider"] = "anthropic"
3838
cloned.Requests[0].CustomID = "chat-2"
3939
cloned.Requests[0].Body[10] = 'X'
40-
itemExtra := cloned.Requests[0].ExtraFields["x_item"]
40+
itemExtra := cloned.Requests[0].ExtraFields.Lookup("x_item")
41+
if len(itemExtra) <= 9 {
42+
t.Fatalf("cloned item extra too short: %q", itemExtra)
43+
}
4144
itemExtra[9] = 'f'
42-
topExtra := cloned.ExtraFields["x_top"]
45+
topExtra := cloned.ExtraFields.Lookup("x_top")
46+
if len(topExtra) <= 9 {
47+
t.Fatalf("cloned top extra too short: %q", topExtra)
48+
}
4349
topExtra[9] = 'f'
4450

4551
if got := original.Metadata["provider"]; got != "openai" {
@@ -51,10 +57,10 @@ func TestCloneBatchRequestDeepCopiesNestedFields(t *testing.T) {
5157
if got := string(original.Requests[0].Body); got != `{"model":"smart","messages":[{"role":"user","content":"hi"}]}` {
5258
t.Fatalf("original body mutated to %s", got)
5359
}
54-
if got := string(original.Requests[0].ExtraFields["x_item"]); got != `{"trace":true}` {
60+
if got := string(original.Requests[0].ExtraFields.Lookup("x_item")); got != `{"trace":true}` {
5561
t.Fatalf("original item extra mutated to %s", got)
5662
}
57-
if got := string(original.ExtraFields["x_top"]); got != `{"debug":true}` {
63+
if got := string(original.ExtraFields.Lookup("x_top")); got != `{"debug":true}` {
5864
t.Fatalf("original top extra mutated to %s", got)
5965
}
6066
}

‎internal/core/chat_content.go‎

Lines changed: 18 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -9,26 +9,26 @@ import (
99

1010
// ContentPart represents a single OpenAI-compatible multimodal chat content part.
1111
type ContentPart struct {
12-
Type string `json:"type"`
13-
Text string `json:"text,omitempty"`
14-
ImageURL *ImageURLContent `json:"image_url,omitempty"`
15-
InputAudio *InputAudioContent `json:"input_audio,omitempty"`
16-
ExtraFields map[string]json.RawMessage `json:"-" swaggerignore:"true"`
12+
Type string `json:"type"`
13+
Text string `json:"text,omitempty"`
14+
ImageURL *ImageURLContent `json:"image_url,omitempty"`
15+
InputAudio *InputAudioContent `json:"input_audio,omitempty"`
16+
ExtraFields UnknownJSONFields `json:"-" swaggerignore:"true"`
1717
}
1818

1919
// ImageURLContent contains an image reference for image_url parts.
2020
type ImageURLContent struct {
21-
URL string `json:"url"`
22-
Detail string `json:"detail,omitempty"`
23-
MediaType string `json:"media_type,omitempty"`
24-
ExtraFields map[string]json.RawMessage `json:"-" swaggerignore:"true"`
21+
URL string `json:"url"`
22+
Detail string `json:"detail,omitempty"`
23+
MediaType string `json:"media_type,omitempty"`
24+
ExtraFields UnknownJSONFields `json:"-" swaggerignore:"true"`
2525
}
2626

2727
// InputAudioContent contains inline audio payload metadata.
2828
type InputAudioContent struct {
29-
Data string `json:"data"`
30-
Format string `json:"format"`
31-
ExtraFields map[string]json.RawMessage `json:"-" swaggerignore:"true"`
29+
Data string `json:"data"`
30+
Format string `json:"format"`
31+
ExtraFields UnknownJSONFields `json:"-" swaggerignore:"true"`
3232
}
3333

3434
func (p *ContentPart) UnmarshalJSON(data []byte) error {
@@ -97,7 +97,7 @@ func (c *ImageURLContent) UnmarshalJSON(data []byte) error {
9797
c.URL = url
9898
c.Detail = ""
9999
c.MediaType = ""
100-
c.ExtraFields = nil
100+
c.ExtraFields = UnknownJSONFields{}
101101
return nil
102102
}
103103

@@ -412,7 +412,7 @@ func normalizeTypedContentPart(part ContentPart) (ContentPart, error) {
412412
return ContentPart{
413413
Type: "text",
414414
Text: part.Text,
415-
ExtraFields: CloneRawJSONMap(part.ExtraFields),
415+
ExtraFields: CloneUnknownJSONFields(part.ExtraFields),
416416
}, nil
417417
case "image_url", "input_image":
418418
if part.ImageURL == nil || part.ImageURL.URL == "" {
@@ -424,9 +424,9 @@ func normalizeTypedContentPart(part ContentPart) (ContentPart, error) {
424424
URL: part.ImageURL.URL,
425425
Detail: part.ImageURL.Detail,
426426
MediaType: part.ImageURL.MediaType,
427-
ExtraFields: CloneRawJSONMap(part.ImageURL.ExtraFields),
427+
ExtraFields: CloneUnknownJSONFields(part.ImageURL.ExtraFields),
428428
},
429-
ExtraFields: CloneRawJSONMap(part.ExtraFields),
429+
ExtraFields: CloneUnknownJSONFields(part.ExtraFields),
430430
}, nil
431431
case "input_audio":
432432
if part.InputAudio == nil || part.InputAudio.Data == "" || part.InputAudio.Format == "" {
@@ -437,9 +437,9 @@ func normalizeTypedContentPart(part ContentPart) (ContentPart, error) {
437437
InputAudio: &InputAudioContent{
438438
Data: part.InputAudio.Data,
439439
Format: part.InputAudio.Format,
440-
ExtraFields: CloneRawJSONMap(part.InputAudio.ExtraFields),
440+
ExtraFields: CloneUnknownJSONFields(part.InputAudio.ExtraFields),
441441
},
442-
ExtraFields: CloneRawJSONMap(part.ExtraFields),
442+
ExtraFields: CloneUnknownJSONFields(part.ExtraFields),
443443
}, nil
444444
default:
445445
return ContentPart{}, fmt.Errorf("unsupported content part type %q", part.Type)

‎internal/core/chat_json_test.go‎

Lines changed: 17 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,15 @@ import (
55
"testing"
66
)
77

8+
func lookupUnknownField(t *testing.T, fields UnknownJSONFields, key string) json.RawMessage {
9+
t.Helper()
10+
raw := fields.Lookup(key)
11+
if raw == nil {
12+
t.Fatalf("unknown field %q missing", key)
13+
}
14+
return raw
15+
}
16+
817
func TestChatRequestJSON_RoundTripPreservesUnknownFields(t *testing.T) {
918
body := []byte(`{
1019
"model":"gpt-4o-mini",
@@ -63,11 +72,12 @@ func TestChatRequestJSON_RoundTripPreservesUnknownFields(t *testing.T) {
6372
if req.Model != "gpt-4o-mini" {
6473
t.Fatalf("Model = %q, want gpt-4o-mini", req.Model)
6574
}
66-
if req.ExtraFields["x_trace"] == nil || string(req.ExtraFields["x_trace"]) != string(wantExtra["x_trace"]) {
67-
t.Fatalf("ExtraFields[x_trace] = %s, want %s", req.ExtraFields["x_trace"], wantExtra["x_trace"])
75+
traceField := lookupUnknownField(t, req.ExtraFields, "x_trace")
76+
if string(traceField) != string(wantExtra.Lookup("x_trace")) {
77+
t.Fatalf("ExtraFields[x_trace] = %s, want %s", traceField, wantExtra.Lookup("x_trace"))
6878
}
6979
var topTrace map[string]any
70-
if err := json.Unmarshal(req.ExtraFields["x_trace"], &topTrace); err != nil {
80+
if err := json.Unmarshal(traceField, &topTrace); err != nil {
7181
t.Fatalf("failed to unmarshal x_trace: %v", err)
7282
}
7383
if topTrace["id"] != "trace-1" {
@@ -77,7 +87,7 @@ func TestChatRequestJSON_RoundTripPreservesUnknownFields(t *testing.T) {
7787
t.Fatalf("len(Messages) = %d, want 1", len(req.Messages))
7888
}
7989
var messageMeta map[string]any
80-
if err := json.Unmarshal(req.Messages[0].ExtraFields["x_message_meta"], &messageMeta); err != nil {
90+
if err := json.Unmarshal(lookupUnknownField(t, req.Messages[0].ExtraFields, "x_message_meta"), &messageMeta); err != nil {
8191
t.Fatalf("failed to unmarshal x_message_meta: %v", err)
8292
}
8393
if messageMeta["id"] != "msg-1" {
@@ -86,11 +96,11 @@ func TestChatRequestJSON_RoundTripPreservesUnknownFields(t *testing.T) {
8696
if len(req.Messages[0].ToolCalls) != 1 {
8797
t.Fatalf("len(ToolCalls) = %d, want 1", len(req.Messages[0].ToolCalls))
8898
}
89-
if string(req.Messages[0].ToolCalls[0].ExtraFields["x_tool_call"]) != "true" {
90-
t.Fatalf("x_tool_call = %s, want true", req.Messages[0].ToolCalls[0].ExtraFields["x_tool_call"])
99+
if got := lookupUnknownField(t, req.Messages[0].ToolCalls[0].ExtraFields, "x_tool_call"); string(got) != "true" {
100+
t.Fatalf("x_tool_call = %s, want true", got)
91101
}
92102
var functionMeta map[string]any
93-
if err := json.Unmarshal(req.Messages[0].ToolCalls[0].Function.ExtraFields["x_function_meta"], &functionMeta); err != nil {
103+
if err := json.Unmarshal(lookupUnknownField(t, req.Messages[0].ToolCalls[0].Function.ExtraFields, "x_function_meta"), &functionMeta); err != nil {
94104
t.Fatalf("failed to unmarshal x_function_meta: %v", err)
95105
}
96106
if functionMeta["strict"] != true {

‎internal/core/embeddings_json_test.go‎

Lines changed: 6 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -48,11 +48,13 @@ func TestEmbeddingRequestJSON_RoundTripPreservesUnknownFields(t *testing.T) {
4848
if req.Dimensions == nil || *req.Dimensions != 256 {
4949
t.Fatalf("Dimensions = %#v, want 256", req.Dimensions)
5050
}
51-
if string(req.ExtraFields["x_trace"]) != string(wantExtra["x_trace"]) {
52-
t.Fatalf("ExtraFields[x_trace] = %s, want %s", req.ExtraFields["x_trace"], wantExtra["x_trace"])
51+
traceField := lookupUnknownField(t, req.ExtraFields, "x_trace")
52+
if string(traceField) != string(wantExtra.Lookup("x_trace")) {
53+
t.Fatalf("ExtraFields[x_trace] = %s, want %s", traceField, wantExtra.Lookup("x_trace"))
5354
}
54-
if string(req.ExtraFields["x_mode"]) != string(wantExtra["x_mode"]) {
55-
t.Fatalf("ExtraFields[x_mode] = %s, want %s", req.ExtraFields["x_mode"], wantExtra["x_mode"])
55+
modeField := lookupUnknownField(t, req.ExtraFields, "x_mode")
56+
if string(modeField) != string(wantExtra.Lookup("x_mode")) {
57+
t.Fatalf("ExtraFields[x_mode] = %s, want %s", modeField, wantExtra.Lookup("x_mode"))
5658
}
5759

5860
roundTrip, err := json.Marshal(req)

0 commit comments

Comments
 (0)