Conversation
There was a problem hiding this comment.
Code Review
This pull request introduces HTTP request body decompression support for gzip, deflate, and zstd encodings within the llm/httpclient package. It also updates several frontend dependencies and adds the klauspost/compress library. Feedback highlights critical issues, including the accidental removal of existing tests in utils_test.go and a potential memory exhaustion vulnerability (Zip Bomb) due to the lack of size limits on decompression readers. Furthermore, the deflate implementation should be switched to zlib for RFC compliance, a suspicious uuid version in the lockfile requires verification, and compression library versions should be synchronized across modules.
| func TestReadHTTPRequest_NoContentEncoding(t *testing.T) { | ||
| body := []byte(`{"model":"gpt-4","messages":[{"role":"user","content":"hello"}]}`) | ||
| req := httptest.NewRequest(http.MethodPost, "/v1/chat/completions", bytes.NewReader(body)) | ||
| req.Header.Set("Content-Type", "application/json") | ||
|
|
||
| got, err := ReadHTTPRequest(req) | ||
| require.NoError(t, err) | ||
| assert.Equal(t, body, got.Body) | ||
| assert.Equal(t, "", got.Headers.Get("Content-Encoding")) | ||
| } | ||
|
|
||
| func TestReadHTTPRequest_IdentityEncoding(t *testing.T) { | ||
| body := []byte(`{"model":"gpt-4","messages":[{"role":"user","content":"hello"}]}`) | ||
| req := httptest.NewRequest(http.MethodPost, "/v1/chat/completions", bytes.NewReader(body)) | ||
| req.Header.Set("Content-Type", "application/json") | ||
| req.Header.Set("Content-Encoding", "identity") | ||
|
|
||
| got, err := ReadHTTPRequest(req) | ||
| require.NoError(t, err) | ||
| assert.Equal(t, body, got.Body) | ||
| assert.Equal(t, "identity", got.Headers.Get("Content-Encoding")) | ||
| } | ||
|
|
||
| func TestReadHTTPRequest_GzipEncoding(t *testing.T) { | ||
| originalBody := []byte(`{"model":"gpt-4","messages":[{"role":"user","content":"hello"}]}`) | ||
|
|
||
| var buf bytes.Buffer | ||
| writer := gzip.NewWriter(&buf) | ||
| _, err := writer.Write(originalBody) | ||
| require.NoError(t, err) | ||
| writer.Close() | ||
|
|
||
| req := httptest.NewRequest(http.MethodPost, "/v1/chat/completions", bytes.NewReader(buf.Bytes())) | ||
| req.Header.Set("Content-Type", "application/json") | ||
| req.Header.Set("Content-Encoding", "gzip") | ||
|
|
||
| got, err := ReadHTTPRequest(req) | ||
| require.NoError(t, err) | ||
| assert.Equal(t, originalBody, got.Body) | ||
| assert.Equal(t, "", got.Headers.Get("Content-Encoding")) | ||
| assert.Equal(t, "", got.Headers.Get("Content-Length")) | ||
| } | ||
|
|
||
| func TestReadHTTPRequest_GzipEncodingXGzip(t *testing.T) { | ||
| originalBody := []byte(`{"model":"gpt-4"}`) | ||
|
|
||
| var buf bytes.Buffer | ||
| writer := gzip.NewWriter(&buf) | ||
| _, err := writer.Write(originalBody) | ||
| require.NoError(t, err) | ||
| writer.Close() | ||
|
|
||
| req := httptest.NewRequest(http.MethodPost, "/v1/chat/completions", bytes.NewReader(buf.Bytes())) | ||
| req.Header.Set("Content-Type", "application/json") | ||
| req.Header.Set("Content-Encoding", "x-gzip") | ||
|
|
||
| got, err := ReadHTTPRequest(req) | ||
| require.NoError(t, err) | ||
| assert.Equal(t, originalBody, got.Body) | ||
| assert.Equal(t, "", got.Headers.Get("Content-Encoding")) | ||
| } | ||
|
|
||
| func TestReadHTTPRequest_DeflateEncoding(t *testing.T) { | ||
| originalBody := []byte(`{"model":"gpt-4","messages":[{"role":"user","content":"hello"}]}`) | ||
|
|
||
| var buf bytes.Buffer | ||
| writer, err := flate.NewWriter(&buf, flate.DefaultCompression) | ||
| require.NoError(t, err) | ||
| _, err = writer.Write(originalBody) | ||
| require.NoError(t, err) | ||
| writer.Close() | ||
|
|
||
| req := httptest.NewRequest(http.MethodPost, "/v1/chat/completions", bytes.NewReader(buf.Bytes())) | ||
| req.Header.Set("Content-Type", "application/json") | ||
| req.Header.Set("Content-Encoding", "deflate") | ||
|
|
||
| got, err := ReadHTTPRequest(req) | ||
| require.NoError(t, err) | ||
| assert.Equal(t, originalBody, got.Body) | ||
| assert.Equal(t, "", got.Headers.Get("Content-Encoding")) | ||
| assert.Equal(t, "", got.Headers.Get("Content-Length")) | ||
| } | ||
|
|
||
| func TestMergeHTTPHeaders(t *testing.T) { | ||
| // Register headers for testing append behavior | ||
| RegisterMergeWithAppendHeaders("User-Agent", "Accept") | ||
| func TestReadHTTPRequest_ZstdEncoding(t *testing.T) { | ||
| originalBody := []byte(`{"model":"gpt-4","messages":[{"role":"user","content":"hello"}]}`) | ||
|
|
||
| encoder, err := zstd.NewWriter(nil) | ||
| require.NoError(t, err) | ||
| compressedBody := encoder.EncodeAll(originalBody, nil) | ||
| encoder.Close() | ||
|
|
||
| req := httptest.NewRequest(http.MethodPost, "/v1/chat/completions", bytes.NewReader(compressedBody)) | ||
| req.Header.Set("Content-Type", "application/json") | ||
| req.Header.Set("Content-Encoding", "zstd") | ||
|
|
||
| got, err := ReadHTTPRequest(req) | ||
| require.NoError(t, err) | ||
| assert.Equal(t, originalBody, got.Body) | ||
| assert.Equal(t, "", got.Headers.Get("Content-Encoding")) | ||
| assert.Equal(t, "", got.Headers.Get("Content-Length")) | ||
| } | ||
|
|
||
| func TestReadHTTPRequest_EncodingCaseInsensitive(t *testing.T) { | ||
| tests := []struct { | ||
| name string // description of this test case | ||
| // Named input parameters for target function. | ||
| dest http.Header | ||
| src http.Header | ||
| want http.Header | ||
| name string | ||
| encoding string | ||
| compress func(t *testing.T, body []byte) []byte | ||
| }{ | ||
| { | ||
| name: "given src Authorization header, should skip sensitive header", | ||
| dest: http.Header{ | ||
| "Content-Type": []string{"application/json"}, | ||
| }, | ||
| src: http.Header{ | ||
| "Content-Type": []string{"application/json"}, | ||
| "Authorization": []string{"Bearer 123456"}, | ||
| }, | ||
| want: http.Header{ | ||
| "Content-Type": []string{"application/json"}, | ||
| name: "gzip uppercase", | ||
| encoding: "GZIP", | ||
| compress: func(t *testing.T, body []byte) []byte { | ||
| var buf bytes.Buffer | ||
| writer := gzip.NewWriter(&buf) | ||
| _, err := writer.Write(body) | ||
| require.NoError(t, err) | ||
| writer.Close() | ||
| return buf.Bytes() | ||
| }, | ||
| }, | ||
| { | ||
| name: "given src User-Agent header, should merge them", | ||
| dest: http.Header{ | ||
| "Content-Type": []string{"application/json"}, | ||
| }, | ||
| src: http.Header{ | ||
| "Content-Type": []string{"application/json"}, | ||
| "User-Agent": []string{"Mozilla/5.0"}, | ||
| }, | ||
| want: http.Header{ | ||
| "Content-Type": []string{"application/json"}, | ||
| "User-Agent": []string{"Mozilla/5.0"}, | ||
| name: "deflate uppercase", | ||
| encoding: "DEFLATE", | ||
| compress: func(t *testing.T, body []byte) []byte { | ||
| var buf bytes.Buffer | ||
| writer, err := flate.NewWriter(&buf, flate.DefaultCompression) | ||
| require.NoError(t, err) | ||
| _, err = writer.Write(body) | ||
| require.NoError(t, err) | ||
| writer.Close() | ||
| return buf.Bytes() | ||
| }, | ||
| }, | ||
| { | ||
| name: "should add non-duplicate values to existing headers", | ||
| dest: http.Header{ | ||
| "Content-Type": []string{"application/json"}, | ||
| "Accept": []string{"application/json"}, | ||
| }, | ||
| src: http.Header{ | ||
| "Accept": []string{"text/plain", "application/json"}, | ||
| "User-Agent": []string{"Mozilla/5.0"}, | ||
| }, | ||
| want: http.Header{ | ||
| "Content-Type": []string{"application/json"}, | ||
| "Accept": []string{"application/json", "text/plain"}, | ||
| "User-Agent": []string{"Mozilla/5.0"}, | ||
| }, | ||
| }, | ||
| { | ||
| name: "should add all non-duplicate values from multiple values", | ||
| dest: http.Header{ | ||
| "Accept": []string{"application/json"}, | ||
| }, | ||
| src: http.Header{ | ||
| "Accept": []string{"text/plain", "application/xml", "text/html"}, | ||
| }, | ||
| want: http.Header{ | ||
| "Accept": []string{"application/json", "text/plain", "application/xml", "text/html"}, | ||
| }, | ||
| }, | ||
| { | ||
| name: "should skip all duplicate values", | ||
| dest: http.Header{ | ||
| "Accept": []string{"application/json", "text/plain"}, | ||
| }, | ||
| src: http.Header{ | ||
| "Accept": []string{"application/json", "text/plain"}, | ||
| }, | ||
| want: http.Header{ | ||
| "Accept": []string{"application/json", "text/plain"}, | ||
| }, | ||
| }, | ||
| { | ||
| name: "should skip only duplicate values and add new ones", | ||
| dest: http.Header{ | ||
| "Accept": []string{"application/json", "text/plain"}, | ||
| }, | ||
| src: http.Header{ | ||
| "Accept": []string{"text/plain", "application/xml", "application/json"}, | ||
| }, | ||
| want: http.Header{ | ||
| "Accept": []string{"application/json", "text/plain", "application/xml"}, | ||
| }, | ||
| }, | ||
| { | ||
| name: "should block transport-managed headers and skip sensitive ones", | ||
| dest: http.Header{ | ||
| "Content-Type": []string{"application/json"}, | ||
| }, | ||
| src: http.Header{ | ||
| "Authorization": []string{"Bearer token"}, | ||
| "Api-Key": []string{"key123"}, | ||
| "X-Api-Key": []string{"xkey456"}, | ||
| "X-Api-Secret": []string{"secret789"}, | ||
| "X-Api-Token": []string{"token000"}, | ||
| "Content-Type": []string{"text/plain"}, | ||
| "Content-Length": []string{"100"}, | ||
| "Transfer-Encoding": []string{"chunked"}, | ||
| "User-Agent": []string{"Test/1.0"}, | ||
| }, | ||
| want: http.Header{ | ||
| "Content-Type": []string{"application/json"}, | ||
| "User-Agent": []string{"Test/1.0"}, | ||
| }, | ||
| }, | ||
| { | ||
| name: "empty src headers should not change dest", | ||
| dest: http.Header{ | ||
| "Content-Type": []string{"application/json"}, | ||
| }, | ||
| src: http.Header{}, | ||
| want: http.Header{ | ||
| "Content-Type": []string{"application/json"}, | ||
| }, | ||
| }, | ||
| { | ||
| name: "empty dest headers should merge non-blocked src headers", | ||
| dest: http.Header{}, | ||
| src: http.Header{ | ||
| "User-Agent": []string{"Test/1.0"}, | ||
| "Accept": []string{"*/*"}, | ||
| "Authorization": []string{"Bearer token"}, | ||
| }, | ||
| want: http.Header{ | ||
| "User-Agent": []string{"Test/1.0"}, | ||
| "Accept": []string{"*/*"}, | ||
| }, | ||
| }, | ||
| { | ||
| name: "should merge multiple custom headers", | ||
| dest: http.Header{ | ||
| "Content-Type": []string{"application/json"}, | ||
| }, | ||
| src: http.Header{ | ||
| "X-Request-ID": []string{"req-123"}, | ||
| "X-Trace-ID": []string{"trace-456"}, | ||
| "User-Agent": []string{"Custom/1.0"}, | ||
| "Accept-Encoding": []string{"gzip, deflate"}, | ||
| }, | ||
| want: http.Header{ | ||
| "Content-Type": []string{"application/json"}, | ||
| "X-Request-ID": []string{"req-123"}, | ||
| "X-Trace-ID": []string{"trace-456"}, | ||
| "User-Agent": []string{"Custom/1.0"}, | ||
| }, | ||
| }, | ||
| { | ||
| name: "should handle headers with multiple values", | ||
| dest: http.Header{ | ||
| "Content-Type": []string{"application/json"}, | ||
| }, | ||
| src: http.Header{ | ||
| "Accept": []string{"application/json", "text/plain"}, | ||
| }, | ||
| want: http.Header{ | ||
| "Content-Type": []string{"application/json"}, | ||
| "Accept": []string{"application/json", "text/plain"}, | ||
| }, | ||
| }, | ||
| { | ||
| name: "should add non-duplicate values when header exists", | ||
| dest: http.Header{ | ||
| "User-Agent": []string{"AxonHub/1.0"}, | ||
| }, | ||
| src: http.Header{ | ||
| "Content-Type": []string{"application/json"}, | ||
| "User-Agent": []string{"Mozilla/5.0"}, | ||
| "Accept": []string{"*/*"}, | ||
| }, | ||
| want: http.Header{ | ||
| "User-Agent": []string{"AxonHub/1.0", "Mozilla/5.0"}, | ||
| "Accept": []string{"*/*"}, | ||
| }, | ||
| }, | ||
| { | ||
| name: "should overwrite non-appendable headers", | ||
| dest: http.Header{ | ||
| "X-Custom-Header": []string{"old-value"}, | ||
| }, | ||
| src: http.Header{ | ||
| "X-Custom-Header": []string{"new-value"}, | ||
| }, | ||
| want: http.Header{ | ||
| "X-Custom-Header": []string{"new-value"}, | ||
| name: "zstd with spaces", | ||
| encoding: " ZSTD ", | ||
| compress: func(t *testing.T, body []byte) []byte { | ||
| encoder, err := zstd.NewWriter(nil) | ||
| require.NoError(t, err) | ||
| compressed := encoder.EncodeAll(body, nil) | ||
| encoder.Close() | ||
| return compressed | ||
| }, | ||
| }, | ||
| } | ||
|
|
||
| for _, tt := range tests { | ||
| t.Run(tt.name, func(t *testing.T) { | ||
| got := MergeHTTPHeaders(tt.dest, tt.src) | ||
| require.Equal(t, tt.want, got) | ||
| originalBody := []byte(`{"model":"gpt-4"}`) | ||
| compressedBody := tt.compress(t, originalBody) | ||
|
|
||
| req := httptest.NewRequest(http.MethodPost, "/v1/chat/completions", bytes.NewReader(compressedBody)) | ||
| req.Header.Set("Content-Type", "application/json") | ||
| req.Header.Set("Content-Encoding", tt.encoding) | ||
|
|
||
| got, err := ReadHTTPRequest(req) | ||
| require.NoError(t, err) | ||
| assert.Equal(t, originalBody, got.Body) | ||
| }) | ||
| } | ||
| } | ||
|
|
||
| // TestHeaderMaps_CanonicalForm verifies every key in the three header maps | ||
| // matches Go's http.CanonicalHeaderKey form. A mismatch causes silent lookup | ||
| // failures when iterating http.Header (which always uses canonical keys). | ||
| func TestHeaderMaps_CanonicalForm(t *testing.T) { | ||
| for name, m := range map[string]map[string]bool{ | ||
| "blockedHeaders": blockedHeaders, | ||
| "sensitiveHeaders": sensitiveHeaders, | ||
| "libManagedHeaders": libManagedHeaders, | ||
| } { | ||
| for k := range m { | ||
| canonical := http.CanonicalHeaderKey(k) | ||
| require.Equal(t, canonical, k, "%s key %q should be canonical form %q", name, k, canonical) | ||
| } | ||
| } | ||
| } | ||
| func TestReadHTTPRequest_UnsupportedContentEncoding(t *testing.T) { | ||
| body := []byte(`{"model":"gpt-4"}`) | ||
| req := httptest.NewRequest(http.MethodPost, "/v1/chat/completions", bytes.NewReader(body)) | ||
| req.Header.Set("Content-Type", "application/json") | ||
| req.Header.Set("Content-Encoding", "br") | ||
|
|
||
| // TestMergeHTTPHeaders_BlocksAllHardcodedHeaders builds an http.Header via | ||
| // Set() (auto-canonicalized, matching real HTTP server behavior) for every | ||
| // header in blockedHeaders, sensitiveHeaders and libManagedHeaders, then | ||
| // verifies none of them appear in the merge result. | ||
| func TestMergeHTTPHeaders_BlocksAllHardcodedHeaders(t *testing.T) { | ||
| src := make(http.Header) | ||
| for k := range blockedHeaders { | ||
| src.Set(k, "blocked-val") | ||
| } | ||
| _, err := ReadHTTPRequest(req) | ||
| require.Error(t, err) | ||
| assert.Contains(t, err.Error(), "unsupported content encoding") | ||
| } | ||
|
|
||
| for k := range sensitiveHeaders { | ||
| src.Set(k, "sensitive-val") | ||
| } | ||
| func TestReadHTTPRequest_InvalidGzipData(t *testing.T) { | ||
| invalidData := []byte("this is not valid gzip compressed data") | ||
| req := httptest.NewRequest(http.MethodPost, "/v1/chat/completions", bytes.NewReader(invalidData)) | ||
| req.Header.Set("Content-Type", "application/json") | ||
| req.Header.Set("Content-Encoding", "gzip") | ||
|
|
||
| for k := range libManagedHeaders { | ||
| src.Set(k, "lib-val") | ||
| } | ||
| _, err := ReadHTTPRequest(req) | ||
| require.Error(t, err) | ||
| assert.Contains(t, err.Error(), "failed to create gzip reader") | ||
| } | ||
|
|
||
| src.Set("X-Custom", "keep-me") | ||
| func TestReadHTTPRequest_InvalidDeflateData(t *testing.T) { | ||
| invalidData := []byte("this is not valid deflate compressed data") | ||
| req := httptest.NewRequest(http.MethodPost, "/v1/chat/completions", bytes.NewReader(invalidData)) | ||
| req.Header.Set("Content-Type", "application/json") | ||
| req.Header.Set("Content-Encoding", "deflate") | ||
|
|
||
| dest := make(http.Header) | ||
| got := MergeHTTPHeaders(dest, src) | ||
| _, err := ReadHTTPRequest(req) | ||
| require.Error(t, err) | ||
| assert.Contains(t, err.Error(), "failed to decompress deflate body") | ||
| } | ||
|
|
||
| for k := range blockedHeaders { | ||
| require.Empty(t, got.Values(k), "blockedHeaders %q should not be merged", k) | ||
| } | ||
| func TestReadHTTPRequest_InvalidZstdData(t *testing.T) { | ||
| invalidData := []byte("this is not valid zstd compressed data") | ||
| req := httptest.NewRequest(http.MethodPost, "/v1/chat/completions", bytes.NewReader(invalidData)) | ||
| req.Header.Set("Content-Type", "application/json") | ||
| req.Header.Set("Content-Encoding", "zstd") | ||
|
|
||
| for k := range sensitiveHeaders { | ||
| require.Empty(t, got.Values(k), "sensitiveHeaders %q should not be merged", k) | ||
| } | ||
| _, err := ReadHTTPRequest(req) | ||
| require.Error(t, err) | ||
| assert.Contains(t, err.Error(), "failed to decode zstd compressed body") | ||
| } | ||
|
|
||
| for k := range libManagedHeaders { | ||
| require.Empty(t, got.Values(k), "libManagedHeaders %q should not be merged", k) | ||
| } | ||
| func TestReadHTTPRequest_EmptyBody(t *testing.T) { | ||
| req := httptest.NewRequest(http.MethodPost, "/v1/chat/completions", nil) | ||
| req.Header.Set("Content-Type", "application/json") | ||
|
|
||
| require.Equal(t, "keep-me", got.Get("X-Custom"), "non-blocked header should be merged") | ||
| got, err := ReadHTTPRequest(req) | ||
| require.NoError(t, err) | ||
| assert.Empty(t, got.Body) | ||
| } | ||
|
|
||
| // TestMaskSensitiveHeaders_MasksAllHardcodedHeaders verifies every header in | ||
| // sensitiveHeaders is masked to "******" by MaskSensitiveHeaders. | ||
| func TestMaskSensitiveHeaders_MasksAllHardcodedHeaders(t *testing.T) { | ||
| headers := make(http.Header) | ||
| for k := range sensitiveHeaders { | ||
| headers.Set(k, "secret-value") | ||
| } | ||
| func TestReadHTTPRequest_EmptyBodyWithContentEncoding(t *testing.T) { | ||
| req := httptest.NewRequest(http.MethodPost, "/v1/chat/completions", nil) | ||
| req.Header.Set("Content-Type", "application/json") | ||
| req.Header.Set("Content-Encoding", "zstd") | ||
|
|
||
| headers.Set("X-Custom", "visible") | ||
| got, err := ReadHTTPRequest(req) | ||
| require.NoError(t, err) | ||
| assert.Empty(t, got.Body) | ||
| } | ||
|
|
||
| got := MaskSensitiveHeaders(headers) | ||
| func TestDecodeRequestBody_NoEncoding(t *testing.T) { | ||
| body := []byte(`{"test":"data"}`) | ||
| headers := http.Header{} | ||
|
|
||
| for k := range sensitiveHeaders { | ||
| require.Equal(t, []string{"******"}, got.Values(k), | ||
| "sensitiveHeaders %q should be masked", k) | ||
| } | ||
| got, err := decodeRequestBody(body, headers) | ||
| require.NoError(t, err) | ||
| assert.Equal(t, body, got) | ||
| } | ||
|
|
||
| func TestDecodeRequestBody_IdentityEncoding(t *testing.T) { | ||
| body := []byte(`{"test":"data"}`) | ||
| headers := http.Header{} | ||
| headers.Set("Content-Encoding", "identity") | ||
|
|
||
| require.Equal(t, []string{"visible"}, got.Values("X-Custom")) | ||
| got, err := decodeRequestBody(body, headers) | ||
| require.NoError(t, err) | ||
| assert.Equal(t, body, got) | ||
| } | ||
|
|
||
| func TestRegisterAppendHeaders(t *testing.T) { | ||
| RegisterMergeWithAppendHeaders("X-New-Append") | ||
| func TestDecodeRequestBody_GzipEncoding(t *testing.T) { | ||
| originalBody := []byte(`{"test":"data"}`) | ||
|
|
||
| dest := http.Header{"X-New-Append": []string{"old"}} | ||
| src := http.Header{"X-New-Append": []string{"new"}} | ||
| var buf bytes.Buffer | ||
| writer := gzip.NewWriter(&buf) | ||
| _, err := writer.Write(originalBody) | ||
| require.NoError(t, err) | ||
| writer.Close() | ||
|
|
||
| got := MergeHTTPHeaders(dest, src) | ||
| require.Equal(t, []string{"old", "new"}, got["X-New-Append"]) | ||
| headers := http.Header{} | ||
| headers.Set("Content-Encoding", "gzip") | ||
| headers.Set("Content-Length", "100") | ||
|
|
||
| got, err := decodeRequestBody(buf.Bytes(), headers) | ||
| require.NoError(t, err) | ||
| assert.Equal(t, originalBody, got) | ||
| assert.Equal(t, "", headers.Get("Content-Encoding")) | ||
| assert.Equal(t, "", headers.Get("Content-Length")) | ||
| } | ||
|
|
||
| func TestMergeHTTPQuery(t *testing.T) { | ||
| tests := []struct { | ||
| name string | ||
| dest url.Values | ||
| src url.Values | ||
| want url.Values | ||
| }{ | ||
| { | ||
| name: "should merge new query parameters", | ||
| dest: url.Values{"q": []string{"golang"}}, | ||
| src: url.Values{"page": []string{"1"}}, | ||
| want: url.Values{"q": []string{"golang"}, "page": []string{"1"}}, | ||
| }, | ||
| { | ||
| name: "should not overwrite existing query parameters", | ||
| dest: url.Values{"q": []string{"golang"}}, | ||
| src: url.Values{"q": []string{"java"}}, | ||
| want: url.Values{"q": []string{"golang"}}, | ||
| }, | ||
| { | ||
| name: "should handle empty src", | ||
| dest: url.Values{"q": []string{"golang"}}, | ||
| src: nil, | ||
| want: url.Values{"q": []string{"golang"}}, | ||
| }, | ||
| { | ||
| name: "should handle empty dest", | ||
| dest: nil, | ||
| src: url.Values{"page": []string{"1"}}, | ||
| want: url.Values{"page": []string{"1"}}, | ||
| }, | ||
| } | ||
| func TestDecodeRequestBody_DeflateEncoding(t *testing.T) { | ||
| originalBody := []byte(`{"test":"data"}`) | ||
|
|
||
| for _, tt := range tests { | ||
| t.Run(tt.name, func(t *testing.T) { | ||
| got := MergeHTTPQuery(tt.dest, tt.src) | ||
| require.Equal(t, tt.want, got) | ||
| }) | ||
| } | ||
| var buf bytes.Buffer | ||
| writer, err := flate.NewWriter(&buf, flate.DefaultCompression) | ||
| require.NoError(t, err) | ||
| _, err = writer.Write(originalBody) | ||
| require.NoError(t, err) | ||
| writer.Close() | ||
|
|
||
| headers := http.Header{} | ||
| headers.Set("Content-Encoding", "deflate") | ||
|
|
||
| got, err := decodeRequestBody(buf.Bytes(), headers) | ||
| require.NoError(t, err) | ||
| assert.Equal(t, originalBody, got) | ||
| assert.Equal(t, "", headers.Get("Content-Encoding")) | ||
| } | ||
|
|
||
| func TestMergeInboundRequest(t *testing.T) { | ||
| t.Run("should merge both headers and query", func(t *testing.T) { | ||
| dest := &Request{ | ||
| Headers: http.Header{"Content-Type": []string{"application/json"}}, | ||
| Query: url.Values{"q": []string{"old"}}, | ||
| } | ||
| src := &Request{ | ||
| Headers: http.Header{"User-Agent": []string{"Test"}}, | ||
| Query: url.Values{"page": []string{"1"}}, | ||
| } | ||
|
|
||
| got := MergeInboundRequest(dest, src) | ||
| require.Equal(t, "application/json", got.Headers.Get("Content-Type")) | ||
| require.Equal(t, "Test", got.Headers.Get("User-Agent")) | ||
| require.Equal(t, "old", got.Query.Get("q")) | ||
| require.Equal(t, "1", got.Query.Get("page")) | ||
| }) | ||
|
|
||
| t.Run("should block Cloudflare headers by prefix", func(t *testing.T) { | ||
| dest := &Request{ | ||
| Headers: http.Header{"Content-Type": []string{"application/json"}}, | ||
| Query: url.Values{}, | ||
| } | ||
| src := &Request{ | ||
| Headers: http.Header{ | ||
| "Cf-Ray": []string{"abc123"}, | ||
| "Cf-Connecting-Ip": []string{"1.2.3.4"}, | ||
| "Cf-Ipcountry": []string{"US"}, | ||
| "Cf-Visitor": []string{`{"scheme":"https"}`}, | ||
| "Cdn-Loop": []string{"cloudflare; loops=1"}, | ||
| "User-Agent": []string{"Test/1.0"}, | ||
| }, | ||
| Query: url.Values{}, | ||
| } | ||
|
|
||
| got := MergeInboundRequest(dest, src) | ||
| require.Empty(t, got.Headers.Get("Cf-Ray")) | ||
| require.Empty(t, got.Headers.Get("Cf-Connecting-Ip")) | ||
| require.Empty(t, got.Headers.Get("Cf-Ipcountry")) | ||
| require.Empty(t, got.Headers.Get("Cf-Visitor")) | ||
| require.Empty(t, got.Headers.Get("Cdn-Loop")) | ||
| require.Equal(t, "Test/1.0", got.Headers.Get("User-Agent")) | ||
| }) | ||
|
|
||
| t.Run("should block hop-by-hop headers from inbound request", func(t *testing.T) { | ||
| dest := &Request{ | ||
| Headers: http.Header{"Content-Type": []string{"application/json"}}, | ||
| Query: url.Values{}, | ||
| } | ||
| src := &Request{ | ||
| Headers: http.Header{ | ||
| "Upgrade": []string{"h2c"}, | ||
| "Keep-Alive": []string{"timeout=5"}, | ||
| "Te": []string{"trailers"}, | ||
| "Trailer": []string{"X-Trailer"}, | ||
| "Proxy-Connection": []string{"keep-alive"}, | ||
| "User-Agent": []string{"Test/1.0"}, | ||
| }, | ||
| Query: url.Values{}, | ||
| } | ||
|
|
||
| got := MergeInboundRequest(dest, src) | ||
| require.Empty(t, got.Headers.Get("Upgrade")) | ||
| require.Empty(t, got.Headers.Get("Keep-Alive")) | ||
| require.Empty(t, got.Headers.Get("Te")) | ||
| require.Empty(t, got.Headers.Get("Trailer")) | ||
| require.Empty(t, got.Headers.Get("Proxy-Connection")) | ||
| require.Equal(t, "Test/1.0", got.Headers.Get("User-Agent")) | ||
| }) | ||
|
|
||
| t.Run("should return dest if src is nil", func(t *testing.T) { | ||
| dest := &Request{Headers: http.Header{"X-Test": []string{"val"}}} | ||
| got := MergeInboundRequest(dest, nil) | ||
| require.Equal(t, dest, got) | ||
| }) | ||
| func TestDecodeRequestBody_ZstdEncoding(t *testing.T) { | ||
| originalBody := []byte(`{"test":"data"}`) | ||
|
|
||
| encoder, err := zstd.NewWriter(nil) | ||
| require.NoError(t, err) | ||
| compressedBody := encoder.EncodeAll(originalBody, nil) | ||
| encoder.Close() | ||
|
|
||
| headers := http.Header{} | ||
| headers.Set("Content-Encoding", "zstd") | ||
|
|
||
| got, err := decodeRequestBody(compressedBody, headers) | ||
| require.NoError(t, err) | ||
| assert.Equal(t, originalBody, got) | ||
| assert.Equal(t, "", headers.Get("Content-Encoding")) | ||
| } |
There was a problem hiding this comment.
It appears that all existing tests in this file (such as TestIsHTTPStatusCodeRetryable, TestMergeHTTPHeaders, TestMergeHTTPQuery, and TestMergeInboundRequest) have been deleted and replaced by the new tests. Please ensure that the new tests are appended to the file instead of overwriting the existing ones to maintain test coverage.
| } | ||
| defer reader.Close() | ||
|
|
||
| decoded, err := io.ReadAll(reader) |
There was a problem hiding this comment.
Using io.ReadAll on a decompression reader without a size limit is vulnerable to memory exhaustion attacks (Zip Bombs). A small malicious payload could expand to a size that exceeds available memory. Consider using io.LimitReader to restrict the maximum decompressed size. This risk also applies to the deflate and zstd implementations.
| uuid@14.0.0: | ||
| resolution: {integrity: sha512-Qo+uWgilfSmAhXCMav1uYFynlQO7fMFiMVZsQqZRMIXp0O7rR7qjkj+cPvBHLgBqi960QCoo/PH2/6ZtVqKvrg==} | ||
| hasBin: true |
| reader := flate.NewReader(bytes.NewReader(body)) | ||
| defer reader.Close() |
There was a problem hiding this comment.
| github.com/google/jsonschema-go v0.3.1-0.20251120200837-98a387e3b975 | ||
| github.com/google/uuid v1.6.0 | ||
| github.com/kaptinlin/jsonrepair v0.2.4 | ||
| github.com/klauspost/compress v1.17.9 |
Greptile SummaryThis PR adds transparent auto-decompression of inbound request bodies for
Confidence Score: 4/5Safe to merge with the zlib fallback gap fixed; the bug only manifests for raw DEFLATE bodies whose first two bytes coincidentally satisfy the zlib checksum, but when it does trigger the request silently fails. The decompression logic is correct for the vast majority of real-world inputs, but llm/httpclient/utils.go — specifically Important Files Changed
Flowchart%%{init: {'theme': 'neutral'}}%%
flowchart TD
A[Incoming HTTP Request] --> B[io.ReadAll body]
B --> C{len body > 0?}
C -- No --> G[req.Body = nil]
C -- Yes --> D{Content-Encoding header?}
D -- None / identity --> E[return body as-is]
D -- gzip / x-gzip --> F1[gzip.NewReader]
F1 -- error --> ERR[return error]
F1 -- ok --> F2[io.ReadAll]
F2 -- error --> ERR
F2 -- ok --> HDR1[Del Content-Encoding Del Content-Length]
HDR1 --> G
D -- deflate --> Z1[zlib.NewReader]
Z1 -- error --> Z2[flate.NewReader io.ReadAll]
Z1 -- ok --> Z3[io.ReadAll]
Z3 -- error --> ERR2[return error no fallback to flate]
Z3 -- ok --> HDR2[Del Content-Encoding Del Content-Length]
Z2 -- error --> ERR
Z2 -- ok --> HDR2
HDR2 --> G
D -- zstd --> S1[zstd decoder.DecodeAll]
S1 -- error --> ERR
S1 -- ok --> HDR3[Del Content-Encoding Del Content-Length]
HDR3 --> G
D -- other --> ERR3[unsupported content encoding error]
G --> H[Return Request]
Reviews (2): Last reviewed commit: "feat: auto decompress request body, clos..." | Re-trigger Greptile |
No description provided.