Skip to content

feat: auto decompress request body, close #1680#1688

Merged
looplj merged 1 commit into
unstablefrom
dev-tmp
May 21, 2026
Merged

feat: auto decompress request body, close #1680#1688
looplj merged 1 commit into
unstablefrom
dev-tmp

Conversation

@looplj

@looplj looplj commented May 21, 2026

Copy link
Copy Markdown
Owner

No description provided.

@gemini-code-assist gemini-code-assist Bot left a comment

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Code Review

This pull request introduces HTTP request body decompression support for gzip, deflate, and zstd encodings within the llm/httpclient package. It also updates several frontend dependencies and adds the klauspost/compress library. Feedback highlights critical issues, including the accidental removal of existing tests in utils_test.go and a potential memory exhaustion vulnerability (Zip Bomb) due to the lack of size limits on decompression readers. Furthermore, the deflate implementation should be switched to zlib for RFC compliance, a suspicious uuid version in the lockfile requires verification, and compression library versions should be synchronized across modules.

Comment on lines +16 to 314
func TestReadHTTPRequest_NoContentEncoding(t *testing.T) {
body := []byte(`{"model":"gpt-4","messages":[{"role":"user","content":"hello"}]}`)
req := httptest.NewRequest(http.MethodPost, "/v1/chat/completions", bytes.NewReader(body))
req.Header.Set("Content-Type", "application/json")

got, err := ReadHTTPRequest(req)
require.NoError(t, err)
assert.Equal(t, body, got.Body)
assert.Equal(t, "", got.Headers.Get("Content-Encoding"))
}

func TestReadHTTPRequest_IdentityEncoding(t *testing.T) {
body := []byte(`{"model":"gpt-4","messages":[{"role":"user","content":"hello"}]}`)
req := httptest.NewRequest(http.MethodPost, "/v1/chat/completions", bytes.NewReader(body))
req.Header.Set("Content-Type", "application/json")
req.Header.Set("Content-Encoding", "identity")

got, err := ReadHTTPRequest(req)
require.NoError(t, err)
assert.Equal(t, body, got.Body)
assert.Equal(t, "identity", got.Headers.Get("Content-Encoding"))
}

func TestReadHTTPRequest_GzipEncoding(t *testing.T) {
originalBody := []byte(`{"model":"gpt-4","messages":[{"role":"user","content":"hello"}]}`)

var buf bytes.Buffer
writer := gzip.NewWriter(&buf)
_, err := writer.Write(originalBody)
require.NoError(t, err)
writer.Close()

req := httptest.NewRequest(http.MethodPost, "/v1/chat/completions", bytes.NewReader(buf.Bytes()))
req.Header.Set("Content-Type", "application/json")
req.Header.Set("Content-Encoding", "gzip")

got, err := ReadHTTPRequest(req)
require.NoError(t, err)
assert.Equal(t, originalBody, got.Body)
assert.Equal(t, "", got.Headers.Get("Content-Encoding"))
assert.Equal(t, "", got.Headers.Get("Content-Length"))
}

func TestReadHTTPRequest_GzipEncodingXGzip(t *testing.T) {
originalBody := []byte(`{"model":"gpt-4"}`)

var buf bytes.Buffer
writer := gzip.NewWriter(&buf)
_, err := writer.Write(originalBody)
require.NoError(t, err)
writer.Close()

req := httptest.NewRequest(http.MethodPost, "/v1/chat/completions", bytes.NewReader(buf.Bytes()))
req.Header.Set("Content-Type", "application/json")
req.Header.Set("Content-Encoding", "x-gzip")

got, err := ReadHTTPRequest(req)
require.NoError(t, err)
assert.Equal(t, originalBody, got.Body)
assert.Equal(t, "", got.Headers.Get("Content-Encoding"))
}

func TestReadHTTPRequest_DeflateEncoding(t *testing.T) {
originalBody := []byte(`{"model":"gpt-4","messages":[{"role":"user","content":"hello"}]}`)

var buf bytes.Buffer
writer, err := flate.NewWriter(&buf, flate.DefaultCompression)
require.NoError(t, err)
_, err = writer.Write(originalBody)
require.NoError(t, err)
writer.Close()

req := httptest.NewRequest(http.MethodPost, "/v1/chat/completions", bytes.NewReader(buf.Bytes()))
req.Header.Set("Content-Type", "application/json")
req.Header.Set("Content-Encoding", "deflate")

got, err := ReadHTTPRequest(req)
require.NoError(t, err)
assert.Equal(t, originalBody, got.Body)
assert.Equal(t, "", got.Headers.Get("Content-Encoding"))
assert.Equal(t, "", got.Headers.Get("Content-Length"))
}

func TestMergeHTTPHeaders(t *testing.T) {
// Register headers for testing append behavior
RegisterMergeWithAppendHeaders("User-Agent", "Accept")
func TestReadHTTPRequest_ZstdEncoding(t *testing.T) {
originalBody := []byte(`{"model":"gpt-4","messages":[{"role":"user","content":"hello"}]}`)

encoder, err := zstd.NewWriter(nil)
require.NoError(t, err)
compressedBody := encoder.EncodeAll(originalBody, nil)
encoder.Close()

req := httptest.NewRequest(http.MethodPost, "/v1/chat/completions", bytes.NewReader(compressedBody))
req.Header.Set("Content-Type", "application/json")
req.Header.Set("Content-Encoding", "zstd")

got, err := ReadHTTPRequest(req)
require.NoError(t, err)
assert.Equal(t, originalBody, got.Body)
assert.Equal(t, "", got.Headers.Get("Content-Encoding"))
assert.Equal(t, "", got.Headers.Get("Content-Length"))
}

func TestReadHTTPRequest_EncodingCaseInsensitive(t *testing.T) {
tests := []struct {
name string // description of this test case
// Named input parameters for target function.
dest http.Header
src http.Header
want http.Header
name string
encoding string
compress func(t *testing.T, body []byte) []byte
}{
{
name: "given src Authorization header, should skip sensitive header",
dest: http.Header{
"Content-Type": []string{"application/json"},
},
src: http.Header{
"Content-Type": []string{"application/json"},
"Authorization": []string{"Bearer 123456"},
},
want: http.Header{
"Content-Type": []string{"application/json"},
name: "gzip uppercase",
encoding: "GZIP",
compress: func(t *testing.T, body []byte) []byte {
var buf bytes.Buffer
writer := gzip.NewWriter(&buf)
_, err := writer.Write(body)
require.NoError(t, err)
writer.Close()
return buf.Bytes()
},
},
{
name: "given src User-Agent header, should merge them",
dest: http.Header{
"Content-Type": []string{"application/json"},
},
src: http.Header{
"Content-Type": []string{"application/json"},
"User-Agent": []string{"Mozilla/5.0"},
},
want: http.Header{
"Content-Type": []string{"application/json"},
"User-Agent": []string{"Mozilla/5.0"},
name: "deflate uppercase",
encoding: "DEFLATE",
compress: func(t *testing.T, body []byte) []byte {
var buf bytes.Buffer
writer, err := flate.NewWriter(&buf, flate.DefaultCompression)
require.NoError(t, err)
_, err = writer.Write(body)
require.NoError(t, err)
writer.Close()
return buf.Bytes()
},
},
{
name: "should add non-duplicate values to existing headers",
dest: http.Header{
"Content-Type": []string{"application/json"},
"Accept": []string{"application/json"},
},
src: http.Header{
"Accept": []string{"text/plain", "application/json"},
"User-Agent": []string{"Mozilla/5.0"},
},
want: http.Header{
"Content-Type": []string{"application/json"},
"Accept": []string{"application/json", "text/plain"},
"User-Agent": []string{"Mozilla/5.0"},
},
},
{
name: "should add all non-duplicate values from multiple values",
dest: http.Header{
"Accept": []string{"application/json"},
},
src: http.Header{
"Accept": []string{"text/plain", "application/xml", "text/html"},
},
want: http.Header{
"Accept": []string{"application/json", "text/plain", "application/xml", "text/html"},
},
},
{
name: "should skip all duplicate values",
dest: http.Header{
"Accept": []string{"application/json", "text/plain"},
},
src: http.Header{
"Accept": []string{"application/json", "text/plain"},
},
want: http.Header{
"Accept": []string{"application/json", "text/plain"},
},
},
{
name: "should skip only duplicate values and add new ones",
dest: http.Header{
"Accept": []string{"application/json", "text/plain"},
},
src: http.Header{
"Accept": []string{"text/plain", "application/xml", "application/json"},
},
want: http.Header{
"Accept": []string{"application/json", "text/plain", "application/xml"},
},
},
{
name: "should block transport-managed headers and skip sensitive ones",
dest: http.Header{
"Content-Type": []string{"application/json"},
},
src: http.Header{
"Authorization": []string{"Bearer token"},
"Api-Key": []string{"key123"},
"X-Api-Key": []string{"xkey456"},
"X-Api-Secret": []string{"secret789"},
"X-Api-Token": []string{"token000"},
"Content-Type": []string{"text/plain"},
"Content-Length": []string{"100"},
"Transfer-Encoding": []string{"chunked"},
"User-Agent": []string{"Test/1.0"},
},
want: http.Header{
"Content-Type": []string{"application/json"},
"User-Agent": []string{"Test/1.0"},
},
},
{
name: "empty src headers should not change dest",
dest: http.Header{
"Content-Type": []string{"application/json"},
},
src: http.Header{},
want: http.Header{
"Content-Type": []string{"application/json"},
},
},
{
name: "empty dest headers should merge non-blocked src headers",
dest: http.Header{},
src: http.Header{
"User-Agent": []string{"Test/1.0"},
"Accept": []string{"*/*"},
"Authorization": []string{"Bearer token"},
},
want: http.Header{
"User-Agent": []string{"Test/1.0"},
"Accept": []string{"*/*"},
},
},
{
name: "should merge multiple custom headers",
dest: http.Header{
"Content-Type": []string{"application/json"},
},
src: http.Header{
"X-Request-ID": []string{"req-123"},
"X-Trace-ID": []string{"trace-456"},
"User-Agent": []string{"Custom/1.0"},
"Accept-Encoding": []string{"gzip, deflate"},
},
want: http.Header{
"Content-Type": []string{"application/json"},
"X-Request-ID": []string{"req-123"},
"X-Trace-ID": []string{"trace-456"},
"User-Agent": []string{"Custom/1.0"},
},
},
{
name: "should handle headers with multiple values",
dest: http.Header{
"Content-Type": []string{"application/json"},
},
src: http.Header{
"Accept": []string{"application/json", "text/plain"},
},
want: http.Header{
"Content-Type": []string{"application/json"},
"Accept": []string{"application/json", "text/plain"},
},
},
{
name: "should add non-duplicate values when header exists",
dest: http.Header{
"User-Agent": []string{"AxonHub/1.0"},
},
src: http.Header{
"Content-Type": []string{"application/json"},
"User-Agent": []string{"Mozilla/5.0"},
"Accept": []string{"*/*"},
},
want: http.Header{
"User-Agent": []string{"AxonHub/1.0", "Mozilla/5.0"},
"Accept": []string{"*/*"},
},
},
{
name: "should overwrite non-appendable headers",
dest: http.Header{
"X-Custom-Header": []string{"old-value"},
},
src: http.Header{
"X-Custom-Header": []string{"new-value"},
},
want: http.Header{
"X-Custom-Header": []string{"new-value"},
name: "zstd with spaces",
encoding: " ZSTD ",
compress: func(t *testing.T, body []byte) []byte {
encoder, err := zstd.NewWriter(nil)
require.NoError(t, err)
compressed := encoder.EncodeAll(body, nil)
encoder.Close()
return compressed
},
},
}

for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got := MergeHTTPHeaders(tt.dest, tt.src)
require.Equal(t, tt.want, got)
originalBody := []byte(`{"model":"gpt-4"}`)
compressedBody := tt.compress(t, originalBody)

req := httptest.NewRequest(http.MethodPost, "/v1/chat/completions", bytes.NewReader(compressedBody))
req.Header.Set("Content-Type", "application/json")
req.Header.Set("Content-Encoding", tt.encoding)

got, err := ReadHTTPRequest(req)
require.NoError(t, err)
assert.Equal(t, originalBody, got.Body)
})
}
}

// TestHeaderMaps_CanonicalForm verifies every key in the three header maps
// matches Go's http.CanonicalHeaderKey form. A mismatch causes silent lookup
// failures when iterating http.Header (which always uses canonical keys).
func TestHeaderMaps_CanonicalForm(t *testing.T) {
for name, m := range map[string]map[string]bool{
"blockedHeaders": blockedHeaders,
"sensitiveHeaders": sensitiveHeaders,
"libManagedHeaders": libManagedHeaders,
} {
for k := range m {
canonical := http.CanonicalHeaderKey(k)
require.Equal(t, canonical, k, "%s key %q should be canonical form %q", name, k, canonical)
}
}
}
func TestReadHTTPRequest_UnsupportedContentEncoding(t *testing.T) {
body := []byte(`{"model":"gpt-4"}`)
req := httptest.NewRequest(http.MethodPost, "/v1/chat/completions", bytes.NewReader(body))
req.Header.Set("Content-Type", "application/json")
req.Header.Set("Content-Encoding", "br")

// TestMergeHTTPHeaders_BlocksAllHardcodedHeaders builds an http.Header via
// Set() (auto-canonicalized, matching real HTTP server behavior) for every
// header in blockedHeaders, sensitiveHeaders and libManagedHeaders, then
// verifies none of them appear in the merge result.
func TestMergeHTTPHeaders_BlocksAllHardcodedHeaders(t *testing.T) {
src := make(http.Header)
for k := range blockedHeaders {
src.Set(k, "blocked-val")
}
_, err := ReadHTTPRequest(req)
require.Error(t, err)
assert.Contains(t, err.Error(), "unsupported content encoding")
}

for k := range sensitiveHeaders {
src.Set(k, "sensitive-val")
}
func TestReadHTTPRequest_InvalidGzipData(t *testing.T) {
invalidData := []byte("this is not valid gzip compressed data")
req := httptest.NewRequest(http.MethodPost, "/v1/chat/completions", bytes.NewReader(invalidData))
req.Header.Set("Content-Type", "application/json")
req.Header.Set("Content-Encoding", "gzip")

for k := range libManagedHeaders {
src.Set(k, "lib-val")
}
_, err := ReadHTTPRequest(req)
require.Error(t, err)
assert.Contains(t, err.Error(), "failed to create gzip reader")
}

src.Set("X-Custom", "keep-me")
func TestReadHTTPRequest_InvalidDeflateData(t *testing.T) {
invalidData := []byte("this is not valid deflate compressed data")
req := httptest.NewRequest(http.MethodPost, "/v1/chat/completions", bytes.NewReader(invalidData))
req.Header.Set("Content-Type", "application/json")
req.Header.Set("Content-Encoding", "deflate")

dest := make(http.Header)
got := MergeHTTPHeaders(dest, src)
_, err := ReadHTTPRequest(req)
require.Error(t, err)
assert.Contains(t, err.Error(), "failed to decompress deflate body")
}

for k := range blockedHeaders {
require.Empty(t, got.Values(k), "blockedHeaders %q should not be merged", k)
}
func TestReadHTTPRequest_InvalidZstdData(t *testing.T) {
invalidData := []byte("this is not valid zstd compressed data")
req := httptest.NewRequest(http.MethodPost, "/v1/chat/completions", bytes.NewReader(invalidData))
req.Header.Set("Content-Type", "application/json")
req.Header.Set("Content-Encoding", "zstd")

for k := range sensitiveHeaders {
require.Empty(t, got.Values(k), "sensitiveHeaders %q should not be merged", k)
}
_, err := ReadHTTPRequest(req)
require.Error(t, err)
assert.Contains(t, err.Error(), "failed to decode zstd compressed body")
}

for k := range libManagedHeaders {
require.Empty(t, got.Values(k), "libManagedHeaders %q should not be merged", k)
}
func TestReadHTTPRequest_EmptyBody(t *testing.T) {
req := httptest.NewRequest(http.MethodPost, "/v1/chat/completions", nil)
req.Header.Set("Content-Type", "application/json")

require.Equal(t, "keep-me", got.Get("X-Custom"), "non-blocked header should be merged")
got, err := ReadHTTPRequest(req)
require.NoError(t, err)
assert.Empty(t, got.Body)
}

// TestMaskSensitiveHeaders_MasksAllHardcodedHeaders verifies every header in
// sensitiveHeaders is masked to "******" by MaskSensitiveHeaders.
func TestMaskSensitiveHeaders_MasksAllHardcodedHeaders(t *testing.T) {
headers := make(http.Header)
for k := range sensitiveHeaders {
headers.Set(k, "secret-value")
}
func TestReadHTTPRequest_EmptyBodyWithContentEncoding(t *testing.T) {
req := httptest.NewRequest(http.MethodPost, "/v1/chat/completions", nil)
req.Header.Set("Content-Type", "application/json")
req.Header.Set("Content-Encoding", "zstd")

headers.Set("X-Custom", "visible")
got, err := ReadHTTPRequest(req)
require.NoError(t, err)
assert.Empty(t, got.Body)
}

got := MaskSensitiveHeaders(headers)
func TestDecodeRequestBody_NoEncoding(t *testing.T) {
body := []byte(`{"test":"data"}`)
headers := http.Header{}

for k := range sensitiveHeaders {
require.Equal(t, []string{"******"}, got.Values(k),
"sensitiveHeaders %q should be masked", k)
}
got, err := decodeRequestBody(body, headers)
require.NoError(t, err)
assert.Equal(t, body, got)
}

func TestDecodeRequestBody_IdentityEncoding(t *testing.T) {
body := []byte(`{"test":"data"}`)
headers := http.Header{}
headers.Set("Content-Encoding", "identity")

require.Equal(t, []string{"visible"}, got.Values("X-Custom"))
got, err := decodeRequestBody(body, headers)
require.NoError(t, err)
assert.Equal(t, body, got)
}

func TestRegisterAppendHeaders(t *testing.T) {
RegisterMergeWithAppendHeaders("X-New-Append")
func TestDecodeRequestBody_GzipEncoding(t *testing.T) {
originalBody := []byte(`{"test":"data"}`)

dest := http.Header{"X-New-Append": []string{"old"}}
src := http.Header{"X-New-Append": []string{"new"}}
var buf bytes.Buffer
writer := gzip.NewWriter(&buf)
_, err := writer.Write(originalBody)
require.NoError(t, err)
writer.Close()

got := MergeHTTPHeaders(dest, src)
require.Equal(t, []string{"old", "new"}, got["X-New-Append"])
headers := http.Header{}
headers.Set("Content-Encoding", "gzip")
headers.Set("Content-Length", "100")

got, err := decodeRequestBody(buf.Bytes(), headers)
require.NoError(t, err)
assert.Equal(t, originalBody, got)
assert.Equal(t, "", headers.Get("Content-Encoding"))
assert.Equal(t, "", headers.Get("Content-Length"))
}

func TestMergeHTTPQuery(t *testing.T) {
tests := []struct {
name string
dest url.Values
src url.Values
want url.Values
}{
{
name: "should merge new query parameters",
dest: url.Values{"q": []string{"golang"}},
src: url.Values{"page": []string{"1"}},
want: url.Values{"q": []string{"golang"}, "page": []string{"1"}},
},
{
name: "should not overwrite existing query parameters",
dest: url.Values{"q": []string{"golang"}},
src: url.Values{"q": []string{"java"}},
want: url.Values{"q": []string{"golang"}},
},
{
name: "should handle empty src",
dest: url.Values{"q": []string{"golang"}},
src: nil,
want: url.Values{"q": []string{"golang"}},
},
{
name: "should handle empty dest",
dest: nil,
src: url.Values{"page": []string{"1"}},
want: url.Values{"page": []string{"1"}},
},
}
func TestDecodeRequestBody_DeflateEncoding(t *testing.T) {
originalBody := []byte(`{"test":"data"}`)

for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got := MergeHTTPQuery(tt.dest, tt.src)
require.Equal(t, tt.want, got)
})
}
var buf bytes.Buffer
writer, err := flate.NewWriter(&buf, flate.DefaultCompression)
require.NoError(t, err)
_, err = writer.Write(originalBody)
require.NoError(t, err)
writer.Close()

headers := http.Header{}
headers.Set("Content-Encoding", "deflate")

got, err := decodeRequestBody(buf.Bytes(), headers)
require.NoError(t, err)
assert.Equal(t, originalBody, got)
assert.Equal(t, "", headers.Get("Content-Encoding"))
}

func TestMergeInboundRequest(t *testing.T) {
t.Run("should merge both headers and query", func(t *testing.T) {
dest := &Request{
Headers: http.Header{"Content-Type": []string{"application/json"}},
Query: url.Values{"q": []string{"old"}},
}
src := &Request{
Headers: http.Header{"User-Agent": []string{"Test"}},
Query: url.Values{"page": []string{"1"}},
}

got := MergeInboundRequest(dest, src)
require.Equal(t, "application/json", got.Headers.Get("Content-Type"))
require.Equal(t, "Test", got.Headers.Get("User-Agent"))
require.Equal(t, "old", got.Query.Get("q"))
require.Equal(t, "1", got.Query.Get("page"))
})

t.Run("should block Cloudflare headers by prefix", func(t *testing.T) {
dest := &Request{
Headers: http.Header{"Content-Type": []string{"application/json"}},
Query: url.Values{},
}
src := &Request{
Headers: http.Header{
"Cf-Ray": []string{"abc123"},
"Cf-Connecting-Ip": []string{"1.2.3.4"},
"Cf-Ipcountry": []string{"US"},
"Cf-Visitor": []string{`{"scheme":"https"}`},
"Cdn-Loop": []string{"cloudflare; loops=1"},
"User-Agent": []string{"Test/1.0"},
},
Query: url.Values{},
}

got := MergeInboundRequest(dest, src)
require.Empty(t, got.Headers.Get("Cf-Ray"))
require.Empty(t, got.Headers.Get("Cf-Connecting-Ip"))
require.Empty(t, got.Headers.Get("Cf-Ipcountry"))
require.Empty(t, got.Headers.Get("Cf-Visitor"))
require.Empty(t, got.Headers.Get("Cdn-Loop"))
require.Equal(t, "Test/1.0", got.Headers.Get("User-Agent"))
})

t.Run("should block hop-by-hop headers from inbound request", func(t *testing.T) {
dest := &Request{
Headers: http.Header{"Content-Type": []string{"application/json"}},
Query: url.Values{},
}
src := &Request{
Headers: http.Header{
"Upgrade": []string{"h2c"},
"Keep-Alive": []string{"timeout=5"},
"Te": []string{"trailers"},
"Trailer": []string{"X-Trailer"},
"Proxy-Connection": []string{"keep-alive"},
"User-Agent": []string{"Test/1.0"},
},
Query: url.Values{},
}

got := MergeInboundRequest(dest, src)
require.Empty(t, got.Headers.Get("Upgrade"))
require.Empty(t, got.Headers.Get("Keep-Alive"))
require.Empty(t, got.Headers.Get("Te"))
require.Empty(t, got.Headers.Get("Trailer"))
require.Empty(t, got.Headers.Get("Proxy-Connection"))
require.Equal(t, "Test/1.0", got.Headers.Get("User-Agent"))
})

t.Run("should return dest if src is nil", func(t *testing.T) {
dest := &Request{Headers: http.Header{"X-Test": []string{"val"}}}
got := MergeInboundRequest(dest, nil)
require.Equal(t, dest, got)
})
func TestDecodeRequestBody_ZstdEncoding(t *testing.T) {
originalBody := []byte(`{"test":"data"}`)

encoder, err := zstd.NewWriter(nil)
require.NoError(t, err)
compressedBody := encoder.EncodeAll(originalBody, nil)
encoder.Close()

headers := http.Header{}
headers.Set("Content-Encoding", "zstd")

got, err := decodeRequestBody(compressedBody, headers)
require.NoError(t, err)
assert.Equal(t, originalBody, got)
assert.Equal(t, "", headers.Get("Content-Encoding"))
}

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

critical

It appears that all existing tests in this file (such as TestIsHTTPStatusCodeRetryable, TestMergeHTTPHeaders, TestMergeHTTPQuery, and TestMergeInboundRequest) have been deleted and replaced by the new tests. Please ensure that the new tests are appended to the file instead of overwriting the existing ones to maintain test coverage.

Comment thread llm/httpclient/utils.go
}
defer reader.Close()

decoded, err := io.ReadAll(reader)

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

security-high high

Using io.ReadAll on a decompression reader without a size limit is vulnerable to memory exhaustion attacks (Zip Bombs). A small malicious payload could expand to a size that exceeds available memory. Consider using io.LimitReader to restrict the maximum decompressed size. This risk also applies to the deflate and zstd implementations.

Comment thread frontend/pnpm-lock.yaml
Comment on lines +5362 to +5364
uuid@14.0.0:
resolution: {integrity: sha512-Qo+uWgilfSmAhXCMav1uYFynlQO7fMFiMVZsQqZRMIXp0O7rR7qjkj+cPvBHLgBqi960QCoo/PH2/6ZtVqKvrg==}
hasBin: true

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

security-high high

The uuid package version 14.0.0 appears to be non-existent on the public npm registry (current version is 11.x). This could indicate a corrupted lockfile or a potential security risk. Please verify this dependency change.

Comment thread llm/httpclient/utils.go Outdated
Comment on lines +79 to +80
reader := flate.NewReader(bytes.NewReader(body))
defer reader.Close()

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

medium

For HTTP Content-Encoding, the deflate value typically refers to the zlib format (RFC 1950). Using compress/flate (which implements raw RFC 1951 deflate) will fail for standard-compliant clients that send zlib-wrapped data. It is recommended to use compress/zlib instead.

Comment thread llm/go.mod
github.com/google/jsonschema-go v0.3.1-0.20251120200837-98a387e3b975
github.com/google/uuid v1.6.0
github.com/kaptinlin/jsonrepair v0.2.4
github.com/klauspost/compress v1.17.9

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

medium

The version of github.com/klauspost/compress used here (v1.17.9) is inconsistent with the version used in the root go.mod (v1.18.0). It is recommended to keep dependency versions synchronized across the project.

@greptile-apps

greptile-apps Bot commented May 21, 2026

Copy link
Copy Markdown
Contributor

Greptile Summary

This PR adds transparent auto-decompression of inbound request bodies for gzip/x-gzip, deflate, and zstd Content-Encoding values, stripping the encoding headers after decoding so upstream LLM providers receive a plain JSON body. The deflate path correctly attempts zlib-wrapped decoding first and falls back to raw DEFLATE when the header check fails, addressing a known RFC ambiguity.

  • Decompression logic (llm/httpclient/utils.go): new decodeRequestBody switch and decodeZlibOrFlate helper; the zlib\u2192raw DEFLATE fallback only fires when zlib.NewReader rejects the header, leaving a gap where raw DEFLATE streams whose first two bytes accidentally satisfy the zlib checksum are mis-detected as zlib and then fail to decode.
  • Tests (llm/httpclient/utils_test.go): thorough coverage of all three encodings, identity, empty body, error cases, and case-insensitive header handling; no test for the edge-case raw DEFLATE / coincidental zlib header collision.
  • Dependencies: adds github.com/klauspost/compress v1.18.0 for zstd support; lock files updated across go.mod/go.sum and frontend pnpm-lock.yaml (frontend lock change appears to be an unrelated dependency cleanup).

Confidence Score: 4/5

Safe to merge with the zlib fallback gap fixed; the bug only manifests for raw DEFLATE bodies whose first two bytes coincidentally satisfy the zlib checksum, but when it does trigger the request silently fails.

The decompression logic is correct for the vast majority of real-world inputs, but decodeZlibOrFlate stops at the first io.ReadAll error after a successful zlib.NewReader call without retrying as raw DEFLATE. A raw DEFLATE stream with a coincidental zlib-like header will be rejected rather than decoded successfully.

llm/httpclient/utils.go — specifically decodeZlibOrFlate

Important Files Changed

Filename Overview
llm/httpclient/utils.go Adds decodeRequestBody and decodeZlibOrFlate helpers for transparent gzip/deflate/zstd decompression of incoming request bodies; incomplete zlib→raw DEFLATE fallback path can incorrectly reject valid raw DEFLATE streams
llm/httpclient/utils_test.go Good coverage of the happy paths and error cases for all three encodings; missing a test for raw DEFLATE whose first two bytes happen to satisfy the zlib CMF+FLG checksum
llm/go.mod Adds github.com/klauspost/compress v1.18.0 for zstd support

Flowchart

%%{init: {'theme': 'neutral'}}%%
flowchart TD
    A[Incoming HTTP Request] --> B[io.ReadAll body]
    B --> C{len body > 0?}
    C -- No --> G[req.Body = nil]
    C -- Yes --> D{Content-Encoding header?}
    D -- None / identity --> E[return body as-is]
    D -- gzip / x-gzip --> F1[gzip.NewReader]
    F1 -- error --> ERR[return error]
    F1 -- ok --> F2[io.ReadAll]
    F2 -- error --> ERR
    F2 -- ok --> HDR1[Del Content-Encoding Del Content-Length]
    HDR1 --> G
    D -- deflate --> Z1[zlib.NewReader]
    Z1 -- error --> Z2[flate.NewReader io.ReadAll]
    Z1 -- ok --> Z3[io.ReadAll]
    Z3 -- error --> ERR2[return error no fallback to flate]
    Z3 -- ok --> HDR2[Del Content-Encoding Del Content-Length]
    Z2 -- error --> ERR
    Z2 -- ok --> HDR2
    HDR2 --> G
    D -- zstd --> S1[zstd decoder.DecodeAll]
    S1 -- error --> ERR
    S1 -- ok --> HDR3[Del Content-Encoding Del Content-Length]
    HDR3 --> G
    D -- other --> ERR3[unsupported content encoding error]
    G --> H[Return Request]
Loading

Reviews (2): Last reviewed commit: "feat: auto decompress request body, clos..." | Re-trigger Greptile

Comment thread llm/httpclient/utils.go
Comment thread llm/httpclient/utils.go
@looplj looplj merged commit b3abfc7 into unstable May 21, 2026
4 checks passed
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment

Labels

None yet

Projects

None yet

Development

Successfully merging this pull request may close these issues.

1 participant