Skip to content

Commit 040eba1

Browse files
vincentkocvincentkoc
committed
refactor: share bounded response reader
1 parent 18d2bc4 commit 040eba1

4 files changed

Lines changed: 46 additions & 141 deletions

File tree

‎scripts/e2e/telegram-bot-api.ts‎

Lines changed: 8 additions & 41 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,5 @@
1+
import { readBoundedResponseText } from "../lib/bounded-response.ts";
2+
13
type JsonObject = Record<string, unknown>;
24

35
type TelegramBotApiOptions = {
@@ -32,46 +34,6 @@ function taggedError(message: string, code: string) {
3234
return Object.assign(new Error(message), { code });
3335
}
3436

35-
async function readBoundedResponseText(
36-
response: Response,
37-
label: string,
38-
byteLimit: number,
39-
timeoutPromise: Promise<never>,
40-
) {
41-
const contentLength = response.headers.get("content-length");
42-
if (contentLength) {
43-
const parsedLength = Number(contentLength);
44-
if (Number.isSafeInteger(parsedLength) && parsedLength > byteLimit) {
45-
await response.body?.cancel().catch(() => {});
46-
throw taggedError(`${label} response body exceeded ${byteLimit} bytes`, "ETOOBIG");
47-
}
48-
}
49-
if (!response.body) {
50-
return "";
51-
}
52-
53-
const reader = response.body.getReader();
54-
const decoder = new TextDecoder();
55-
let byteCount = 0;
56-
let text = "";
57-
try {
58-
while (true) {
59-
const { done, value } = await Promise.race([reader.read(), timeoutPromise]);
60-
if (done) {
61-
return text + decoder.decode();
62-
}
63-
byteCount += value.byteLength;
64-
if (byteCount > byteLimit) {
65-
await reader.cancel().catch(() => {});
66-
throw taggedError(`${label} response body exceeded ${byteLimit} bytes`, "ETOOBIG");
67-
}
68-
text += decoder.decode(value, { stream: true });
69-
}
70-
} finally {
71-
reader.releaseLock();
72-
}
73-
}
74-
7537
function parseJsonPayload(rawPayload: string, label: string) {
7638
try {
7739
return JSON.parse(rawPayload) as JsonObject;
@@ -111,7 +73,12 @@ export async function telegramBotApi(
11173
}),
11274
timeoutPromise,
11375
]);
114-
const rawPayload = await readBoundedResponseText(response, label, maxBodyBytes, timeoutPromise);
76+
const rawPayload = await readBoundedResponseText(response, label, maxBodyBytes, {
77+
createTooLargeError(message) {
78+
return taggedError(message, "ETOOBIG");
79+
},
80+
timeoutPromise,
81+
});
11582
const payload = parseJsonPayload(rawPayload, label);
11683
if (!response.ok || payload.ok !== true) {
11784
throw new Error(

‎scripts/e2e/telegram-user-credential-io.ts‎

Lines changed: 4 additions & 45 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
import { spawn } from "node:child_process";
2+
import { readBoundedResponseText } from "../lib/bounded-response.ts";
23

34
export type JsonObject = Record<string, unknown>;
45

@@ -138,46 +139,6 @@ export function runCommand(
138139
});
139140
}
140141

141-
async function readBoundedResponseText(
142-
response: Response,
143-
label: string,
144-
byteLimit: number,
145-
timeoutPromise: Promise<never>,
146-
) {
147-
const contentLength = response.headers.get("content-length");
148-
if (contentLength) {
149-
const parsedLength = Number(contentLength);
150-
if (Number.isSafeInteger(parsedLength) && parsedLength > byteLimit) {
151-
await response.body?.cancel().catch(() => {});
152-
throw bodyTooLargeError(`${label} response body exceeded ${byteLimit} bytes`);
153-
}
154-
}
155-
if (!response.body) {
156-
return "";
157-
}
158-
159-
const reader = response.body.getReader();
160-
const decoder = new TextDecoder();
161-
let byteCount = 0;
162-
let text = "";
163-
try {
164-
while (true) {
165-
const { done, value } = await Promise.race([reader.read(), timeoutPromise]);
166-
if (done) {
167-
return text + decoder.decode();
168-
}
169-
byteCount += value.byteLength;
170-
if (byteCount > byteLimit) {
171-
await reader.cancel().catch(() => {});
172-
throw bodyTooLargeError(`${label} response body exceeded ${byteLimit} bytes`);
173-
}
174-
text += decoder.decode(value, { stream: true });
175-
}
176-
} finally {
177-
reader.releaseLock();
178-
}
179-
}
180-
181142
export async function fetchJsonWithTimeout(params: FetchJsonParams) {
182143
const timeoutMs = Math.max(1, params.timeoutMs);
183144
const maxBodyBytes = resolveFetchBodyLimit(params.maxBodyBytes);
@@ -200,12 +161,10 @@ export async function fetchJsonWithTimeout(params: FetchJsonParams) {
200161
}),
201162
timeoutPromise,
202163
]);
203-
const rawPayload = await readBoundedResponseText(
204-
response,
205-
params.label,
206-
maxBodyBytes,
164+
const rawPayload = await readBoundedResponseText(response, params.label, maxBodyBytes, {
165+
createTooLargeError: bodyTooLargeError,
207166
timeoutPromise,
208-
);
167+
});
209168
const payload = JSON.parse(rawPayload) as JsonObject;
210169
return { payload, response };
211170
} finally {

‎scripts/lib/bounded-response.ts‎

Lines changed: 24 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -1,11 +1,27 @@
1+
type BoundedResponseTextOptions = {
2+
createTooLargeError?: (message: string) => Error;
3+
formatTooLargeMessage?: (label: string, maxBytes: number) => string;
4+
timeoutPromise?: Promise<never>;
5+
};
6+
7+
const defaultTooLargeMessage = (label: string, maxBytes: number) =>
8+
`${label} response body exceeded ${maxBytes} bytes`;
9+
10+
const defaultTooLargeError = (message: string) => new Error(`${message}.`);
11+
112
export async function readBoundedResponseText(
213
response: Response,
314
label: string,
415
maxBytes: number,
16+
options: BoundedResponseTextOptions = {},
517
): Promise<string> {
6-
const contentLength = Number.parseInt(response.headers.get("content-length") ?? "", 10);
7-
if (Number.isFinite(contentLength) && contentLength > maxBytes) {
8-
throw new Error(`${label} response body exceeded ${maxBytes} bytes.`);
18+
const formatTooLargeMessage = options.formatTooLargeMessage ?? defaultTooLargeMessage;
19+
const createTooLargeError = options.createTooLargeError ?? defaultTooLargeError;
20+
const tooLargeError = () => createTooLargeError(formatTooLargeMessage(label, maxBytes));
21+
const contentLength = Number(response.headers.get("content-length") ?? "");
22+
if (Number.isSafeInteger(contentLength) && contentLength > maxBytes) {
23+
await response.body?.cancel().catch(() => undefined);
24+
throw tooLargeError();
925
}
1026

1127
if (!response.body) {
@@ -16,11 +32,12 @@ export async function readBoundedResponseText(
1632
const decoder = new TextDecoder();
1733
const chunks: string[] = [];
1834
let totalBytes = 0;
19-
let canceled = false;
2035

2136
try {
2237
for (;;) {
23-
const { done, value } = await reader.read();
38+
const { done, value } = await (options.timeoutPromise
39+
? Promise.race([reader.read(), options.timeoutPromise])
40+
: reader.read());
2441
if (done) {
2542
const tail = decoder.decode();
2643
if (tail) {
@@ -31,16 +48,13 @@ export async function readBoundedResponseText(
3148

3249
totalBytes += value.byteLength;
3350
if (totalBytes > maxBytes) {
34-
canceled = true;
3551
await reader.cancel().catch(() => undefined);
36-
throw new Error(`${label} response body exceeded ${maxBytes} bytes.`);
52+
throw tooLargeError();
3753
}
3854
chunks.push(decoder.decode(value, { stream: true }));
3955
}
4056
} finally {
41-
if (!canceled) {
42-
reader.releaseLock();
43-
}
57+
reader.releaseLock();
4458
}
4559

4660
return chunks.join("");

‎scripts/tool-search-gateway-e2e.ts‎

Lines changed: 10 additions & 45 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@ import { stageQaMockAuthProfiles } from "../extensions/qa-lab/src/providers/shar
99
import { buildQaGatewayConfig } from "../extensions/qa-lab/src/qa-gateway-config.js";
1010
import { resetConfigRuntimeState } from "../src/config/config.js";
1111
import { startGatewayServer } from "../src/gateway/server.js";
12+
import { readBoundedResponseText } from "./lib/bounded-response.ts";
1213

1314
type Lane = "normal" | "code";
1415

@@ -56,50 +57,8 @@ function timeoutError(message: string) {
5657
return Object.assign(new Error(message), { code: "ETIMEDOUT" });
5758
}
5859

59-
function bodyTooLargeError(url: string, byteLimit: number) {
60-
return Object.assign(new Error(`HTTP response from ${url} exceeded ${byteLimit} bytes`), {
61-
code: "ETOOBIG",
62-
});
63-
}
64-
65-
async function readBoundedResponseText(
66-
response: Response,
67-
url: string,
68-
byteLimit: number,
69-
timeoutPromise: Promise<never>,
70-
) {
71-
const contentLength = response.headers.get("content-length");
72-
if (contentLength) {
73-
const parsedLength = Number(contentLength);
74-
if (Number.isSafeInteger(parsedLength) && parsedLength > byteLimit) {
75-
await response.body?.cancel().catch(() => {});
76-
throw bodyTooLargeError(url, byteLimit);
77-
}
78-
}
79-
if (!response.body) {
80-
return "";
81-
}
82-
83-
const reader = response.body.getReader();
84-
const decoder = new TextDecoder();
85-
let byteCount = 0;
86-
let text = "";
87-
try {
88-
while (true) {
89-
const { done, value } = await Promise.race([reader.read(), timeoutPromise]);
90-
if (done) {
91-
return text + decoder.decode();
92-
}
93-
byteCount += value.byteLength;
94-
if (byteCount > byteLimit) {
95-
await reader.cancel().catch(() => {});
96-
throw bodyTooLargeError(url, byteLimit);
97-
}
98-
text += decoder.decode(value, { stream: true });
99-
}
100-
} finally {
101-
reader.releaseLock();
102-
}
60+
function bodyTooLargeErrorMessage(url: string, byteLimit: number) {
61+
return `HTTP response from ${url} exceeded ${byteLimit} bytes`;
10362
}
10463

10564
async function freePort(): Promise<number> {
@@ -208,7 +167,13 @@ export async function fetchJson(
208167
}),
209168
timeoutPromise,
210169
]);
211-
text = await readBoundedResponseText(response, url, maxBodyBytes, timeoutPromise);
170+
text = await readBoundedResponseText(response, url, maxBodyBytes, {
171+
createTooLargeError(message) {
172+
return Object.assign(new Error(message), { code: "ETOOBIG" });
173+
},
174+
formatTooLargeMessage: bodyTooLargeErrorMessage,
175+
timeoutPromise,
176+
});
212177
} finally {
213178
if (timeout) {
214179
clearTimeout(timeout);

0 commit comments

Comments
 (0)