Skip to content

Commit 28550c3

Browse files
vincentkocvincentkoc
committed
fix(e2e): harden Parallels host timeouts
1 parent 3e91c68 commit 28550c3

2 files changed

Lines changed: 458 additions & 17 deletions

File tree

‎scripts/e2e/parallels/host-command.ts‎

Lines changed: 310 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
import { spawn, spawnSync, type SpawnOptions } from "node:child_process";
1+
import { spawn, spawnSync, type SpawnOptions, type SpawnSyncReturns } from "node:child_process";
22
import { writeFile } from "node:fs/promises";
33
import path from "node:path";
44
import { fileURLToPath } from "node:url";
@@ -9,6 +9,13 @@ import type { CommandResult, RunOptions } from "./types.ts";
99

1010
export const repoRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "../../..");
1111

12+
const HOST_COMMAND_MAX_BUFFER_BYTES = 50 * 1024 * 1024;
13+
const HOST_COMMAND_WRAPPER_EXTRA_BUFFER_BYTES = 1024 * 1024;
14+
const HOST_COMMAND_WRAPPER_BACKSTOP_MS = 5_000;
15+
const HOST_COMMAND_CHILD_PID_PREFIX = "__OPENCLAW_HOST_COMMAND_CHILD_PID__";
16+
const HOST_COMMAND_SPAWN_ERROR_PREFIX = "__OPENCLAW_HOST_COMMAND_SPAWN_ERROR__";
17+
const HOST_COMMAND_TIMEOUT_PREFIX = "__OPENCLAW_HOST_COMMAND_TIMEOUT__";
18+
1219
type HostCommandInvocation = {
1320
args: string[];
1421
command: string;
@@ -47,6 +54,161 @@ export function die(message: string): never {
4754
process.exit(1);
4855
}
4956

57+
function signalHostCommandProcess(pid: number | undefined, signal: NodeJS.Signals): void {
58+
if (!pid) {
59+
return;
60+
}
61+
try {
62+
if (process.platform === "win32") {
63+
process.kill(pid, signal);
64+
} else {
65+
process.kill(-pid, signal);
66+
}
67+
} catch (error) {
68+
const code = (error as NodeJS.ErrnoException).code;
69+
if (code !== "ESRCH") {
70+
warn(
71+
`failed to send ${signal} to timed host command process ${pid}: ${
72+
code ?? String(error)
73+
}`,
74+
);
75+
}
76+
}
77+
}
78+
79+
const POSIX_TIMEOUT_WRAPPER = String.raw`
80+
const { spawn } = require("node:child_process");
81+
const { readFileSync, writeSync } = require("node:fs");
82+
83+
const payload = JSON.parse(readFileSync(0, "utf8"));
84+
const child = spawn(payload.command, payload.args, {
85+
cwd: payload.cwd,
86+
detached: true,
87+
env: payload.env,
88+
shell: payload.shell,
89+
stdio: ["pipe", "pipe", "pipe"],
90+
});
91+
writeSync(
92+
3,
93+
${JSON.stringify(HOST_COMMAND_CHILD_PID_PREFIX)} + JSON.stringify({
94+
pid: child.pid || null,
95+
}) + "\n",
96+
);
97+
98+
let timedOut = false;
99+
let killTimer;
100+
let outputExceeded = false;
101+
let stderrBytes = 0;
102+
let stdoutBytes = 0;
103+
104+
function writeAllSync(fd, chunk) {
105+
let offset = 0;
106+
while (offset < chunk.byteLength) {
107+
offset += writeSync(fd, chunk, offset, chunk.byteLength - offset);
108+
}
109+
}
110+
111+
function signalGroup(signal) {
112+
if (!child.pid) {
113+
return;
114+
}
115+
try {
116+
process.kill(-child.pid, signal);
117+
} catch (error) {
118+
if (error && error.code !== "ESRCH") {
119+
process.stderr.write("failed to send " + signal + " to timed host command process " + child.pid + ": " + (error.code || String(error)) + "\n");
120+
}
121+
}
122+
}
123+
124+
function forwardBounded(stream, chunk) {
125+
const currentBytes = stream === "stdout" ? stdoutBytes : stderrBytes;
126+
const nextBytes = currentBytes + chunk.byteLength;
127+
const limit = payload.maxBufferBytes;
128+
if (stream === "stdout") {
129+
stdoutBytes = nextBytes;
130+
} else {
131+
stderrBytes = nextBytes;
132+
}
133+
if (outputExceeded) {
134+
return;
135+
}
136+
if (nextBytes <= limit) {
137+
writeAllSync(stream === "stdout" ? 1 : 2, chunk);
138+
return;
139+
}
140+
outputExceeded = true;
141+
const allowedBytes = Math.max(0, limit - currentBytes);
142+
if (allowedBytes > 0) {
143+
writeAllSync(stream === "stdout" ? 1 : 2, chunk.subarray(0, allowedBytes));
144+
}
145+
writeAllSync(
146+
2,
147+
Buffer.from("host command output exceeded " + limit + " bytes; terminating process group\n"),
148+
);
149+
signalGroup("SIGKILL");
150+
}
151+
152+
for (const signal of ["SIGHUP", "SIGINT", "SIGTERM"]) {
153+
process.once(signal, () => {
154+
signalGroup(signal);
155+
process.kill(process.pid, signal);
156+
});
157+
}
158+
159+
const timeout = setTimeout(() => {
160+
timedOut = true;
161+
signalGroup("SIGTERM");
162+
killTimer = setTimeout(() => signalGroup("SIGKILL"), 100);
163+
killTimer.unref();
164+
}, payload.timeoutMs);
165+
timeout.unref();
166+
167+
child.stdout.on("data", (chunk) => forwardBounded("stdout", chunk));
168+
child.stderr.on("data", (chunk) => forwardBounded("stderr", chunk));
169+
child.stdin.on("error", (error) => {
170+
if (error && error.code !== "EPIPE" && error.code !== "ECONNRESET") {
171+
writeAllSync(2, Buffer.from("host command stdin write failed: " + (error.code || String(error)) + "\n"));
172+
}
173+
});
174+
child.on("error", (error) => {
175+
clearTimeout(timeout);
176+
if (killTimer) {
177+
clearTimeout(killTimer);
178+
}
179+
writeSync(
180+
3,
181+
${JSON.stringify(HOST_COMMAND_SPAWN_ERROR_PREFIX)} + JSON.stringify({
182+
code: error.code || null,
183+
message: error.message,
184+
}) + "\n",
185+
);
186+
process.stderr.write(error.message + "\n");
187+
process.exit(127);
188+
});
189+
child.on("close", (code, signal) => {
190+
clearTimeout(timeout);
191+
if (killTimer) {
192+
clearTimeout(killTimer);
193+
}
194+
if (timedOut) {
195+
signalGroup("SIGKILL");
196+
writeSync(3, ${JSON.stringify(HOST_COMMAND_TIMEOUT_PREFIX)} + "{}\n");
197+
process.exit(124);
198+
}
199+
if (outputExceeded) {
200+
process.exit(1);
201+
}
202+
process.exit(code ?? (signal ? 128 : 1));
203+
});
204+
205+
if (payload.input != null) {
206+
child.stdin.end(payload.input);
207+
} else {
208+
child.stdin.end();
209+
}
210+
`;
211+
50212
export function shellQuote(value: string): string {
51213
return `'${value.replaceAll("'", `'"'"'`)}'`;
52214
}
@@ -116,20 +278,46 @@ export function resolveHostCommandInvocation(
116278
export function run(command: string, args: string[], options: RunOptions = {}): CommandResult {
117279
const env = { ...process.env, ...options.env };
118280
const invocation = resolveHostCommandInvocation(command, args, { env });
119-
const result = spawnSync(invocation.command, invocation.args, {
120-
cwd: options.cwd ?? repoRoot,
121-
encoding: "utf8",
122-
env: invocation.env ?? env,
123-
input: options.input,
124-
killSignal: "SIGKILL",
125-
maxBuffer: 50 * 1024 * 1024,
126-
stdio: options.quiet ? ["pipe", "pipe", "pipe"] : ["pipe", "pipe", "pipe"],
127-
shell: invocation.shell,
128-
timeout: options.timeoutMs,
129-
windowsVerbatimArguments: invocation.windowsVerbatimArguments,
130-
});
281+
const usesPosixTimedWrapper = process.platform !== "win32" && options.timeoutMs !== undefined;
282+
const result =
283+
usesPosixTimedWrapper
284+
? runPosixTimedCommandSync(invocation, env, options)
285+
: spawnSync(invocation.command, invocation.args, {
286+
cwd: options.cwd ?? repoRoot,
287+
encoding: "utf8",
288+
env: invocation.env ?? env,
289+
input: options.input,
290+
killSignal: "SIGKILL",
291+
maxBuffer: HOST_COMMAND_MAX_BUFFER_BYTES,
292+
stdio: options.quiet ? ["pipe", "pipe", "pipe"] : ["pipe", "pipe", "pipe"],
293+
shell: invocation.shell,
294+
timeout: options.timeoutMs,
295+
windowsVerbatimArguments: invocation.windowsVerbatimArguments,
296+
});
131297

132-
const timedOut = (result.error as NodeJS.ErrnoException | undefined)?.code === "ETIMEDOUT";
298+
let wrapperTimedOut = false;
299+
if (usesPosixTimedWrapper) {
300+
const wrapperControl = typeof result.output[3] === "string" ? result.output[3] : "";
301+
const outerWrapperTimedOut =
302+
(result.error as NodeJS.ErrnoException | undefined)?.code === "ETIMEDOUT";
303+
if (outerWrapperTimedOut) {
304+
signalHostCommandProcess(parsePosixTimedWrapperChildPid(wrapperControl), "SIGKILL");
305+
}
306+
wrapperTimedOut = outerWrapperTimedOut || hasPosixTimedWrapperTimeout(wrapperControl);
307+
const spawnError = parsePosixTimedWrapperSpawnError(wrapperControl);
308+
if (spawnError) {
309+
throw spawnError;
310+
}
311+
}
312+
const timedOut =
313+
wrapperTimedOut || (result.error as NodeJS.ErrnoException | undefined)?.code === "ETIMEDOUT";
314+
if (wrapperTimedOut && options.check !== false) {
315+
const error = new Error(
316+
`${command} ${args.join(" ")} timed out after ${options.timeoutMs}ms`,
317+
) as NodeJS.ErrnoException;
318+
error.code = "ETIMEDOUT";
319+
throw error;
320+
}
133321
if (result.error && !(timedOut && options.check === false)) {
134322
throw result.error;
135323
}
@@ -152,6 +340,76 @@ export function run(command: string, args: string[], options: RunOptions = {}):
152340
return commandResult;
153341
}
154342

343+
function hasPosixTimedWrapperTimeout(controlOutput: string): boolean {
344+
return controlOutput.split("\n").some((entry) => entry.startsWith(HOST_COMMAND_TIMEOUT_PREFIX));
345+
}
346+
347+
function parsePosixTimedWrapperChildPid(controlOutput: string): number | undefined {
348+
const line = controlOutput
349+
.split("\n")
350+
.find((entry) => entry.startsWith(HOST_COMMAND_CHILD_PID_PREFIX));
351+
if (!line) {
352+
return undefined;
353+
}
354+
try {
355+
const parsed = JSON.parse(line.slice(HOST_COMMAND_CHILD_PID_PREFIX.length)) as {
356+
pid?: unknown;
357+
};
358+
return typeof parsed.pid === "number" ? parsed.pid : undefined;
359+
} catch {
360+
return undefined;
361+
}
362+
}
363+
364+
function parsePosixTimedWrapperSpawnError(stderr: string): NodeJS.ErrnoException | null {
365+
const line = stderr
366+
.split("\n")
367+
.find((entry) => entry.startsWith(HOST_COMMAND_SPAWN_ERROR_PREFIX));
368+
if (!line) {
369+
return null;
370+
}
371+
const raw = line.slice(HOST_COMMAND_SPAWN_ERROR_PREFIX.length);
372+
try {
373+
const parsed = JSON.parse(raw) as { code?: unknown; message?: unknown };
374+
const error = new Error(
375+
typeof parsed.message === "string" ? parsed.message : "host command spawn failed",
376+
) as NodeJS.ErrnoException;
377+
if (typeof parsed.code === "string") {
378+
error.code = parsed.code;
379+
}
380+
return error;
381+
} catch {
382+
return new Error("host command spawn failed") as NodeJS.ErrnoException;
383+
}
384+
}
385+
386+
function runPosixTimedCommandSync(
387+
invocation: HostCommandInvocation,
388+
env: NodeJS.ProcessEnv,
389+
options: RunOptions,
390+
): SpawnSyncReturns<string> {
391+
const payload = JSON.stringify({
392+
args: invocation.args,
393+
command: invocation.command,
394+
cwd: options.cwd ?? repoRoot,
395+
env: invocation.env ?? env,
396+
input: options.input,
397+
maxBufferBytes: HOST_COMMAND_MAX_BUFFER_BYTES,
398+
shell: invocation.shell,
399+
timeoutMs: options.timeoutMs,
400+
});
401+
return spawnSync(process.execPath, ["-e", POSIX_TIMEOUT_WRAPPER], {
402+
cwd: options.cwd ?? repoRoot,
403+
encoding: "utf8",
404+
env,
405+
input: payload,
406+
killSignal: "SIGKILL",
407+
maxBuffer: HOST_COMMAND_MAX_BUFFER_BYTES * 2 + HOST_COMMAND_WRAPPER_EXTRA_BUFFER_BYTES,
408+
stdio: ["pipe", "pipe", "pipe", "pipe"],
409+
timeout: (options.timeoutMs ?? 0) + HOST_COMMAND_WRAPPER_BACKSTOP_MS,
410+
});
411+
}
412+
155413
export function sh(script: string, options: RunOptions = {}): CommandResult {
156414
return run("bash", ["-lc", script], options);
157415
}
@@ -166,11 +424,31 @@ export async function runStreaming(
166424
const invocation = resolveHostCommandInvocation(command, args, { env });
167425
const child = spawn(invocation.command, invocation.args, {
168426
cwd: options.cwd ?? repoRoot,
427+
detached: process.platform !== "win32" && options.timeoutMs != null,
169428
env: invocation.env ?? env,
170429
shell: invocation.shell,
171430
stdio: ["pipe", "pipe", "pipe"],
172431
windowsVerbatimArguments: invocation.windowsVerbatimArguments,
173432
} satisfies SpawnOptions);
433+
const childPid = child.pid;
434+
const parentSignalHandlers = new Map<NodeJS.Signals, () => void>();
435+
const removeParentSignalHandlers = (): void => {
436+
for (const [signal, handler] of parentSignalHandlers) {
437+
process.off(signal, handler);
438+
}
439+
parentSignalHandlers.clear();
440+
};
441+
if (process.platform !== "win32" && options.timeoutMs != null) {
442+
for (const signal of ["SIGHUP", "SIGINT", "SIGTERM"] as const) {
443+
const handler = (): void => {
444+
signalHostCommandProcess(childPid, signal);
445+
removeParentSignalHandlers();
446+
process.kill(process.pid, signal);
447+
};
448+
parentSignalHandlers.set(signal, handler);
449+
process.once(signal, handler);
450+
}
451+
}
174452

175453
let log = "";
176454
const append = (chunk: Buffer): void => {
@@ -195,21 +473,36 @@ export async function runStreaming(
195473
}
196474

197475
let timedOut = false;
476+
let killTimer: NodeJS.Timeout | undefined;
198477
const timer =
199478
options.timeoutMs == null
200479
? undefined
201480
: setTimeout(() => {
202481
timedOut = true;
203-
child.kill("SIGTERM");
204-
setTimeout(() => child.kill("SIGKILL"), 2_000).unref();
482+
signalHostCommandProcess(childPid, "SIGTERM");
483+
killTimer = setTimeout(() => signalHostCommandProcess(childPid, "SIGKILL"), 2_000);
484+
killTimer.unref();
205485
}, options.timeoutMs);
206486

207-
child.on("error", reject);
487+
child.on("error", (error) => {
488+
if (killTimer) {
489+
clearTimeout(killTimer);
490+
}
491+
removeParentSignalHandlers();
492+
reject(error);
493+
});
208494
child.on("close", (code, signal) => {
209495
void (async () => {
210496
if (timer) {
211497
clearTimeout(timer);
212498
}
499+
if (killTimer) {
500+
clearTimeout(killTimer);
501+
}
502+
removeParentSignalHandlers();
503+
if (timedOut) {
504+
signalHostCommandProcess(childPid, "SIGKILL");
505+
}
213506
if (options.logPath) {
214507
await writeFile(options.logPath, log, "utf8");
215508
}

0 commit comments

Comments
 (0)