Skip to content

Commit 69a4061

Browse files
vincentkocvincentkoc
committed
fix(e2e): preserve docker cleanup failure artifacts
1 parent ffea7fa commit 69a4061

4 files changed

Lines changed: 312 additions & 46 deletions

File tree

‎scripts/docker-e2e-rerun.mjs‎

Lines changed: 51 additions & 37 deletions
Original file line numberDiff line numberDiff line change
@@ -161,6 +161,27 @@ function ghWorkflowCommand(lanes, ref, workflow, reuseInputs = {}) {
161161
return fields.join(" ");
162162
}
163163

164+
function failureName(failure) {
165+
return failure.name || failure.lane || "";
166+
}
167+
168+
function failedEntryFromRecord(failure, file, ref, workflow, reuseInputs) {
169+
const lane = failureName(failure);
170+
const targetable = failure.targetable !== false;
171+
return {
172+
ghWorkflowCommand: targetable
173+
? failure.ghWorkflowCommand || ghWorkflowCommand([lane], ref, workflow, reuseInputs)
174+
: "",
175+
lane,
176+
localRerunCommand: failure.rerunCommand,
177+
logFile: failure.logFile,
178+
reuseInputs,
179+
source: file,
180+
status: failure.status,
181+
targetable,
182+
};
183+
}
184+
164185
function detectRepo() {
165186
return run("gh", ["repo", "view", "--json", "nameWithOwner", "--jq", ".nameWithOwner"]).trim();
166187
}
@@ -183,31 +204,18 @@ function failedLaneEntriesFromJson(file, ref, workflow) {
183204
const source = path.basename(file);
184205
if (source === "failures.json" && Array.isArray(parsed.lanes)) {
185206
return parsed.lanes
186-
.filter((lane) => lane.name)
187-
.map((lane) => ({
188-
ghWorkflowCommand:
189-
lane.ghWorkflowCommand || ghWorkflowCommand([lane.name], ref, workflow, reuseInputs),
190-
lane: lane.name,
191-
localRerunCommand: lane.rerunCommand,
192-
logFile: lane.logFile,
193-
reuseInputs,
194-
source: file,
195-
status: lane.status,
196-
}));
207+
.filter((lane) => failureName(lane))
208+
.map((lane) => failedEntryFromRecord(lane, file, ref, workflow, reuseInputs));
197209
}
198210

199211
const lanes = Array.isArray(parsed.lanes) ? parsed.lanes : [];
200-
return lanes
201-
.filter((lane) => lane.status !== 0 && lane.name)
202-
.map((lane) => ({
203-
ghWorkflowCommand: ghWorkflowCommand([lane.name], ref, workflow, reuseInputs),
204-
lane: lane.name,
205-
localRerunCommand: lane.rerunCommand,
206-
logFile: lane.logFile,
207-
reuseInputs,
208-
source: file,
209-
status: lane.status,
210-
}));
212+
const failures =
213+
Array.isArray(parsed.failures) && parsed.failures.length > 0
214+
? parsed.failures
215+
: lanes.filter((lane) => lane.status !== 0);
216+
return failures
217+
.filter((lane) => failureName(lane))
218+
.map((lane) => failedEntryFromRecord(lane, file, ref, workflow, reuseInputs));
211219
}
212220

213221
function mergeByLane(entries) {
@@ -275,23 +283,29 @@ function printEntries(entries, ref, workflow, runValue) {
275283
console.log("No failed Docker E2E lanes found.");
276284
return;
277285
}
278-
console.log(`Failed lanes: ${entries.map((entry) => entry.lane).join(", ")}`);
279-
console.log("");
280-
console.log("Combined GitHub rerun:");
281-
console.log(
282-
ghWorkflowCommand(
283-
entries.map((entry) => entry.lane),
284-
ref,
285-
workflow,
286-
commonReuseInputs(entries),
287-
),
288-
);
289-
console.log("");
290-
console.log("Per-lane GitHub reruns:");
291-
for (const entry of entries) {
286+
const workflowEntries = entries.filter((entry) => entry.targetable !== false);
287+
console.log(`Failed Docker E2E entries: ${entries.map((entry) => entry.lane).join(", ")}`);
288+
if (workflowEntries.length > 0) {
289+
console.log("");
290+
console.log("Combined GitHub rerun:");
292291
console.log(
293-
`- ${entry.lane}: ${entry.ghWorkflowCommand || ghWorkflowCommand([entry.lane], ref, workflow)}`,
292+
ghWorkflowCommand(
293+
workflowEntries.map((entry) => entry.lane),
294+
ref,
295+
workflow,
296+
commonReuseInputs(workflowEntries),
297+
),
294298
);
299+
console.log("");
300+
console.log("Per-lane GitHub reruns:");
301+
for (const entry of workflowEntries) {
302+
console.log(
303+
`- ${entry.lane}: ${entry.ghWorkflowCommand || ghWorkflowCommand([entry.lane], ref, workflow)}`,
304+
);
305+
}
306+
} else {
307+
console.log("");
308+
console.log("No targetable failed Docker E2E lanes found.");
295309
}
296310
console.log("");
297311
console.log("Local rerun starting points:");

‎scripts/test-docker-all.mjs‎

Lines changed: 132 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -40,6 +40,7 @@ const DEFAULT_LANE_TIMEOUT_MS = 120 * 60 * 1000;
4040
const DEFAULT_LANE_START_STAGGER_MS = 2_000;
4141
const DEFAULT_STATUS_INTERVAL_MS = 30_000;
4242
const DEFAULT_PREFLIGHT_RUN_TIMEOUT_MS = 60_000;
43+
const CLEANUP_SMOKE_NAME = "cleanup-smoke";
4344
export const SHELL_CAPTURE_MAX_CHARS = 1024 * 1024;
4445
export const LOG_TAIL_MAX_BYTES = 1024 * 1024;
4546
const DEFAULT_TIMINGS_FILE = path.join(ROOT_DIR, ".artifacts/docker-tests/lane-timings.json");
@@ -456,8 +457,10 @@ async function writeFailureIndex(logDir, summary) {
456457
const failures = Array.isArray(summary.failures)
457458
? summary.failures
458459
: (summary.lanes ?? []).filter((lane) => lane.status !== 0);
460+
const workflowRerunFailures = failures.filter((failure) => failure.targetable !== false);
459461
const lanes = failures.map((failure) => ({
460-
ghWorkflowCommand: githubWorkflowRerunCommand([failure.name], ref),
462+
ghWorkflowCommand:
463+
failure.targetable === false ? undefined : githubWorkflowRerunCommand([failure.name], ref),
461464
image: failure.image,
462465
imageKind: failure.imageKind,
463466
lane: failure.name,
@@ -466,13 +469,14 @@ async function writeFailureIndex(logDir, summary) {
466469
noOutputTimedOut: failure.noOutputTimedOut,
467470
rerunCommand: failure.rerunCommand,
468471
status: failure.status,
472+
targetable: failure.targetable,
469473
timedOut: failure.timedOut,
470474
}));
471475
const failureIndex = {
472476
combinedGhWorkflowCommand:
473-
lanes.length > 0
477+
workflowRerunFailures.length > 0
474478
? githubWorkflowRerunCommand(
475-
lanes.map((lane) => lane.lane),
479+
workflowRerunFailures.map((failure) => failure.name),
476480
ref,
477481
)
478482
: undefined,
@@ -712,6 +716,85 @@ async function runForeground(label, command, env) {
712716
}
713717
}
714718

719+
async function recordCleanupSmokeFailure(error, baseEnv, logDir, command, startedAtMs) {
720+
const status = 1;
721+
const logFile = path.join(logDir, `${CLEANUP_SMOKE_NAME}.log`);
722+
const message = error instanceof Error ? error.message : String(error);
723+
await fs.promises.writeFile(
724+
logFile,
725+
[
726+
`==> [${CLEANUP_SMOKE_NAME}] command: ${command}`,
727+
`==> [${CLEANUP_SMOKE_NAME}] status: ${status}`,
728+
`==> [${CLEANUP_SMOKE_NAME}] error: ${message}`,
729+
]
730+
.filter(Boolean)
731+
.join("\n"),
732+
);
733+
return {
734+
command,
735+
attempts: [
736+
{
737+
attempt: 1,
738+
elapsedSeconds: phaseElapsedSeconds(startedAtMs),
739+
finishedAt: new Date().toISOString(),
740+
noOutputTimedOut: false,
741+
startedAt: new Date(startedAtMs).toISOString(),
742+
status,
743+
timedOut: false,
744+
},
745+
],
746+
elapsedSeconds: phaseElapsedSeconds(startedAtMs),
747+
finishedAt: new Date().toISOString(),
748+
image: baseEnv.OPENCLAW_DOCKER_E2E_IMAGE,
749+
logFile,
750+
name: CLEANUP_SMOKE_NAME,
751+
noOutputTimedOut: false,
752+
rerunCommand: command,
753+
startedAt: new Date(startedAtMs).toISOString(),
754+
status,
755+
targetable: false,
756+
timedOut: false,
757+
};
758+
}
759+
760+
async function runCleanupSmoke(baseEnv, logDir, command, startedAtMs) {
761+
const logFile = path.join(logDir, `${CLEANUP_SMOKE_NAME}.log`);
762+
const result = await runShellCommand({
763+
command,
764+
env: baseEnv,
765+
label: CLEANUP_SMOKE_NAME,
766+
logFile,
767+
});
768+
if (result.status === 0) {
769+
return undefined;
770+
}
771+
return {
772+
command,
773+
attempts: [
774+
{
775+
attempt: 1,
776+
elapsedSeconds: phaseElapsedSeconds(startedAtMs),
777+
finishedAt: new Date().toISOString(),
778+
noOutputTimedOut: result.noOutputTimedOut,
779+
startedAt: new Date(startedAtMs).toISOString(),
780+
status: result.status,
781+
timedOut: result.timedOut,
782+
},
783+
],
784+
elapsedSeconds: phaseElapsedSeconds(startedAtMs),
785+
finishedAt: new Date().toISOString(),
786+
image: baseEnv.OPENCLAW_DOCKER_E2E_IMAGE,
787+
logFile,
788+
name: CLEANUP_SMOKE_NAME,
789+
noOutputTimedOut: result.noOutputTimedOut,
790+
rerunCommand: command,
791+
startedAt: new Date(startedAtMs).toISOString(),
792+
status: result.status,
793+
targetable: false,
794+
timedOut: result.timedOut,
795+
};
796+
}
797+
715798
async function runForegroundGroup(entries, env) {
716799
const failures = [];
717800
for (const entry of entries) {
@@ -1469,17 +1552,58 @@ async function main() {
14691552
}
14701553

14711554
if (profile === DEFAULT_PROFILE && selectedLaneNames.length === 0) {
1472-
await runPhase(phases, "cleanup-smoke", {}, async () => {
1473-
await runForeground(
1474-
"Run cleanup smoke after parallel lanes",
1475-
"pnpm test:docker:cleanup",
1555+
const cleanupSmokeCommand = "pnpm test:docker:cleanup";
1556+
const cleanupStartedAtMs = Date.now();
1557+
let cleanupFailure;
1558+
try {
1559+
await runPhase(phases, CLEANUP_SMOKE_NAME, {}, async () => {
1560+
cleanupFailure = await runCleanupSmoke(
1561+
baseEnv,
1562+
logDir,
1563+
cleanupSmokeCommand,
1564+
cleanupStartedAtMs,
1565+
);
1566+
if (cleanupFailure) {
1567+
throw new Error(
1568+
`Run cleanup smoke after parallel lanes failed with status ${cleanupFailure.status}`,
1569+
);
1570+
}
1571+
});
1572+
} catch (error) {
1573+
cleanupFailure ??= await recordCleanupSmokeFailure(
1574+
error,
14761575
baseEnv,
1576+
logDir,
1577+
cleanupSmokeCommand,
1578+
cleanupStartedAtMs,
14771579
);
1478-
});
1580+
}
1581+
if (cleanupFailure) {
1582+
failures.push(cleanupFailure);
1583+
}
14791584
} else {
14801585
console.log("==> Cleanup smoke after parallel lanes: skipped for selected/release lanes");
14811586
}
14821587
await writeTimingStore(timingStore, allResults);
1588+
if (failures.length > 0) {
1589+
await writeRunSummary(logDir, {
1590+
chunk: releaseChunk || undefined,
1591+
failures,
1592+
image: baseEnv.OPENCLAW_DOCKER_E2E_IMAGE,
1593+
images: {
1594+
bare: baseEnv.OPENCLAW_DOCKER_E2E_BARE_IMAGE,
1595+
functional: baseEnv.OPENCLAW_DOCKER_E2E_FUNCTIONAL_IMAGE,
1596+
},
1597+
lanes: allResults,
1598+
phases,
1599+
profile,
1600+
selectedLanes: selectedLaneNames.length > 0 ? selectedLaneNames : undefined,
1601+
startedAt: runStartedAt,
1602+
status: "failed",
1603+
});
1604+
await printFailureSummary(failures, tailLines);
1605+
process.exit(1);
1606+
}
14831607
await writeRunSummary(logDir, {
14841608
chunk: releaseChunk || undefined,
14851609
failures,

‎test/scripts/docker-all-scheduler.test.ts‎

Lines changed: 80 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
// Docker All Scheduler tests cover docker all scheduler script behavior.
22
import { spawnSync } from "node:child_process";
3-
import { existsSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs";
3+
import { chmodSync, existsSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs";
44
import { tmpdir } from "node:os";
55
import path from "node:path";
66
import { setTimeout as delay } from "node:timers/promises";
@@ -177,6 +177,85 @@ describe("scripts/test-docker-all scheduler", () => {
177177
}
178178
});
179179

180+
posixIt("writes Docker run artifacts when cleanup smoke fails", () => {
181+
const root = mkdtempSync(`${tmpdir()}/openclaw-docker-all-cleanup-`);
182+
const logDir = path.join(root, "logs");
183+
const packageTgz = path.join(root, "openclaw-current.tgz");
184+
const fakePnpm = path.join(root, "pnpm");
185+
writeFileSync(packageTgz, "fake package\n", "utf8");
186+
writeFileSync(
187+
fakePnpm,
188+
`#!/usr/bin/env node
189+
const command = process.argv.slice(2).join(" ");
190+
if (command === "test:docker:cleanup") {
191+
console.error("cleanup smoke failed intentionally");
192+
process.exit(42);
193+
}
194+
process.exit(0);
195+
`,
196+
"utf8",
197+
);
198+
chmodSync(fakePnpm, 0o755);
199+
200+
try {
201+
const result = spawnSync(process.execPath, ["scripts/test-docker-all.mjs"], {
202+
cwd: process.cwd(),
203+
encoding: "utf8",
204+
env: {
205+
...process.env,
206+
OPENCLAW_CURRENT_PACKAGE_TGZ: packageTgz,
207+
OPENCLAW_DOCKER_ALL_BUILD: "0",
208+
OPENCLAW_DOCKER_ALL_LIVE_MODE: "skip",
209+
OPENCLAW_DOCKER_ALL_LOG_DIR: logDir,
210+
OPENCLAW_DOCKER_ALL_PARALLELISM: "16",
211+
OPENCLAW_DOCKER_ALL_PREFLIGHT: "0",
212+
OPENCLAW_DOCKER_ALL_START_STAGGER_MS: "0",
213+
OPENCLAW_DOCKER_ALL_STATUS_INTERVAL_MS: "0",
214+
OPENCLAW_DOCKER_ALL_TAIL_PARALLELISM: "16",
215+
OPENCLAW_DOCKER_ALL_TIMINGS: "0",
216+
PATH: `${root}${path.delimiter}${process.env.PATH ?? ""}`,
217+
},
218+
});
219+
220+
expect(result.status).toBe(1);
221+
expect(result.stderr).toContain("cleanup smoke failed intentionally");
222+
223+
const summary = JSON.parse(readFileSync(path.join(logDir, "summary.json"), "utf8"));
224+
expect(summary.status).toBe("failed");
225+
expect(summary.failures).toHaveLength(1);
226+
expect(summary.failures[0]).toMatchObject({
227+
name: "cleanup-smoke",
228+
rerunCommand: "pnpm test:docker:cleanup",
229+
status: 42,
230+
targetable: false,
231+
});
232+
expect(summary.lanes.some((lane: { name?: string }) => lane.name === "cleanup-smoke")).toBe(
233+
false,
234+
);
235+
expect(summary.phases.at(-1)).toMatchObject({
236+
name: "cleanup-smoke",
237+
status: "failed",
238+
});
239+
240+
const failureIndex = JSON.parse(readFileSync(path.join(logDir, "failures.json"), "utf8"));
241+
expect(failureIndex.status).toBe("failed");
242+
expect(failureIndex.combinedGhWorkflowCommand).toBeUndefined();
243+
expect(failureIndex.lanes[0]?.ghWorkflowCommand).toBeUndefined();
244+
expect(failureIndex.lanes).toEqual([
245+
expect.objectContaining({
246+
lane: "cleanup-smoke",
247+
rerunCommand: "pnpm test:docker:cleanup",
248+
status: 42,
249+
targetable: false,
250+
}),
251+
]);
252+
const cleanupLog = readFileSync(path.join(logDir, "cleanup-smoke.log"), "utf8");
253+
expect(cleanupLog).toContain("cleanup smoke failed intentionally");
254+
} finally {
255+
rmSync(root, { force: true, recursive: true });
256+
}
257+
});
258+
180259
it("allows an overweight lane to start alone under low parallelism", () => {
181260
expect(
182261
canStartSchedulerLane(

0 commit comments

Comments
 (0)