Skip to content

Commit f2677d5

Browse files
vincentkocvincentkoc
committed
fix(test): require startup bench report overlap
1 parent cd80610 commit f2677d5

2 files changed

Lines changed: 170 additions & 0 deletions

File tree

‎scripts/test-cli-startup-bench-budget.mjs‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -154,15 +154,34 @@ const current = readJsonFile(resolveCurrentReportPath());
154154
const baselineCases = indexCases(baseline);
155155
const currentCases = indexCases(current);
156156
const shouldRequireEveryBaselineCase = opts.preset === "all";
157+
const matchedBaselineCaseIds = [...baselineCases.keys()].filter((id) => currentCases.has(id));
157158

158159
let failed = false;
159160

161+
if (currentCases.size === 0) {
162+
console.error(
163+
`[test-cli-startup-bench-budget] current report has no cases for preset ${opts.preset}`,
164+
);
165+
failed = true;
166+
}
167+
160168
for (const [id] of baselineCases) {
161169
if (shouldRequireEveryBaselineCase && !currentCases.has(id)) {
162170
console.error(`[test-cli-startup-bench-budget] missing current case ${String(id)}`);
163171
failed = true;
164172
}
165173
}
174+
if (
175+
!opts.skipBaseline &&
176+
!shouldRequireEveryBaselineCase &&
177+
baselineCases.size > 0 &&
178+
matchedBaselineCaseIds.length === 0
179+
) {
180+
console.error(
181+
`[test-cli-startup-bench-budget] no current cases matched the baseline for preset ${opts.preset}`,
182+
);
183+
failed = true;
184+
}
166185

167186
for (const currentCase of currentCases.values()) {
168187
const samples = currentCase.samples ?? [];

‎test/scripts/cli-startup-bench-spawner.test.ts‎

Lines changed: 151 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -104,6 +104,157 @@ describe("CLI startup benchmark script spawners", () => {
104104
}
105105
});
106106

107+
it("rejects narrowed preset reports with no matching current cases", () => {
108+
const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "openclaw-bench-budget-empty-test-"));
109+
try {
110+
const baselinePath = path.join(tmpDir, "baseline.json");
111+
const reportPath = path.join(tmpDir, "current.json");
112+
fs.writeFileSync(
113+
baselinePath,
114+
JSON.stringify({
115+
primary: {
116+
cases: [
117+
{
118+
id: "gatewayStatusJson",
119+
name: "gateway status --json",
120+
samples: [{ ms: 10, firstOutputMs: 5, maxRssMb: 10, exitCode: 0, signal: null }],
121+
summary: {
122+
durationMs: { avg: 10, p50: 10, p95: 10, min: 10, max: 10 },
123+
firstOutputMs: { avg: 5, p50: 5, p95: 5, min: 5, max: 5 },
124+
maxRssMb: { avg: 10, p50: 10, p95: 10, min: 10, max: 10 },
125+
},
126+
},
127+
],
128+
},
129+
}),
130+
);
131+
fs.writeFileSync(reportPath, JSON.stringify({ primary: { cases: [] } }));
132+
133+
const result = spawnSync(
134+
process.execPath,
135+
[
136+
"scripts/test-cli-startup-bench-budget.mjs",
137+
"--baseline",
138+
baselinePath,
139+
"--report",
140+
reportPath,
141+
"--preset",
142+
"real",
143+
],
144+
{ cwd: process.cwd(), encoding: "utf8" },
145+
);
146+
147+
expect(result.status).toBe(1);
148+
expect(result.stderr).toContain(
149+
"[test-cli-startup-bench-budget] current report has no cases for preset real",
150+
);
151+
} finally {
152+
fs.rmSync(tmpDir, { recursive: true, force: true });
153+
}
154+
});
155+
156+
it("rejects narrowed preset reports with unrelated current cases when baseline checks run", () => {
157+
const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "openclaw-bench-budget-overlap-test-"));
158+
try {
159+
const baselinePath = path.join(tmpDir, "baseline.json");
160+
const reportPath = path.join(tmpDir, "current.json");
161+
const makeCase = (id: string, name: string) => ({
162+
id,
163+
name,
164+
samples: [{ ms: 10, firstOutputMs: 5, maxRssMb: 10, exitCode: 0, signal: null }],
165+
summary: {
166+
durationMs: { avg: 10, p50: 10, p95: 10, min: 10, max: 10 },
167+
firstOutputMs: { avg: 5, p50: 5, p95: 5, min: 5, max: 5 },
168+
maxRssMb: { avg: 10, p50: 10, p95: 10, min: 10, max: 10 },
169+
},
170+
});
171+
172+
fs.writeFileSync(
173+
baselinePath,
174+
JSON.stringify({ primary: { cases: [makeCase("fixtureOnly", "fixture only")] } }),
175+
);
176+
fs.writeFileSync(
177+
reportPath,
178+
JSON.stringify({ primary: { cases: [makeCase("targetOnly", "target only")] } }),
179+
);
180+
181+
const result = spawnSync(
182+
process.execPath,
183+
[
184+
"scripts/test-cli-startup-bench-budget.mjs",
185+
"--baseline",
186+
baselinePath,
187+
"--report",
188+
reportPath,
189+
"--preset",
190+
"real",
191+
],
192+
{
193+
cwd: process.cwd(),
194+
encoding: "utf8",
195+
env: {
196+
...process.env,
197+
OPENCLAW_STARTUP_BENCH_ENFORCE_NONCANONICAL_ARCH: "1",
198+
},
199+
},
200+
);
201+
202+
expect(result.status).toBe(1);
203+
expect(result.stderr).toContain(
204+
"[test-cli-startup-bench-budget] no current cases matched the baseline for preset real",
205+
);
206+
} finally {
207+
fs.rmSync(tmpDir, { recursive: true, force: true });
208+
}
209+
});
210+
211+
it("allows skip-baseline reports without fixture case overlap", () => {
212+
const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "openclaw-bench-budget-skip-test-"));
213+
try {
214+
const baselinePath = path.join(tmpDir, "baseline.json");
215+
const reportPath = path.join(tmpDir, "current.json");
216+
const makeCase = (id: string, name: string) => ({
217+
id,
218+
name,
219+
samples: [{ ms: 10, firstOutputMs: 5, maxRssMb: 10, exitCode: 0, signal: null }],
220+
summary: {
221+
durationMs: { avg: 10, p50: 10, p95: 10, min: 10, max: 10 },
222+
firstOutputMs: { avg: 5, p50: 5, p95: 5, min: 5, max: 5 },
223+
maxRssMb: { avg: 10, p50: 10, p95: 10, min: 10, max: 10 },
224+
},
225+
});
226+
227+
fs.writeFileSync(
228+
baselinePath,
229+
JSON.stringify({ primary: { cases: [makeCase("fixtureOnly", "fixture only")] } }),
230+
);
231+
fs.writeFileSync(
232+
reportPath,
233+
JSON.stringify({ primary: { cases: [makeCase("targetOnly", "target only")] } }),
234+
);
235+
236+
const result = spawnSync(
237+
process.execPath,
238+
[
239+
"scripts/test-cli-startup-bench-budget.mjs",
240+
"--baseline",
241+
baselinePath,
242+
"--report",
243+
reportPath,
244+
"--preset",
245+
"real",
246+
"--skip-baseline",
247+
],
248+
{ cwd: process.cwd(), encoding: "utf8" },
249+
);
250+
251+
expect(result.status).toBe(0);
252+
expect(result.stderr).not.toContain("no current cases matched the baseline");
253+
} finally {
254+
fs.rmSync(tmpDir, { recursive: true, force: true });
255+
}
256+
});
257+
107258
it("skips x64 startup budgets on noncanonical architectures", () => {
108259
const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "openclaw-bench-budget-arch-test-"));
109260
try {

0 commit comments

Comments
 (0)