Skip to content

Commit 59d2e76

Browse files
steipetesteipete
authored
refactor: centralize live provider drift policy (#82033)
1 parent 7e9a863 commit 59d2e76

8 files changed

Lines changed: 271 additions & 202 deletions

‎src/agents/live-cache-regression-runner.test.ts‎

Lines changed: 0 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -84,20 +84,6 @@ describe("live cache regression runner", () => {
8484
).toBe(false);
8585
});
8686

87-
it("classifies Anthropic account drift as non-cache provider state", () => {
88-
expect(
89-
__testing.isAnthropicAccountDrift(
90-
new Error("Your credit balance is too low to access the Anthropic API."),
91-
),
92-
).toBe(true);
93-
expect(
94-
__testing.isAnthropicAccountDrift(
95-
'401 {"error":{"message":"The API key you provided is invalid."}}',
96-
),
97-
).toBe(true);
98-
expect(__testing.isAnthropicAccountDrift("anthropic:image cacheRead=0 < min=4500")).toBe(false);
99-
});
100-
10187
it("retries a cache probe twice when provider text misses the sentinel", () => {
10288
expect(
10389
__testing.shouldRetryCacheProbeText({

‎src/agents/live-cache-regression-runner.ts‎

Lines changed: 23 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -15,12 +15,10 @@ import {
1515
extractAssistantText,
1616
type LiveResolvedModel,
1717
logLiveCache,
18-
resolveLiveDirectModel,
18+
resolveLiveDirectModelPool,
19+
withLiveDirectModelApiKey,
1920
} from "./live-cache-test-support.js";
20-
import {
21-
isAuthErrorMessage,
22-
isBillingErrorMessage,
23-
} from "./pi-embedded-helpers/failover-matches.js";
21+
import { shouldSkipLiveProviderDrift } from "./live-test-provider-drift.js";
2422

2523
const OPENAI_TIMEOUT_MS = 120_000;
2624
const ANTHROPIC_TIMEOUT_MS = 120_000;
@@ -603,53 +601,52 @@ function appendBaselineFindings(target: BaselineFindings, source: BaselineFindin
603601
target.warnings.push(...source.warnings);
604602
}
605603

606-
function isAnthropicAccountDrift(error: unknown): boolean {
607-
const message = error instanceof Error ? error.message : String(error);
608-
return isBillingErrorMessage(message) || isAuthErrorMessage(message);
609-
}
610-
611604
function isAnthropicEmptyCacheProbe(error: unknown): boolean {
612605
return error instanceof CacheProbeTextMismatchError && error.text.trim().length === 0;
613606
}
614607

615-
function cloneFixtureWithKey(fixture: LiveResolvedModel, apiKey: string): LiveResolvedModel {
616-
return { ...fixture, apiKey };
608+
function shouldSkipAnthropicCacheProviderDrift(error: unknown): boolean {
609+
return Boolean(
610+
shouldSkipLiveProviderDrift({
611+
error,
612+
allowAuth: true,
613+
allowBilling: true,
614+
}),
615+
);
617616
}
618617

619618
async function runAnthropicCacheLane(params: {
619+
apiKeys: readonly string[];
620620
fixture: LiveResolvedModel;
621621
lane: CacheLane;
622622
pngBase64: string;
623623
runToken: string;
624624
warnings: string[];
625625
}): Promise<{ attempt?: Awaited<ReturnType<typeof runRepeatedLaneWithBaselineRetry>> }> {
626-
const keys =
627-
params.fixture.apiKeys && params.fixture.apiKeys.length > 0
628-
? params.fixture.apiKeys
629-
: [params.fixture.apiKey];
626+
const keys = params.apiKeys.length > 0 ? params.apiKeys : [params.fixture.apiKey];
630627
let lastError: unknown;
631628
for (const [index, apiKey] of keys.entries()) {
632629
try {
633630
return {
634631
attempt: await runRepeatedLaneWithBaselineRetry({
635632
lane: params.lane,
636633
providerTag: "anthropic",
637-
fixture: cloneFixtureWithKey(params.fixture, apiKey),
634+
fixture: withLiveDirectModelApiKey(params.fixture, apiKey),
638635
runToken: params.runToken,
639636
pngBase64: params.pngBase64,
640637
}),
641638
};
642639
} catch (error) {
643640
lastError = error;
644-
if (isAnthropicAccountDrift(error) && index + 1 < keys.length) {
641+
if (shouldSkipAnthropicCacheProviderDrift(error) && index + 1 < keys.length) {
645642
logLiveCache(`anthropic ${params.lane} account drift; retrying with next key`);
646643
continue;
647644
}
648645
break;
649646
}
650647
}
651648

652-
if (isAnthropicAccountDrift(lastError) || isAnthropicEmptyCacheProbe(lastError)) {
649+
if (shouldSkipAnthropicCacheProviderDrift(lastError) || isAnthropicEmptyCacheProbe(lastError)) {
653650
const reason = isAnthropicEmptyCacheProbe(lastError) ? "empty response" : "account drift";
654651
const warning = `anthropic ${params.lane} skipped: ${reason}`;
655652
params.warnings.push(warning);
@@ -671,7 +668,7 @@ async function runAnthropicDisabledCacheLane(params: {
671668
sessionId: `live-cache-regression-${params.runToken}-anthropic-disabled`,
672669
});
673670
} catch (error) {
674-
if (isAnthropicAccountDrift(error) || isAnthropicEmptyCacheProbe(error)) {
671+
if (shouldSkipAnthropicCacheProviderDrift(error) || isAnthropicEmptyCacheProbe(error)) {
675672
const warning = "anthropic disabled skipped: account drift";
676673
params.warnings.push(warning);
677674
logLiveCache(warning);
@@ -684,7 +681,6 @@ async function runAnthropicDisabledCacheLane(params: {
684681
export const __testing = {
685682
assertAgainstBaseline,
686683
evaluateAgainstBaseline,
687-
isAnthropicAccountDrift,
688684
resolveCacheProbeMaxTokens,
689685
shouldAcceptEmptyCacheProbe,
690686
shouldRetryCacheProbeText,
@@ -694,13 +690,13 @@ export const __testing = {
694690
export async function runLiveCacheRegression(): Promise<LiveCacheRegressionResult> {
695691
const pngBase64 = (await fs.readFile(LIVE_TEST_PNG_URL)).toString("base64");
696692
const runToken = randomUUID().slice(0, 13);
697-
const openai = await resolveLiveDirectModel({
693+
const openai = await resolveLiveDirectModelPool({
698694
provider: "openai",
699695
api: "openai-responses",
700696
envVar: "OPENCLAW_LIVE_OPENAI_CACHE_MODEL",
701697
preferredModelIds: ["gpt-4.1", "gpt-5.2", "gpt-5.4-mini", "gpt-5.4", "gpt-5.5"],
702698
});
703-
const anthropic = await resolveLiveDirectModel({
699+
const anthropic = await resolveLiveDirectModelPool({
704700
provider: "anthropic",
705701
api: "anthropic-messages",
706702
envVar: "OPENCLAW_LIVE_ANTHROPIC_CACHE_MODEL",
@@ -718,7 +714,7 @@ export async function runLiveCacheRegression(): Promise<LiveCacheRegressionResul
718714
const openaiAttempt = await runRepeatedLaneWithBaselineRetry({
719715
lane,
720716
providerTag: "openai",
721-
fixture: openai,
717+
fixture: openai.fixture,
722718
runToken,
723719
pngBase64,
724720
});
@@ -738,8 +734,9 @@ export async function runLiveCacheRegression(): Promise<LiveCacheRegressionResul
738734
appendBaselineFindings({ regressions, warnings }, openaiAttempt.findings);
739735

740736
const { attempt: anthropicAttempt } = await runAnthropicCacheLane({
737+
apiKeys: anthropic.apiKeys,
741738
lane,
742-
fixture: anthropic,
739+
fixture: anthropic.fixture,
743740
runToken,
744741
pngBase64,
745742
warnings,
@@ -765,7 +762,7 @@ export async function runLiveCacheRegression(): Promise<LiveCacheRegressionResul
765762
}
766763

767764
const disabled = await runAnthropicDisabledCacheLane({
768-
fixture: anthropic,
765+
fixture: anthropic.fixture,
769766
runToken,
770767
warnings,
771768
});

‎src/agents/live-cache-test-support.ts‎

Lines changed: 28 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -24,10 +24,14 @@ const DEFAULT_TIMEOUT_MS = 90_000;
2424

2525
export type LiveResolvedModel = {
2626
apiKey: string;
27-
apiKeys?: string[];
2827
model: Model<Api>;
2928
};
3029

30+
export type LiveResolvedModelPool = {
31+
apiKeys: string[];
32+
fixture: LiveResolvedModel;
33+
};
34+
3135
function toInt(value: string | undefined, fallback: number): number {
3236
const trimmed = value?.trim();
3337
if (!trimmed) {
@@ -162,12 +166,12 @@ export function computeCacheHitRate(usage: {
162166
return cacheRead / totalPrompt;
163167
}
164168

165-
export async function resolveLiveDirectModel(params: {
169+
export async function resolveLiveDirectModelPool(params: {
166170
provider: "anthropic" | "openai";
167171
api: "anthropic-messages" | "openai-responses";
168172
envVar: string;
169173
preferredModelIds: readonly string[];
170-
}): Promise<LiveResolvedModel> {
174+
}): Promise<LiveResolvedModelPool> {
171175
const liveKeys = collectProviderApiKeys(params.provider);
172176
const rawModel = process.env[params.envVar]?.trim();
173177
const parsed = rawModel ? parseModelRef(rawModel, params.provider) : null;
@@ -188,9 +192,11 @@ export async function resolveLiveDirectModel(params: {
188192
}
189193
logLiveCache(`resolved ${params.provider} model ${selectedModel.id} from live env key`);
190194
return {
191-
model: selectedModel,
192-
apiKey: liveKeys[0] ?? "",
193195
apiKeys: liveKeys,
196+
fixture: {
197+
model: selectedModel,
198+
apiKey: liveKeys[0] ?? "",
199+
},
194200
};
195201
}
196202

@@ -237,8 +243,23 @@ export async function resolveLiveDirectModel(params: {
237243
`resolved ${params.provider} model ${resolvedModel.id} from configured auth storage`,
238244
);
239245
return {
240-
model: resolvedModel,
241-
apiKey,
242246
apiKeys: [apiKey],
247+
fixture: {
248+
model: resolvedModel,
249+
apiKey,
250+
},
243251
};
244252
}
253+
254+
export async function resolveLiveDirectModel(
255+
params: Parameters<typeof resolveLiveDirectModelPool>[0],
256+
): Promise<LiveResolvedModel> {
257+
return (await resolveLiveDirectModelPool(params)).fixture;
258+
}
259+
260+
export function withLiveDirectModelApiKey(
261+
fixture: LiveResolvedModel,
262+
apiKey: string,
263+
): LiveResolvedModel {
264+
return { ...fixture, apiKey };
265+
}

‎src/agents/live-test-provider-drift.test.ts‎

Lines changed: 66 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,66 @@
1+
import { describe, expect, it } from "vitest";
2+
import {
3+
isLiveAuthDrift,
4+
isLiveBillingDrift,
5+
isLiveProviderUnavailableDrift,
6+
isLiveRateLimitDrift,
7+
shouldSkipLiveProviderDrift,
8+
} from "./live-test-provider-drift.js";
9+
10+
describe("live test provider drift", () => {
11+
it("classifies provider account drift", () => {
12+
expect(
13+
isLiveBillingDrift(new Error("Your credit balance is too low to access the Anthropic API.")),
14+
).toBe(true);
15+
expect(isLiveBillingDrift("billing has been disabled for this API key")).toBe(true);
16+
expect(isLiveBillingDrift("insufficient credit")).toBe(true);
17+
expect(
18+
isLiveAuthDrift('401 {"error":{"message":"The API key you provided is invalid."}}'),
19+
).toBe(true);
20+
});
21+
22+
it("classifies API-key rate-limit drift", () => {
23+
expect(isLiveRateLimitDrift("resource exhausted")).toBe(true);
24+
});
25+
26+
it("classifies transient provider availability drift", () => {
27+
expect(
28+
isLiveProviderUnavailableDrift(
29+
"521 <!DOCTYPE html><html><head><title>Web server is down</title></head><body>Cloudflare</body></html>",
30+
),
31+
).toBe(true);
32+
expect(
33+
isLiveProviderUnavailableDrift(
34+
"Error: <html><head><title>Service Unavailable</title></head><body>try again</body></html>",
35+
),
36+
).toBe(true);
37+
expect(
38+
isLiveProviderUnavailableDrift(
39+
"Error: <html><head><title>500 Internal Server Error</title></head><body>try again</body></html>",
40+
),
41+
).toBe(true);
42+
expect(
43+
isLiveProviderUnavailableDrift("provider returned error: 502 Internal Server Error"),
44+
).toBe(true);
45+
expect(
46+
isLiveProviderUnavailableDrift(
47+
"Service temporarily unavailable. The model is at capacity and currently cannot serve this request.",
48+
),
49+
).toBe(true);
50+
});
51+
52+
it("returns explicit skip labels only for enabled drift classes", () => {
53+
expect(
54+
shouldSkipLiveProviderDrift({
55+
error: '401 {"error":{"message":"The API key you provided is invalid."}}',
56+
allowAuth: true,
57+
}),
58+
).toEqual({ reason: "auth", label: "auth drift" });
59+
expect(
60+
shouldSkipLiveProviderDrift({
61+
error: '401 {"error":{"message":"The API key you provided is invalid."}}',
62+
allowBilling: true,
63+
}),
64+
).toBeUndefined();
65+
});
66+
});

0 commit comments

Comments
 (0)