Skip to content

Commit f0bfb3f

Browse files
hxy91819Copilotclawsweeper[bot]
authored
test(tools): add unmocked image custom-provider auth regression (#85733)
Summary: - The branch adds an unmocked image-tool custom-provider auth regression test, fixes split agents Vitest config routing, adds routing coverage, and records a changelog entry. - PR surface: Tests +203, Docs +1, Other +8. Total +212 across 4 files. - Reproducibility: not applicable. as a current-main failing issue: the production runtime bug was addressed by the linked predecessor, and this PR adds regression coverage plus test-routing verification for that path. Automerge notes: - PR branch already contained follow-up commit before automerge: test(tools): polish image auth regression and fix agents vitest routing - PR branch already contained follow-up commit before automerge: test(tools): remove proof test filename after regression rename - PR branch already contained follow-up commit before automerge: fix(test): remove duplicate agent shard constants - PR branch already contained follow-up commit before automerge: test(tools): add unmocked image custom-provider auth regression - PR branch already contained follow-up commit before automerge: fix(clawsweeper): address review for automerge-openclaw-openclaw-8573… Validation: - ClawSweeper review passed for head cff5476. - Required merge gates passed before the squash merge. Prepared head SHA: cff5476 Review: #85733 (comment) Co-authored-by: Mason Huang <masonxhuang@tencent.com> Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Co-authored-by: clawsweeper <274271284+clawsweeper[bot]@users.noreply.github.com> Co-authored-by: clawsweeper[bot] <274271284+clawsweeper[bot]@users.noreply.github.com> Approved-by: hxy91819 Co-authored-by: hxy91819 <8814856+hxy91819@users.noreply.github.com>
1 parent a3ae5c8 commit f0bfb3f

4 files changed

Lines changed: 212 additions & 0 deletions

File tree

‎CHANGELOG.md‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@ Docs: https://docs.openclaw.ai
2222
- Agents: release embedded-attempt session locks from outer teardown so post-prompt exceptions cannot wedge later requests behind `SessionWriteLockTimeoutError`. Fixes #86014. Thanks @openperf.
2323
- Discord/OpenAI voice: rotate Realtime sessions at provider max duration without logging the expected session-expiry event as an error.
2424
- Agents/media: derive bundled plugin local-media trust from plugin tool metadata instead of importing the full plugin registry on subscription paths. (#84409) Thanks @samzong.
25+
- Image tool: keep config-backed custom-provider API keys usable for auto-discovered vision models, including deferred image-tool execution without env keys or auth profiles. (#85733)
2526
- Memory/local embeddings: run local GGUF embeddings in an isolated worker sidecar and degrade to configured fallback or keyword search on worker failure so native embedding crashes do not take down the Gateway. (#85348) Thanks @osolmaz.
2627
- Gateway: clear the runtime config snapshot before `SIGUSR1` in-process restarts so config changes survive the next gateway loop. (#86388) Thanks @XuZehan-iCenter.
2728
- Models: show OAuth delegation markers as configured `models.json` auth while keeping runtime route usability checks strict. (#86378) Thanks @rohitjavvadi.

‎scripts/test-projects.test-support.mjs‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -240,6 +240,10 @@ const VITEST_CONFIG_BY_KIND = {
240240
agentSupport: AGENTS_SUPPORT_VITEST_CONFIG,
241241
agentTools: AGENTS_TOOLS_VITEST_CONFIG,
242242
agent: AGENTS_VITEST_CONFIG,
243+
agentsCore: AGENTS_CORE_VITEST_CONFIG,
244+
agentsPiEmbedded: AGENTS_PI_EMBEDDED_VITEST_CONFIG,
245+
agentsSupport: AGENTS_SUPPORT_VITEST_CONFIG,
246+
agentsTools: AGENTS_TOOLS_VITEST_CONFIG,
243247
autoReplyCore: AUTO_REPLY_CORE_VITEST_CONFIG,
244248
autoReplyReply: AUTO_REPLY_REPLY_VITEST_CONFIG,
245249
autoReplyTopLevel: AUTO_REPLY_TOP_LEVEL_VITEST_CONFIG,
@@ -1713,6 +1717,10 @@ export function buildVitestRunPlans(
17131717
"agentSupport",
17141718
"agentTools",
17151719
"agent",
1720+
"agentsCore",
1721+
"agentsPiEmbedded",
1722+
"agentsSupport",
1723+
"agentsTools",
17161724
"plugin",
17171725
"ui",
17181726
"uiE2e",

‎src/agents/tools/image-tool.custom-provider-auth.regression.test.ts‎

Lines changed: 185 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,185 @@
1+
import fs from "node:fs/promises";
2+
import os from "node:os";
3+
import path from "node:path";
4+
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
5+
import type { OpenClawConfig } from "../../config/config.js";
6+
import type { ModelDefinitionConfig } from "../../config/types.models.js";
7+
import type { ImageDescriptionRequest } from "../../plugin-sdk/media-understanding.js";
8+
import { getApiKeyForModel, hasUsableCustomProviderApiKey } from "../model-auth.js";
9+
import { resolveImageToolFactoryAvailable } from "../openclaw-tools.media-factory-plan.js";
10+
import { createImageTool, resolveImageModelConfigForTool, testing } from "./image-tool.js";
11+
import { hasProviderAuthForTool } from "./model-config.helpers.js";
12+
13+
const USER_PROVIDER = "hatchery-qwen3.6-plus";
14+
const USER_MODEL = "qwen3.6-plus";
15+
const USER_PRIMARY = `${USER_PROVIDER}/${USER_MODEL}`;
16+
const CONFIG_API_KEY = "sk-user-configured-key"; // pragma: allowlist secret
17+
18+
const ONE_PIXEL_PNG_B64 =
19+
"iVBORw0KGgoAAAANSUhEUgAAAEAAAABACAIAAAAlC+aJAAAAIGNIUk0AAHomAACAhAAA+gAAAIDoAAB1MAAA6mAAADqYAAAXcJy6UTwAAAAGYktHRAD/AP8A/6C9p5MAAAAHdElNRQfqBBsGAQr00ED3AAAAJXRFWHRkYXRlOmNyZWF0ZQAyMDI2LTA0LTI3VDA2OjAxOjEwKzAwOjAwPU3tXwAAACV0RVh0ZGF0ZTptb2RpZnkAMjAyNi0wNC0yN1QwNjowMToxMCswMDowMEwQVeMAAAAodEVYdGRhdGU6dGltZXN0YW1wADIwMjYtMDQtMjdUMDY6MDE6MTArMDA6MDAbBXQ8AAAAeElEQVRo3u3awQnDQBAEwT2Q8w/YAikIP5rF1RFMca+FO8/s7rrnqjcA1BsA6g0A9QaAesOfA77zqTf8Blj/AgAAAAAAAJsDqAOoA6gDqAOoc9TXAdQB1AHUAdQB1AHUAdQB1AHU7Qc46gEAAAAANrcecGZ2f8B/ASYSQPlKoEJ/AAAAAElFTkSuQmCC";
20+
21+
function makeVisionModel(id: string): ModelDefinitionConfig {
22+
return {
23+
id,
24+
name: id,
25+
reasoning: false,
26+
input: ["text", "image"],
27+
cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 },
28+
contextWindow: 128_000,
29+
maxTokens: 8_192,
30+
};
31+
}
32+
33+
function createUserReportedConfig(params?: { includeApiKey?: boolean }): OpenClawConfig {
34+
const includeApiKey = params?.includeApiKey ?? true;
35+
return {
36+
agents: {
37+
defaults: {
38+
model: { primary: USER_PRIMARY },
39+
},
40+
},
41+
models: {
42+
providers: {
43+
[USER_PROVIDER]: {
44+
baseUrl: "https://example.com/v1",
45+
api: "openai-completions",
46+
...(includeApiKey ? { apiKey: CONFIG_API_KEY } : {}),
47+
models: [makeVisionModel(USER_MODEL)],
48+
},
49+
},
50+
},
51+
};
52+
}
53+
54+
async function withEmptyAgentDir<T>(run: (agentDir: string) => Promise<T>): Promise<T> {
55+
const agentDir = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-image-auth-regression-"));
56+
try {
57+
return await run(agentDir);
58+
} finally {
59+
await fs.rm(agentDir, { recursive: true, force: true });
60+
}
61+
}
62+
63+
describe("image custom provider auth regression", () => {
64+
const priorFetch = global.fetch;
65+
66+
beforeEach(() => {
67+
for (const key of Object.keys(process.env)) {
68+
if (key.endsWith("_API_KEY") || key.endsWith("_OAUTH_TOKEN")) {
69+
vi.stubEnv(key, "");
70+
}
71+
}
72+
testing.setProviderDepsForTest({
73+
buildProviderRegistry: () => new Map(),
74+
getMediaUnderstandingProvider: () => undefined,
75+
describeImageWithModel: async (params: ImageDescriptionRequest) => ({
76+
text: `seen:${params.provider}/${params.model}`,
77+
model: params.model,
78+
}),
79+
describeImagesWithModel: async (params) => ({
80+
text: `seen:${params.provider}/${params.model}`,
81+
model: params.model,
82+
}),
83+
resolveAutoMediaKeyProviders: () => [],
84+
resolveDefaultMediaModel: () => undefined,
85+
});
86+
});
87+
88+
afterEach(() => {
89+
vi.unstubAllEnvs();
90+
global.fetch = priorFetch;
91+
testing.setProviderDepsForTest(undefined);
92+
});
93+
94+
it("uses real model-auth to accept config-only custom provider credentials", async () => {
95+
const cfg = createUserReportedConfig();
96+
expect(hasUsableCustomProviderApiKey(cfg, USER_PROVIDER)).toBe(true);
97+
expect(hasProviderAuthForTool({ provider: USER_PROVIDER, cfg })).toBe(true);
98+
});
99+
100+
it("auto-discovers the user-reported vision model without env key or auth profile", async () => {
101+
await withEmptyAgentDir(async (agentDir) => {
102+
const cfg = createUserReportedConfig();
103+
expect(resolveImageModelConfigForTool({ cfg, agentDir })).toEqual({
104+
primary: USER_PRIMARY,
105+
});
106+
});
107+
});
108+
109+
it("registers the image tool on the production factory path when the primary model has vision", async () => {
110+
await withEmptyAgentDir(async (agentDir) => {
111+
const cfg = createUserReportedConfig();
112+
expect(
113+
resolveImageToolFactoryAvailable({
114+
config: cfg,
115+
agentDir,
116+
modelHasVision: true,
117+
}),
118+
).toBe(true);
119+
});
120+
});
121+
122+
it("executes deferred image tool discovery with config-backed auth and runtime key resolution", async () => {
123+
await withEmptyAgentDir(async (agentDir) => {
124+
const cfg = createUserReportedConfig();
125+
const auth = await getApiKeyForModel({
126+
model: {
127+
id: USER_MODEL,
128+
name: USER_MODEL,
129+
provider: USER_PROVIDER,
130+
api: "openai-completions",
131+
baseUrl: "https://example.com/v1",
132+
reasoning: false,
133+
input: ["text", "image"],
134+
cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 },
135+
contextWindow: 128_000,
136+
maxTokens: 8_192,
137+
},
138+
cfg,
139+
agentDir,
140+
});
141+
expect(auth.apiKey).toBe(CONFIG_API_KEY);
142+
expect(auth.source).toContain("models.json");
143+
144+
const tool = createImageTool({
145+
config: cfg,
146+
agentDir,
147+
deferAutoModelResolution: true,
148+
modelHasVision: true,
149+
});
150+
expect(typeof tool?.execute).toBe("function");
151+
152+
const result = await tool!.execute("regression-1", {
153+
prompt: "Read this screenshot.",
154+
image: `data:image/png;base64,${ONE_PIXEL_PNG_B64}`,
155+
});
156+
157+
const payload = result as { content?: Array<{ type?: string; text?: string }> };
158+
const text = payload.content?.find((entry) => entry.type === "text")?.text ?? "";
159+
expect(text).toContain(`seen:${USER_PRIMARY}`);
160+
expect(text).not.toMatch(/No image model is configured/i);
161+
});
162+
});
163+
164+
it("still rejects the same config when apiKey is missing", async () => {
165+
await withEmptyAgentDir(async (agentDir) => {
166+
const cfg = createUserReportedConfig({ includeApiKey: false });
167+
expect(hasUsableCustomProviderApiKey(cfg, USER_PROVIDER)).toBe(false);
168+
expect(hasProviderAuthForTool({ provider: USER_PROVIDER, cfg })).toBe(false);
169+
expect(resolveImageModelConfigForTool({ cfg, agentDir })).toBeNull();
170+
171+
const tool = createImageTool({
172+
config: cfg,
173+
agentDir,
174+
deferAutoModelResolution: true,
175+
modelHasVision: true,
176+
});
177+
await expect(
178+
tool!.execute("regression-2", {
179+
prompt: "Read this screenshot.",
180+
image: `data:image/png;base64,${ONE_PIXEL_PNG_B64}`,
181+
}),
182+
).rejects.toThrow(/No image model is configured/);
183+
});
184+
});
185+
});

‎test/scripts/test-projects.test.ts‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -936,6 +936,24 @@ describe("scripts/test-projects changed-target routing", () => {
936936
expect(repoSourceReads.length).toBeLessThan(100);
937937
});
938938

939+
it.each([
940+
"test/vitest/vitest.agents-core.config.ts",
941+
"test/vitest/vitest.agents-pi-embedded.config.ts",
942+
"test/vitest/vitest.agents-support.config.ts",
943+
"test/vitest/vitest.agents-tools.config.ts",
944+
])("routes split agents vitest config %s to itself", (target) => {
945+
const plans = buildVitestRunPlans([target], process.cwd());
946+
947+
expect(plans).toEqual([
948+
{
949+
config: target,
950+
forwardedArgs: [],
951+
includePatterns: null,
952+
watchMode: false,
953+
},
954+
]);
955+
});
956+
939957
it.each([
940958
"src/gateway/gateway.test.ts",
941959
"src/gateway/server.startup-matrix-migration.integration.test.ts",

0 commit comments

Comments
 (0)