Skip to content

Commit 928a75a

Browse files
shakkernerdshakkernerd
committed
refactor: route chat send media through user turn input
1 parent e5e6543 commit 928a75a

9 files changed

Lines changed: 121 additions & 203 deletions

File tree

‎src/auto-reply/dispatch.ts‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@ import { isOutboundDeliveryError } from "../infra/outbound/deliver-types.js";
1313
import { logMessageReceived } from "../logging/diagnostic.js";
1414
import { hasOutboundReplyContent } from "../plugin-sdk/reply-payload.js";
1515
import { getGlobalHookRunner } from "../plugins/hook-runner-global.js";
16+
import type { UserTurnInput } from "../sessions/user-turn-transcript.js";
1617
import type { SilentReplyConversationType } from "../shared/silent-reply-policy.js";
1718
import {
1819
resolveCommandTurnContext,
@@ -381,6 +382,7 @@ export async function dispatchInboundMessage(params: {
381382
cfg: OpenClawConfig;
382383
dispatcher: ReplyDispatcher;
383384
replyOptions?: Omit<GetReplyOptions, "onBlockReply">;
385+
userTurnInput?: UserTurnInput;
384386
replyResolver?: GetReplyFromConfig;
385387
}): Promise<DispatchInboundResult> {
386388
const finalized = measureDiagnosticsTimelineSpanSync(
@@ -412,6 +414,7 @@ export async function dispatchInboundMessage(params: {
412414
cfg: params.cfg,
413415
dispatcher: params.dispatcher,
414416
replyOptions: params.replyOptions,
417+
userTurnInput: params.userTurnInput,
415418
replyResolver: params.replyResolver,
416419
}),
417420
{
@@ -429,6 +432,7 @@ export async function dispatchInboundMessageWithBufferedDispatcher(params: {
429432
cfg: OpenClawConfig;
430433
dispatcherOptions: ReplyDispatcherWithTypingOptions;
431434
replyOptions?: Omit<GetReplyOptions, "onBlockReply">;
435+
userTurnInput?: UserTurnInput;
432436
replyResolver?: GetReplyFromConfig;
433437
}): Promise<DispatchInboundResult> {
434438
const finalized = finalizeInboundContext(params.ctx);
@@ -485,6 +489,7 @@ export async function dispatchInboundMessageWithBufferedDispatcher(params: {
485489
...params.replyOptions,
486490
...replyOptions,
487491
},
492+
userTurnInput: params.userTurnInput,
488493
});
489494
} finally {
490495
try {
@@ -507,6 +512,7 @@ export async function dispatchInboundMessageWithDispatcher(params: {
507512
cfg: OpenClawConfig;
508513
dispatcherOptions: ReplyDispatcherOptions;
509514
replyOptions?: Omit<GetReplyOptions, "onBlockReply">;
515+
userTurnInput?: UserTurnInput;
510516
replyResolver?: GetReplyFromConfig;
511517
}): Promise<DispatchInboundResult> {
512518
const silentReplyContext = resolveDispatcherSilentReplyContext(params.ctx, params.cfg);
@@ -522,5 +528,6 @@ export async function dispatchInboundMessageWithDispatcher(params: {
522528
dispatcher,
523529
replyResolver: params.replyResolver,
524530
replyOptions: params.replyOptions,
531+
userTurnInput: params.userTurnInput,
525532
});
526533
}

‎src/auto-reply/reply/dispatch-from-config.ts‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2512,6 +2512,7 @@ export async function dispatchReplyFromConfig(
25122512
},
25132513
},
25142514
replyConfig,
2515+
params.userTurnInput,
25152516
),
25162517
),
25172518
);

‎src/auto-reply/reply/dispatch-from-config.types.ts‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
import type { OpenClawConfig } from "../../config/types.openclaw.js";
2+
import type { UserTurnInput } from "../../sessions/user-turn-transcript.js";
23
import type { GetReplyOptions, SourceReplyDeliveryMode } from "../get-reply-options.types.js";
34
import type { FinalizedMsgContext } from "../templating.js";
45
import type { FormatAbortReplyText, TryFastAbortFromMessage } from "./abort.runtime-types.js";
@@ -18,6 +19,7 @@ export type DispatchFromConfigParams = {
1819
cfg: OpenClawConfig;
1920
dispatcher: ReplyDispatcher;
2021
replyOptions?: Omit<GetReplyOptions, "onBlockReply">;
22+
userTurnInput?: UserTurnInput;
2123
replyResolver?: GetReplyFromConfig;
2224
fastAbortResolver?: TryFastAbortFromMessage;
2325
formatAbortReplyTextResolver?: FormatAbortReplyText;

‎src/auto-reply/reply/get-reply-run.ts‎

Lines changed: 11 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,7 @@ import {
3535
import {
3636
buildPersistedUserTurnMediaInputsFromFields,
3737
buildPersistedUserTurnMessage,
38+
type UserTurnInput,
3839
} from "../../sessions/user-turn-transcript.js";
3940
import { createLazyImportLoader } from "../../shared/lazy-promise.js";
4041
import type { SilentReplyConversationType } from "../../shared/silent-reply-policy.js";
@@ -407,6 +408,7 @@ type RunPreparedReplyParams = {
407408
workspaceDir: string;
408409
abortedLastRun: boolean;
409410
autoFallbackPrimaryProbe?: AutoFallbackPrimaryProbe;
411+
userTurnInput?: UserTurnInput;
410412
};
411413

412414
export async function runPreparedReply(
@@ -1132,13 +1134,17 @@ export async function runPreparedReply(
11321134
? (internalOpts?.queuedFollowupAbortSignal ?? opts?.abortSignal)
11331135
: undefined;
11341136
const userTurnMediaForPersistence = buildPersistedUserTurnMediaInputsFromFields(ctx);
1135-
const userMessageForPersistence =
1136-
userTurnMediaForPersistence.length > 0
1137-
? buildPersistedUserTurnMessage({
1137+
const userTurnInput =
1138+
params.userTurnInput ??
1139+
(userTurnMediaForPersistence.length > 0
1140+
? {
11381141
text: baseBodyTrimmedRaw,
11391142
media: userTurnMediaForPersistence,
1140-
})
1141-
: undefined;
1143+
}
1144+
: undefined);
1145+
const userMessageForPersistence = userTurnInput
1146+
? buildPersistedUserTurnMessage(userTurnInput)
1147+
: undefined;
11421148
const followupRun = {
11431149
prompt: queuedBody,
11441150
transcriptPrompt: transcriptCommandBody,

‎src/auto-reply/reply/get-reply.ts‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@ import { formatErrorMessage } from "../../infra/errors.js";
1818
import { createSubsystemLogger } from "../../logging/subsystem.js";
1919
import { buildAgentHookContextChannelFields } from "../../plugins/hook-agent-context.js";
2020
import { defaultRuntime } from "../../runtime.js";
21+
import type { UserTurnInput } from "../../sessions/user-turn-transcript.js";
2122
import { createLazyImportLoader } from "../../shared/lazy-promise.js";
2223
import { normalizeOptionalString } from "../../shared/string-coerce.js";
2324
import { normalizeStringEntries } from "../../shared/string-normalization.js";
@@ -211,6 +212,7 @@ export async function getReplyFromConfig(
211212
ctx: MsgContext,
212213
opts?: GetReplyOptions,
213214
configOverride?: OpenClawConfig,
215+
userTurnInput?: UserTurnInput,
214216
): Promise<ReplyPayload | ReplyPayload[] | undefined> {
215217
const isFastTestEnv = process.env.OPENCLAW_TEST_FAST === "1";
216218
const cfg = resolveGetReplyConfig({
@@ -694,6 +696,7 @@ export async function getReplyFromConfig(
694696
workspaceDir,
695697
abortedLastRun,
696698
autoFallbackPrimaryProbe,
699+
userTurnInput,
697700
}),
698701
);
699702
logResolverTiming("completed", "fast_directive_prepared_reply");
@@ -995,6 +998,7 @@ export async function getReplyFromConfig(
995998
workspaceDir,
996999
abortedLastRun,
9971000
autoFallbackPrimaryProbe: runAutoFallbackPrimaryProbe,
1001+
userTurnInput,
9981002
}),
9991003
);
10001004
logResolverTiming("completed", "prepared_reply");

‎src/auto-reply/reply/get-reply.types.ts‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
import type { OpenClawConfig } from "../../config/types.openclaw.js";
2+
import type { UserTurnInput } from "../../sessions/user-turn-transcript.js";
23
import type { GetReplyOptions } from "../get-reply-options.types.js";
34
import type { ReplyPayload } from "../reply-payload.js";
45
import type { MsgContext } from "../templating.js";
@@ -7,4 +8,5 @@ export type GetReplyFromConfig = (
78
ctx: MsgContext,
89
opts?: GetReplyOptions,
910
configOverride?: OpenClawConfig,
11+
userTurnInput?: UserTurnInput,
1012
) => Promise<ReplyPayload | ReplyPayload[] | undefined>;

‎src/gateway/server-methods/chat.directive-tags.test.ts‎

Lines changed: 37 additions & 60 deletions
Original file line numberDiff line numberDiff line change
@@ -62,6 +62,7 @@ const mockState = vi.hoisted(() => ({
6262
lastDispatchCtx: undefined as MsgContext | undefined,
6363
lastDispatchImages: undefined as Array<{ mimeType: string; data: string }> | undefined,
6464
lastDispatchImageOrder: undefined as string[] | undefined,
65+
lastDispatchUserTurnInput: undefined as unknown,
6566
modelCatalog: null as ModelCatalogEntry[] | null,
6667
emittedTranscriptUpdates: [] as Array<{
6768
sessionFile: string;
@@ -194,10 +195,12 @@ vi.mock("../../auto-reply/dispatch.js", () => ({
194195
images?: Array<{ mimeType: string; data: string }>;
195196
imageOrder?: string[];
196197
};
198+
userTurnInput?: unknown;
197199
}) => {
198200
mockState.lastDispatchCtx = params.ctx;
199201
mockState.lastDispatchImages = params.replyOptions?.images;
200202
mockState.lastDispatchImageOrder = params.replyOptions?.imageOrder;
203+
mockState.lastDispatchUserTurnInput = params.userTurnInput;
201204
if (mockState.dispatchError) {
202205
throw mockState.dispatchError;
203206
}
@@ -703,6 +706,7 @@ describe("chat directive tag stripping for non-streaming final payloads", () =>
703706
mockState.lastDispatchCtx = undefined;
704707
mockState.lastDispatchImages = undefined;
705708
mockState.lastDispatchImageOrder = undefined;
709+
mockState.lastDispatchUserTurnInput = undefined;
706710
mockState.modelCatalog = null;
707711
mockState.emittedTranscriptUpdates = [];
708712
mockState.savedMediaResults = [];
@@ -2986,7 +2990,7 @@ describe("chat directive tag stripping for non-streaming final payloads", () =>
29862990
expect(mockState.lastDispatchCtx?.CommandBody).toBe("bench update");
29872991
});
29882992

2989-
it("emits a user transcript update when chat.send starts an agent run", async () => {
2993+
it("leaves text-only agent-run user persistence to Pi", async () => {
29902994
createTranscriptFixture("openclaw-chat-send-user-transcript-agent-run-");
29912995
mockState.finalText = "ok";
29922996
mockState.triggerAgentRunStart = true;
@@ -3001,13 +3005,8 @@ describe("chat directive tag stripping for non-streaming final payloads", () =>
30013005
expectBroadcast: false,
30023006
});
30033007

3004-
const userUpdate = findUserUpdate();
3005-
const message = userUpdateMessage(userUpdate);
3006-
expect(userUpdate?.sessionFile.endsWith("sess.jsonl")).toBe(true);
3007-
expect(userUpdate?.sessionKey).toBe("main");
3008-
expect(message?.role).toBe("user");
3009-
expect(message?.content).toBe("hello from dashboard");
3010-
expect(typeof message?.timestamp).toBe("number");
3008+
expect(findUserUpdate()).toBeUndefined();
3009+
expect(mockState.lastDispatchUserTurnInput).toBeUndefined();
30113010
const finalBroadcast = (
30123011
context.broadcast as unknown as ReturnType<typeof vi.fn>
30133012
).mock.calls.find((call) => call[0] === "chat" && call[1]?.state === "final")?.[1];
@@ -3100,7 +3099,7 @@ describe("chat directive tag stripping for non-streaming final payloads", () =>
31003099
expect(userUpdates).toHaveLength(0);
31013100
});
31023101

3103-
it("adds persisted media paths to the user transcript update", async () => {
3102+
it("prepares persisted media paths for Pi user-turn persistence", async () => {
31043103
createTranscriptFixture("openclaw-chat-send-user-transcript-images-");
31053104
mockState.finalText = "ok";
31063105
mockState.triggerAgentRunStart = true;
@@ -3135,9 +3134,6 @@ describe("chat directive tag stripping for non-streaming final payloads", () =>
31353134
});
31363135

31373136
await waitForAssertion(() => {
3138-
const userUpdate = findUserUpdate();
3139-
expect(userUpdate?.sessionFile.endsWith("sess.jsonl")).toBe(true);
3140-
expect(userUpdate?.sessionKey).toBe("main");
31413137
expect(mockState.savedMediaCalls).toEqual([
31423138
{
31433139
contentType: "image/png",
@@ -3152,33 +3148,28 @@ describe("chat directive tag stripping for non-streaming final payloads", () =>
31523148
]);
31533149
expect(typeof mockState.savedMediaCalls[0]?.size).toBe("number");
31543150
expect(typeof mockState.savedMediaCalls[1]?.size).toBe("number");
3155-
const message = userUpdateMessage(userUpdate) as
3151+
const userTurnInput = mockState.lastDispatchUserTurnInput as
31563152
| {
3157-
content?: unknown;
3158-
MediaPath?: string;
3159-
MediaPaths?: string[];
3160-
MediaType?: string;
3161-
MediaTypes?: string[];
3153+
text?: unknown;
3154+
media?: Array<{ path?: string; contentType?: string }>;
31623155
}
31633156
| undefined;
3164-
if (!message) {
3165-
throw new Error("expected user transcript update with media metadata");
3157+
if (!userTurnInput) {
3158+
throw new Error("expected user turn input with media metadata");
31663159
}
3167-
expect(message.content).toBe("edit these");
3168-
expect(message.MediaPath).toBe("/tmp/chat-send-image-a.png");
3169-
expect(message.MediaPaths).toEqual([
3170-
"/tmp/chat-send-image-a.png",
3171-
"/tmp/chat-send-image-b.jpg",
3160+
expect(findUserUpdate()).toBeUndefined();
3161+
expect(userTurnInput.text).toBe("edit these");
3162+
expect(userTurnInput.media).toEqual([
3163+
{ path: "/tmp/chat-send-image-a.png", contentType: "image/png" },
3164+
{ path: "/tmp/chat-send-image-b.jpg", contentType: "image/jpeg" },
31723165
]);
3173-
expect(message.MediaType).toBe("image/png");
3174-
expect(message.MediaTypes).toEqual(["image/png", "image/jpeg"]);
31753166
expect(mockState.lastDispatchCtx?.MediaPath).toBeUndefined();
31763167
expect(mockState.lastDispatchCtx?.MediaPaths).toBeUndefined();
31773168
expect(mockState.lastDispatchImages).toHaveLength(2);
31783169
});
31793170
});
31803171

3181-
it("persists non-image chat.send attachments as media refs without dispatch images", async () => {
3172+
it("prepares non-image chat.send attachments as media refs without dispatch images", async () => {
31823173
createTranscriptFixture("openclaw-chat-send-user-transcript-file-");
31833174
mockState.finalText = "ok";
31843175
mockState.triggerAgentRunStart = true;
@@ -3208,14 +3199,10 @@ describe("chat directive tag stripping for non-streaming final payloads", () =>
32083199
});
32093200

32103201
await waitForAssertion(() => {
3211-
const userUpdate = findUserUpdate();
3212-
const message = userUpdateMessage(userUpdate) as
3202+
const userTurnInput = mockState.lastDispatchUserTurnInput as
32133203
| {
3214-
content?: unknown;
3215-
MediaPath?: string;
3216-
MediaPaths?: string[];
3217-
MediaType?: string;
3218-
MediaTypes?: string[];
3204+
text?: unknown;
3205+
media?: Array<{ path?: string; contentType?: string }>;
32193206
}
32203207
| undefined;
32213208
expect(mockState.lastDispatchImages).toBeUndefined();
@@ -3224,11 +3211,11 @@ describe("chat directive tag stripping for non-streaming final payloads", () =>
32243211
expect(mockState.savedMediaCalls[0]?.contentType).toBe("application/pdf");
32253212
expect(mockState.savedMediaCalls[0]?.subdir).toBe("inbound");
32263213
expect(typeof mockState.savedMediaCalls[0]?.size).toBe("number");
3227-
expect(message?.content).toBe("summarize this");
3228-
expect(message?.MediaPath).toBe("/tmp/chat-send-brief.pdf");
3229-
expect(message?.MediaPaths).toEqual(["/tmp/chat-send-brief.pdf"]);
3230-
expect(message?.MediaType).toBe("application/pdf");
3231-
expect(message?.MediaTypes).toEqual(["application/pdf"]);
3214+
expect(findUserUpdate()).toBeUndefined();
3215+
expect(userTurnInput?.text).toBe("summarize this");
3216+
expect(userTurnInput?.media).toEqual([
3217+
{ path: "/tmp/chat-send-brief.pdf", contentType: "application/pdf" },
3218+
]);
32323219
});
32333220
});
32343221

@@ -3280,28 +3267,20 @@ describe("chat directive tag stripping for non-streaming final payloads", () =>
32803267
});
32813268

32823269
await waitForAssertion(() => {
3283-
const userUpdate = mockState.emittedTranscriptUpdates.find(
3284-
(update) =>
3285-
typeof update.message === "object" &&
3286-
update.message !== null &&
3287-
(update.message as { role?: unknown }).role === "user",
3288-
);
3289-
const message = userUpdate?.message as
3270+
const userTurnInput = mockState.lastDispatchUserTurnInput as
32903271
| {
3291-
MediaPath?: string;
3292-
MediaPaths?: string[];
3293-
MediaType?: string;
3294-
MediaTypes?: string[];
3272+
media?: Array<{ path?: string; contentType?: string }>;
32953273
}
32963274
| undefined;
3297-
expect(message?.MediaPath).toBe("/tmp/chat-send-inline.png");
3298-
expect(message?.MediaPaths).toEqual(["/tmp/chat-send-inline.png", "/tmp/offloaded-big.png"]);
3299-
expect(message?.MediaType).toBe("image/png");
3300-
expect(message?.MediaTypes).toEqual(["image/png", "image/png"]);
3275+
expect(findUserUpdate()).toBeUndefined();
3276+
expect(userTurnInput?.media).toEqual([
3277+
{ path: "/tmp/chat-send-inline.png", contentType: "image/png" },
3278+
{ path: "/tmp/offloaded-big.png", contentType: "image/png" },
3279+
]);
33013280
});
33023281
});
33033282

3304-
it("skips transcript media notes for ACP bridge clients", async () => {
3283+
it("leaves ACP bridge user persistence to the agent runtime", async () => {
33053284
createTranscriptFixture("openclaw-chat-send-user-transcript-acp-images-");
33063285
mockState.finalText = "ok";
33073286
mockState.triggerAgentRunStart = true;
@@ -3339,11 +3318,9 @@ describe("chat directive tag stripping for non-streaming final payloads", () =>
33393318
});
33403319

33413320
await waitForAssertion(() => {
3342-
const userUpdate = findUserUpdate();
3343-
const message = userUpdateMessage(userUpdate);
33443321
expect(mockState.savedMediaCalls).toStrictEqual([]);
3345-
expect(message?.role).toBe("user");
3346-
expect(message?.content).toBe("bridge image");
3322+
expect(findUserUpdate()).toBeUndefined();
3323+
expect(mockState.lastDispatchUserTurnInput).toBeUndefined();
33473324
});
33483325
});
33493326

0 commit comments

Comments
 (0)