Skip to content

Commit 275caeb

Browse files
vincentkocvincentkoc
authored
fix(ui): render pending sends in chat thread
Render submitted Control UI sends directly in the chat thread before the Gateway acknowledges `chat.send`. Pending sends now share acknowledged user-message content rendering for text and attachments, stay searchable with active chat filters, and failed queued sends remain queue-only. Verification: - focused UI Vitest suite: 201 tests passed - oxlint, core tsgo, core-test tsgo, diff check - Testbox changed gate: tbx_01kt0vnr2bv55aa6x588r77x0z - autoreview clean
1 parent 0f2732b commit 275caeb

6 files changed

Lines changed: 243 additions & 61 deletions

File tree

‎ui/src/ui/chat/build-chat-items.test.ts‎

Lines changed: 116 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -413,6 +413,122 @@ describe("buildChatItems", () => {
413413
expect(messageRecord(requireGroup(items[1])).content).toBe("Missing timestamp.");
414414
});
415415

416+
it("renders submitted queued sends as user turns before chat.send ACK", () => {
417+
const groups = messageGroups({
418+
messages: [{ role: "assistant", content: "Ready.", timestamp: 1 }],
419+
queue: [
420+
{
421+
id: "pending-send-1",
422+
text: "first visible send",
423+
createdAt: 2,
424+
sendSubmittedAtMs: 10,
425+
sendState: "sending",
426+
},
427+
],
428+
});
429+
430+
expect(groups.map((group) => group.role)).toEqual(["assistant", "user"]);
431+
expect(messageRecord(groups[1]).content).toStrictEqual([
432+
{ type: "text", text: "first visible send" },
433+
]);
434+
});
435+
436+
it("renders submitted queued attachment sends with attachment blocks before chat.send ACK", () => {
437+
const groups = messageGroups({
438+
queue: [
439+
{
440+
id: "pending-attachment-send-1",
441+
text: "see attached",
442+
createdAt: 2,
443+
sendSubmittedAtMs: 10,
444+
sendState: "sending",
445+
attachments: [
446+
{
447+
id: "attachment-1",
448+
mimeType: "image/png",
449+
fileName: "screenshot.png",
450+
previewUrl: "/media/screenshot.png",
451+
},
452+
],
453+
},
454+
],
455+
});
456+
457+
expect(groups).toHaveLength(1);
458+
expect(messageRecord(groups[0]).content).toStrictEqual([
459+
{ type: "text", text: "see attached" },
460+
{
461+
type: "image",
462+
url: "/media/screenshot.png",
463+
source: { type: "url", url: "/media/screenshot.png" },
464+
},
465+
]);
466+
});
467+
468+
it("does not collapse pending sends with matching history text", () => {
469+
const groups = messageGroups({
470+
messages: [{ role: "user", content: "same prompt", timestamp: 1 }],
471+
queue: [
472+
{
473+
id: "pending-send-1",
474+
text: "same prompt",
475+
createdAt: 2,
476+
sendSubmittedAtMs: 10,
477+
sendState: "sending",
478+
},
479+
],
480+
});
481+
482+
expect(groups).toHaveLength(1);
483+
expect(groups[0].messages).toHaveLength(2);
484+
expect(groups[0].messages[0].duplicateCount).toBeUndefined();
485+
expect(groups[0].messages[1].duplicateCount).toBeUndefined();
486+
});
487+
488+
it("keeps failed queued sends out of the thread", () => {
489+
const groups = messageGroups({
490+
queue: [
491+
{
492+
id: "failed-send-1",
493+
text: "restore me to the composer",
494+
createdAt: 1,
495+
sendSubmittedAtMs: 10,
496+
sendState: "failed",
497+
},
498+
],
499+
});
500+
501+
expect(groups).toStrictEqual([]);
502+
});
503+
504+
it("filters submitted queued sends while chat search is active", () => {
505+
const groups = messageGroups({
506+
searchOpen: true,
507+
searchQuery: "matching",
508+
queue: [
509+
{
510+
id: "pending-send-1",
511+
text: "matching prompt",
512+
createdAt: 1,
513+
sendSubmittedAtMs: 10,
514+
sendState: "sending",
515+
},
516+
{
517+
id: "pending-send-2",
518+
text: "unrelated prompt",
519+
createdAt: 2,
520+
sendSubmittedAtMs: 11,
521+
sendState: "sending",
522+
},
523+
],
524+
});
525+
526+
expect(groups).toHaveLength(1);
527+
expect(messageRecord(groups[0]).content).toStrictEqual([
528+
{ type: "text", text: "matching prompt" },
529+
]);
530+
});
531+
416532
it("attaches lifted canvas previews to the nearest assistant turn", () => {
417533
const groups = messageGroups({
418534
messages: [

‎ui/src/ui/chat/build-chat-items.ts‎

Lines changed: 58 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
import type { ChatItem, MessageGroup, NormalizedMessage, ToolCard } from "../types/chat-types.ts";
2+
import type { ChatQueueItem } from "../ui-types.ts";
23
import {
34
isAssistantHeartbeatAckForDisplay,
45
stripHeartbeatTokenForDisplay,
@@ -9,6 +10,7 @@ import { normalizeMessage, stripMessageDisplayMetadataText } from "./message-nor
910
import { normalizeRoleForGrouping } from "./role-normalizer.ts";
1011
import { messageMatchesSearchQuery } from "./search-match.ts";
1112
import { extractToolCardsCached, extractToolPreview } from "./tool-cards.ts";
13+
import { buildUserChatMessageContentBlocks } from "./user-message-content.ts";
1214

1315
export type BuildChatItemsProps = {
1416
sessionKey: string;
@@ -17,6 +19,7 @@ export type BuildChatItemsProps = {
1719
streamSegments: Array<{ text: string; ts: number }>;
1820
stream: string | null;
1921
streamStartedAt: number | null;
22+
queue?: ChatQueueItem[];
2023
showToolCalls: boolean;
2124
searchOpen?: boolean;
2225
searchQuery?: string;
@@ -223,6 +226,10 @@ function groupMessages(items: ChatItem[]): Array<ChatItem | MessageGroup> {
223226
}
224227

225228
function collapseDuplicateDisplaySignature(message: unknown): string | null {
229+
const marker = asRecord(message)?.["__openclaw"];
230+
if (asRecord(marker)?.kind === "pending-send") {
231+
return null;
232+
}
226233
const normalized = safeNormalizeMessage(message);
227234
if (!normalized) {
228235
return null;
@@ -292,6 +299,34 @@ function trimAccumulatedStreamPrefix(text: string, previousText: string | null):
292299
return text.slice(previousText.length).trimStart();
293300
}
294301

302+
function shouldRenderQueuedSendInThread(item: ChatQueueItem): boolean {
303+
if (typeof item.sendSubmittedAtMs !== "number" || item.sendState === "failed") {
304+
return false;
305+
}
306+
return (
307+
item.sendState === "waiting-model" ||
308+
item.sendState === "sending" ||
309+
item.sendState === "waiting-reconnect"
310+
);
311+
}
312+
313+
function queuedSendThreadMessage(item: ChatQueueItem): Record<string, unknown> | null {
314+
const content = buildUserChatMessageContentBlocks(item.text, item.attachments);
315+
if (content.length === 0) {
316+
return null;
317+
}
318+
return {
319+
role: "user",
320+
content,
321+
timestamp: item.createdAt,
322+
__openclaw: {
323+
kind: "pending-send",
324+
id: item.id,
325+
state: item.sendState,
326+
},
327+
};
328+
}
329+
295330
function rawMessageTimestamp(message: unknown): number | null {
296331
const timestamp = asRecord(message)?.timestamp;
297332
return typeof timestamp === "number" && Number.isFinite(timestamp) ? timestamp : null;
@@ -535,6 +570,29 @@ export function buildChatItems(props: BuildChatItemsProps): Array<ChatItem | Mes
535570
message: msg,
536571
});
537572
}
573+
const queuedSends = Array.isArray(props.queue) ? props.queue : [];
574+
for (const queued of queuedSends) {
575+
if (!shouldRenderQueuedSendInThread(queued)) {
576+
continue;
577+
}
578+
const message = queuedSendThreadMessage(queued);
579+
if (!message) {
580+
continue;
581+
}
582+
const searchQuery = props.searchQuery ?? "";
583+
if (
584+
props.searchOpen &&
585+
searchQuery.trim() &&
586+
!messageMatchesSearchQuery(message, searchQuery)
587+
) {
588+
continue;
589+
}
590+
items.push({
591+
kind: "message",
592+
key: `pending-send:${queued.id}`,
593+
message,
594+
});
595+
}
538596
for (const liftedCanvasSource of liftedCanvasSources) {
539597
const assistantIndex = findNearestAssistantMessageIndex(items, liftedCanvasSource.timestamp);
540598
if (assistantIndex == null) {

‎ui/src/ui/chat/user-message-content.ts‎

Lines changed: 63 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,63 @@
1+
import type { ChatAttachment } from "../ui-types.ts";
2+
import { getChatAttachmentPreviewUrl } from "./attachment-payload-store.ts";
3+
4+
export type UserChatMessageContentBlock = {
5+
type: string;
6+
text?: string;
7+
url?: string;
8+
source?: unknown;
9+
attachment?: {
10+
url: string;
11+
kind: "audio" | "document";
12+
label: string;
13+
mimeType?: string;
14+
};
15+
};
16+
17+
function isInlineDataUrl(value: string): boolean {
18+
return /^\s*data:/iu.test(value);
19+
}
20+
21+
function formatInlineImageAttachmentPlaceholder(attachment: ChatAttachment): string {
22+
const label = attachment.fileName?.trim();
23+
return label ? `Attached image: ${label}` : "Attached image";
24+
}
25+
26+
export function buildUserChatMessageContentBlocks(
27+
message: string,
28+
attachments?: readonly ChatAttachment[],
29+
): UserChatMessageContentBlock[] {
30+
const blocks: UserChatMessageContentBlock[] = [];
31+
const text = message.trim();
32+
if (text) {
33+
blocks.push({ type: "text", text });
34+
}
35+
for (const attachment of attachments ?? []) {
36+
const previewUrl = getChatAttachmentPreviewUrl(attachment);
37+
if (!previewUrl) {
38+
continue;
39+
}
40+
if (attachment.mimeType.startsWith("image/")) {
41+
if (isInlineDataUrl(previewUrl)) {
42+
blocks.push({ type: "text", text: formatInlineImageAttachmentPlaceholder(attachment) });
43+
continue;
44+
}
45+
blocks.push({
46+
type: "image",
47+
url: previewUrl,
48+
source: { type: "url", url: previewUrl },
49+
});
50+
continue;
51+
}
52+
blocks.push({
53+
type: "attachment",
54+
attachment: {
55+
url: previewUrl,
56+
kind: attachment.mimeType.startsWith("audio/") ? "audio" : "document",
57+
label: attachment.fileName?.trim() || "Attached file",
58+
mimeType: attachment.mimeType,
59+
},
60+
});
61+
}
62+
return blocks;
63+
}

‎ui/src/ui/controllers/chat.ts‎

Lines changed: 3 additions & 60 deletions
Original file line numberDiff line numberDiff line change
@@ -1,14 +1,12 @@
11
import { resetToolStream } from "../app-tool-stream.ts";
2-
import {
3-
getChatAttachmentDataUrl,
4-
getChatAttachmentPreviewUrl,
5-
} from "../chat/attachment-payload-store.ts";
2+
import { getChatAttachmentDataUrl } from "../chat/attachment-payload-store.ts";
63
import {
74
isAssistantHeartbeatAckForDisplay,
85
stripHeartbeatTokenForDisplay,
96
} from "../chat/heartbeat-display.ts";
107
import { extractText } from "../chat/message-extract.ts";
118
import { reconcileChatRunLifecycle } from "../chat/run-lifecycle.ts";
9+
import { buildUserChatMessageContentBlocks } from "../chat/user-message-content.ts";
1210
import { formatConnectError } from "../connect-error.ts";
1311
import { GatewayRequestError, type GatewayBrowserClient, type GatewayHelloOk } from "../gateway.ts";
1412
import {
@@ -634,15 +632,6 @@ function dataUrlToBase64(dataUrl: string): { content: string; mimeType: string }
634632
return { mimeType: match[1], content: match[2] };
635633
}
636634

637-
function isInlineDataUrl(value: string): boolean {
638-
return /^\s*data:/iu.test(value);
639-
}
640-
641-
function formatInlineImageAttachmentPlaceholder(attachment: ChatAttachment): string {
642-
const label = attachment.fileName?.trim();
643-
return label ? `Attached image: ${label}` : "Attached image";
644-
}
645-
646635
function buildApiAttachments(attachments?: ChatAttachment[]) {
647636
const hasAttachments = attachments && attachments.length > 0;
648637
return hasAttachments
@@ -861,57 +850,11 @@ export function appendUserChatMessage(
861850
attachments?: ChatAttachment[],
862851
timestamp = Date.now(),
863852
) {
864-
const msg = message.trim();
865-
const hasAttachments = attachments && attachments.length > 0;
866-
const contentBlocks: Array<{
867-
type: string;
868-
text?: string;
869-
url?: string;
870-
source?: unknown;
871-
attachment?: {
872-
url: string;
873-
kind: "audio" | "document";
874-
label: string;
875-
mimeType?: string;
876-
};
877-
}> = [];
878-
if (msg) {
879-
contentBlocks.push({ type: "text", text: msg });
880-
}
881-
if (hasAttachments) {
882-
for (const att of attachments) {
883-
const previewUrl = getChatAttachmentPreviewUrl(att);
884-
if (!previewUrl) {
885-
continue;
886-
}
887-
if (att.mimeType.startsWith("image/")) {
888-
if (isInlineDataUrl(previewUrl)) {
889-
contentBlocks.push({ type: "text", text: formatInlineImageAttachmentPlaceholder(att) });
890-
continue;
891-
}
892-
contentBlocks.push({
893-
type: "image",
894-
url: previewUrl,
895-
source: { type: "url", url: previewUrl },
896-
});
897-
continue;
898-
}
899-
contentBlocks.push({
900-
type: "attachment",
901-
attachment: {
902-
url: previewUrl,
903-
kind: att.mimeType.startsWith("audio/") ? "audio" : "document",
904-
label: att.fileName?.trim() || "Attached file",
905-
mimeType: att.mimeType,
906-
},
907-
});
908-
}
909-
}
910853
state.chatMessages = [
911854
...state.chatMessages,
912855
{
913856
role: "user",
914-
content: contentBlocks,
857+
content: buildUserChatMessageContentBlocks(message, attachments),
915858
timestamp,
916859
},
917860
];

‎ui/src/ui/e2e/chat-flow.e2e.test.ts‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -272,6 +272,7 @@ describeControlUiE2e("Control UI mocked Gateway E2E", () => {
272272
expect(params.sessionKey).toBe("main");
273273

274274
const runId = requireString(params.idempotencyKey, "chat send idempotency key");
275+
await page.locator(".chat-thread").getByText(prompt).waitFor({ timeout: 10_000 });
275276
await gateway.resolveDeferred("chat.history", {
276277
messages: [],
277278
sessionId: "control-ui-e2e-session",
@@ -310,7 +311,7 @@ describeControlUiE2e("Control UI mocked Gateway E2E", () => {
310311

311312
await page.locator(".chat-queue").getByText("Sending").waitFor({ timeout: 10_000 });
312313
await page.locator(".chat-queue").getByText(prompt).waitFor({ timeout: 10_000 });
313-
expect(await page.locator(".chat-thread").getByText(prompt).count()).toBe(0);
314+
await page.locator(".chat-thread").getByText(prompt).waitFor({ timeout: 10_000 });
314315

315316
await gateway.resolveDeferred("chat.send", { runId, status: "started" });
316317

0 commit comments

Comments
 (0)