Skip to content

Commit 57c952f

Browse files
steipetesteipete
authored
fix: add resilient media processing fallbacks (#83568)
1 parent 53d14d0 commit 57c952f

37 files changed

Lines changed: 1418 additions & 409 deletions

‎CHANGELOG.md‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -39,6 +39,7 @@ Docs: https://docs.openclaw.ai
3939

4040
### Fixes
4141

42+
- Media: install Sharp with the root package and fall back to sips, Windows native imaging, ImageMagick, GraphicsMagick, or ffmpeg for image resizing/conversion when Sharp is unavailable. Fixes #83401. Thanks @scotthuang.
4243
- Telegram: deliver generated media completions back into forum topics by preserving topic IDs across requester-agent handoff. (#83556) Thanks @fuller-stack-dev.
4344
- Gateway: defer update-check startup until after readiness so package update checks no longer block sidecar-ready startup, while preserving update broadcasts and shutdown cleanup. (#83520) Thanks @samzong.
4445
- Agents/video: hide `video_generate` reference-audio parameters unless a registered video provider supports audio inputs.

‎extensions/browser/src/browser/screenshot.test.ts‎

Lines changed: 39 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,22 @@ import { describe, expect, it } from "vitest";
33
import { normalizeBrowserScreenshot } from "./screenshot.js";
44

55
describe("browser screenshot normalization", () => {
6+
const unavailableImageBackend = process.platform === "win32" ? "sips" : "windows-native";
7+
8+
async function withUnavailableImageBackend<T>(fn: () => Promise<T>): Promise<T> {
9+
const previousBackend = process.env.OPENCLAW_IMAGE_BACKEND;
10+
process.env.OPENCLAW_IMAGE_BACKEND = unavailableImageBackend;
11+
try {
12+
return await fn();
13+
} finally {
14+
if (previousBackend === undefined) {
15+
delete process.env.OPENCLAW_IMAGE_BACKEND;
16+
} else {
17+
process.env.OPENCLAW_IMAGE_BACKEND = previousBackend;
18+
}
19+
}
20+
}
21+
622
it("shrinks oversized images to <=2000x2000 and <=5MB", async () => {
723
const bigPng = await sharp({
824
create: {
@@ -47,4 +63,27 @@ describe("browser screenshot normalization", () => {
4763

4864
expect(normalized.buffer.equals(jpeg)).toBe(true);
4965
});
66+
67+
it("rejects screenshots above max side when no image processor is available", async () => {
68+
const png = await sharp({
69+
create: {
70+
width: 420,
71+
height: 120,
72+
channels: 3,
73+
background: { r: 12, g: 34, b: 56 },
74+
},
75+
})
76+
.png({ compressionLevel: 9 })
77+
.toBuffer();
78+
expect(png.byteLength).toBeLessThan(5 * 1024 * 1024);
79+
80+
await withUnavailableImageBackend(async () => {
81+
await expect(
82+
normalizeBrowserScreenshot(png, {
83+
maxSide: 120,
84+
maxBytes: 5 * 1024 * 1024,
85+
}),
86+
).rejects.toThrow(/image processor unavailable/i);
87+
});
88+
});
5089
});

‎extensions/browser/src/browser/screenshot.ts‎

Lines changed: 25 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -2,8 +2,9 @@ import {
22
buildImageResizeSideGrid,
33
getImageMetadata,
44
IMAGE_REDUCE_QUALITY_STEPS,
5+
isImageProcessorUnavailableError,
56
resizeToJpeg,
6-
} from "../media/image-ops.js";
7+
} from "../media/media-services.js";
78

89
export const DEFAULT_BROWSER_SCREENSHOT_MAX_SIDE = 2000;
910
export const DEFAULT_BROWSER_SCREENSHOT_MAX_BYTES = 5 * 1024 * 1024;
@@ -31,15 +32,25 @@ export async function normalizeBrowserScreenshot(
3132
const sideGrid = buildImageResizeSideGrid(maxSide, sideStart);
3233

3334
let smallest: { buffer: Buffer; size: number } | null = null;
35+
let processorUnavailableError: unknown;
3436

3537
for (const side of sideGrid) {
3638
for (const quality of IMAGE_REDUCE_QUALITY_STEPS) {
37-
const out = await resizeToJpeg({
38-
buffer,
39-
maxSide: side,
40-
quality,
41-
withoutEnlargement: true,
42-
});
39+
let out: Buffer;
40+
try {
41+
out = await resizeToJpeg({
42+
buffer,
43+
maxSide: side,
44+
quality,
45+
withoutEnlargement: true,
46+
});
47+
} catch (err) {
48+
if (isImageProcessorUnavailableError(err)) {
49+
processorUnavailableError = err;
50+
break;
51+
}
52+
throw err;
53+
}
4354

4455
if (!smallest || out.byteLength < smallest.size) {
4556
smallest = { buffer: out, size: out.byteLength };
@@ -49,6 +60,13 @@ export async function normalizeBrowserScreenshot(
4960
return { buffer: out, contentType: "image/jpeg" };
5061
}
5162
}
63+
if (processorUnavailableError) {
64+
break;
65+
}
66+
}
67+
68+
if (processorUnavailableError) {
69+
throw processorUnavailableError;
5270
}
5371

5472
const best = smallest?.buffer ?? buffer;

‎extensions/browser/src/media/image-ops.ts‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,4 +3,4 @@ export {
33
buildImageResizeSideGrid,
44
getImageMetadata,
55
resizeToJpeg,
6-
} from "../sdk-setup-tools.js";
6+
} from "./media-services.js";

‎extensions/browser/src/media/media-services.ts‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
export {
2+
IMAGE_REDUCE_QUALITY_STEPS,
3+
buildImageResizeSideGrid,
4+
getImageMetadata,
5+
isImageProcessorUnavailableError,
6+
resizeToJpeg,
7+
} from "../sdk-setup-tools.js";

‎extensions/browser/src/sdk-setup-tools.ts‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,7 @@ export {
2323
IMAGE_REDUCE_QUALITY_STEPS,
2424
buildImageResizeSideGrid,
2525
getImageMetadata,
26+
isImageProcessorUnavailableError,
2627
resizeToJpeg,
2728
} from "openclaw/plugin-sdk/media-runtime";
2829
export { detectMime } from "openclaw/plugin-sdk/media-mime";

‎extensions/imessage/src/monitor/media-staging.ts‎

Lines changed: 13 additions & 48 deletions
Original file line numberDiff line numberDiff line change
@@ -1,17 +1,10 @@
1-
import { execFile } from "node:child_process";
21
import fs from "node:fs/promises";
32
import path from "node:path";
4-
import { promisify } from "node:util";
53
import { isInboundPathAllowed } from "openclaw/plugin-sdk/media-runtime";
64
import { saveMediaBuffer } from "openclaw/plugin-sdk/media-store";
7-
import { buildRandomTempFilePath } from "openclaw/plugin-sdk/temp-path";
5+
import { loadWebMedia } from "openclaw/plugin-sdk/web-media";
86
import type { IMessageAttachment } from "./types.js";
97

10-
const execFileAsync = promisify(execFile);
11-
12-
const HEIC_CONVERSION_TIMEOUT_MS = 15_000;
13-
const HEIC_CONVERSION_MAX_BUFFER_BYTES = 64 * 1024;
14-
158
export type StagedIMessageAttachment = {
169
path: string;
1710
contentType?: string;
@@ -73,43 +66,6 @@ async function resolveAllowedCanonicalAttachmentPath(params: {
7366
return canonicalPath;
7467
}
7568

76-
async function convertHeicToJpegWithSips(sourcePath: string, maxBytes: number): Promise<Buffer> {
77-
const tempPath = buildRandomTempFilePath({
78-
prefix: "openclaw-imessage",
79-
extension: "jpg",
80-
});
81-
try {
82-
await execFileAsync(
83-
"sips",
84-
[
85-
"-s",
86-
"format",
87-
"jpeg",
88-
"-s",
89-
"formatOptions",
90-
"90",
91-
"-Z",
92-
"4096",
93-
sourcePath,
94-
"--out",
95-
tempPath,
96-
],
97-
{
98-
timeout: HEIC_CONVERSION_TIMEOUT_MS,
99-
maxBuffer: HEIC_CONVERSION_MAX_BUFFER_BYTES,
100-
killSignal: "SIGKILL",
101-
},
102-
);
103-
const stat = await fs.stat(tempPath);
104-
if (stat.size > maxBytes) {
105-
throw new Error(`converted media exceeds ${Math.round(maxBytes / (1024 * 1024))}MB limit`);
106-
}
107-
return await fs.readFile(tempPath);
108-
} finally {
109-
await fs.rm(tempPath, { force: true }).catch(() => {});
110-
}
111-
}
112-
11369
async function readAttachmentBuffer(params: {
11470
attachmentPath: string;
11571
mimeType?: string | null;
@@ -142,11 +98,20 @@ async function readAttachmentBuffer(params: {
14298

14399
if (isHeicAttachment(params.attachmentPath, params.mimeType)) {
144100
try {
145-
const convert = params.deps.convertHeicToJpeg ?? convertHeicToJpegWithSips;
101+
const convert = params.deps.convertHeicToJpeg;
102+
const converted = convert
103+
? {
104+
buffer: await convert(canonicalPath, params.maxBytes),
105+
fileName: jpegFilenameForAttachment(params.attachmentPath),
106+
}
107+
: await loadWebMedia(canonicalPath, {
108+
maxBytes: params.maxBytes,
109+
localRoots: [path.dirname(canonicalPath)],
110+
});
146111
return {
147-
buffer: await convert(canonicalPath, params.maxBytes),
112+
buffer: converted.buffer,
148113
contentType: "image/jpeg",
149-
originalFilename: jpegFilenameForAttachment(params.attachmentPath),
114+
originalFilename: converted.fileName ?? jpegFilenameForAttachment(params.attachmentPath),
150115
};
151116
} catch (err) {
152117
params.deps.logVerbose?.(

‎extensions/speech-core/src/audio-transcode.test.ts‎

Lines changed: 0 additions & 64 deletions
This file was deleted.

0 commit comments

Comments
 (0)