Skip to content

Commit faa443a

Browse files
authored
fix(chat/ios): downscale image attachments before send
Resize iOS chat PhotosPicker image attachments through the shared JPEG transcoder before staging/sending. Cap long edge and payload bytes, strip source metadata, preserve previews from processed data, and add focused processor/view-model regression tests.\n\nFixes #68524.\nSupersedes #73710.
1 parent 61ae9b7 commit faa443a

6 files changed

Lines changed: 423 additions & 11 deletions

File tree

CHANGELOG.md

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@ Docs: https://docs.openclaw.ai
1010

1111
### Fixes
1212

13+
- iOS/chat: resize PhotosPicker image attachments to capped JPEGs before staging and sending, stripping source metadata and keeping oversized camera photos under the chat upload budget. Fixes #68524. Thanks @BunsDev.
1314
- Codex startup: treat selectable configured OpenAI agent models as Codex runtime requirements during plugin auto-enable, startup planning, and doctor install repair, so Anthropic-primary configs can still switch to OpenAI/Codex cleanly.
1415
- Agents: preserve source-reply delivery metadata when merging tool-returned media into the final reply, keeping message-tool-only replies deliverable and mirrored. Thanks @pashpashpash and @vincentkoc.
1516
- macOS/companion: require system TLS trust before pinning a first-use direct `wss://` gateway certificate and honor `gateway.remote.tlsFingerprint` as the explicit pin for remote node-mode sessions, so fresh endpoints fail closed when macOS cannot trust the certificate unless configured out of band. Fixes #50642. Thanks @BunsDev.

apps/shared/OpenClawKit/Sources/OpenClawChatUI/ChatViewModel.swift

Lines changed: 25 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@ private let chatUILogger = Logger(subsystem: "ai.openclaw", category: "OpenClawC
1717
// swiftlint:disable:next type_body_length
1818
public final class OpenClawChatViewModel {
1919
public static let defaultModelSelectionID = "__default__"
20+
private static let maxAttachmentBytes = 5_000_000
2021

2122
public private(set) var messages: [OpenClawChatMessage] = []
2223
public var input: String = ""
@@ -1298,11 +1299,6 @@ public final class OpenClawChatViewModel {
12981299
}
12991300

13001301
private func addImageAttachment(url: URL?, data: Data, fileName: String, mimeType: String) async {
1301-
if data.count > 5_000_000 {
1302-
self.errorText = "Attachment \(fileName) exceeds 5 MB limit"
1303-
return
1304-
}
1305-
13061302
let uti: UTType = {
13071303
if let url {
13081304
return UTType(filenameExtension: url.pathExtension) ?? .data
@@ -1314,13 +1310,33 @@ public final class OpenClawChatViewModel {
13141310
return
13151311
}
13161312

1317-
let preview = Self.previewImage(data: data)
1313+
let processed: Data
1314+
do {
1315+
processed = try await Task.detached(priority: .userInitiated) {
1316+
try ChatImageProcessor.processForUpload(data: data)
1317+
}.value
1318+
} catch {
1319+
self.errorText = "Could not process \(fileName): \(error.localizedDescription)"
1320+
return
1321+
}
1322+
1323+
if processed.count > Self.maxAttachmentBytes {
1324+
self.errorText = "Attachment \(fileName) exceeds 5 MB limit after resizing"
1325+
return
1326+
}
1327+
1328+
let outputFileName: String = {
1329+
let baseName = (fileName as NSString).deletingPathExtension
1330+
return baseName.isEmpty ? "image.jpg" : "\(baseName).jpg"
1331+
}()
1332+
1333+
let preview = Self.previewImage(data: processed)
13181334
self.attachments.append(
13191335
OpenClawPendingAttachment(
13201336
url: url,
1321-
data: data,
1322-
fileName: fileName,
1323-
mimeType: mimeType,
1337+
data: processed,
1338+
fileName: outputFileName,
1339+
mimeType: "image/jpeg",
13241340
preview: preview))
13251341
}
13261342

Lines changed: 44 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,44 @@
1+
import Foundation
2+
3+
/// Chat-specific image upload policy built on the shared JPEG transcoder.
4+
public enum ChatImageProcessor {
5+
public static let maxLongEdgePx = 1600
6+
public static let jpegQuality = 0.8
7+
public static let maxPayloadBytes = 3_500_000
8+
9+
public enum ProcessError: Error, LocalizedError, Sendable {
10+
case notAnImage
11+
case decodeFailed
12+
case encodeFailed
13+
14+
public var errorDescription: String? {
15+
switch self {
16+
case .notAnImage:
17+
"The data is not a recognizable image."
18+
case .decodeFailed:
19+
"The image could not be decoded."
20+
case .encodeFailed:
21+
"The image could not be resized to fit the chat upload limit."
22+
}
23+
}
24+
}
25+
26+
public static func processForUpload(data: Data) throws -> Data {
27+
do {
28+
let result = try JPEGTranscoder.transcodeToJPEG(
29+
imageData: data,
30+
maxLongEdgePx: self.maxLongEdgePx,
31+
quality: self.jpegQuality,
32+
maxBytes: self.maxPayloadBytes)
33+
return result.data
34+
} catch JPEGTranscodeError.decodeFailed {
35+
throw ProcessError.notAnImage
36+
} catch JPEGTranscodeError.propertiesMissing {
37+
throw ProcessError.decodeFailed
38+
} catch JPEGTranscodeError.sizeLimitExceeded {
39+
throw ProcessError.encodeFailed
40+
} catch {
41+
throw ProcessError.encodeFailed
42+
}
43+
}
44+
}

apps/shared/OpenClawKit/Sources/OpenClawKit/JPEGTranscoder.swift

Lines changed: 57 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -37,6 +37,26 @@ public struct JPEGTranscoder: Sendable {
3737
maxWidthPx: Int?,
3838
quality: Double,
3939
maxBytes: Int? = nil) throws -> (data: Data, widthPx: Int, heightPx: Int)
40+
{
41+
try self.transcodeToJPEG(
42+
imageData: imageData,
43+
maxWidthPx: maxWidthPx,
44+
maxLongEdgePx: nil,
45+
quality: quality,
46+
maxBytes: maxBytes)
47+
}
48+
49+
/// Re-encodes image data to JPEG, optionally downscaling so the *oriented* longest edge is <= `maxLongEdgePx`.
50+
///
51+
/// When `maxLongEdgePx` is provided it takes precedence over `maxWidthPx`.
52+
/// - Important: This normalizes EXIF orientation (the output pixels are rotated if needed; orientation tag is not
53+
/// relied on).
54+
public static func transcodeToJPEG(
55+
imageData: Data,
56+
maxWidthPx: Int? = nil,
57+
maxLongEdgePx: Int?,
58+
quality: Double,
59+
maxBytes: Int? = nil) throws -> (data: Data, widthPx: Int, heightPx: Int)
4060
{
4161
guard let src = CGImageSourceCreateWithData(imageData as CFData, nil) else {
4262
throw JPEGTranscodeError.decodeFailed
@@ -63,6 +83,10 @@ public struct JPEGTranscoder: Sendable {
6383

6484
let maxDim = max(orientedWidth, orientedHeight)
6585
var targetMaxPixelSize: Int = {
86+
if let maxLongEdgePx, maxLongEdgePx > 0 {
87+
guard maxDim > maxLongEdgePx else { return maxDim } // never upscale
88+
return maxLongEdgePx
89+
}
6690
guard let maxWidthPx, maxWidthPx > 0 else { return maxDim }
6791
guard orientedWidth > maxWidthPx else { return maxDim } // never upscale
6892

@@ -81,19 +105,20 @@ public struct JPEGTranscoder: Sendable {
81105
guard let img = CGImageSourceCreateThumbnailAtIndex(src, 0, thumbOpts as CFDictionary) else {
82106
throw JPEGTranscodeError.decodeFailed
83107
}
108+
let opaqueImage = Self.flattenAlphaIfNeeded(img)
84109

85110
let out = NSMutableData()
86111
guard let dest = CGImageDestinationCreateWithData(out, UTType.jpeg.identifier as CFString, 1, nil) else {
87112
throw JPEGTranscodeError.encodeFailed
88113
}
89114
let q = self.clampQuality(quality)
90115
let encodeProps = [kCGImageDestinationLossyCompressionQuality: q] as CFDictionary
91-
CGImageDestinationAddImage(dest, img, encodeProps)
116+
CGImageDestinationAddImage(dest, opaqueImage, encodeProps)
92117
guard CGImageDestinationFinalize(dest) else {
93118
throw JPEGTranscodeError.encodeFailed
94119
}
95120

96-
return (out as Data, img.width, img.height)
121+
return (out as Data, opaqueImage.width, opaqueImage.height)
97122
}
98123

99124
guard let maxBytes, maxBytes > 0 else {
@@ -132,4 +157,34 @@ public struct JPEGTranscoder: Sendable {
132157

133158
return best
134159
}
160+
161+
/// JPEG cannot store alpha. Flatten transparent sources over white before encoding so ImageIO does not composite
162+
/// transparent pixels onto black by default.
163+
private static func flattenAlphaIfNeeded(_ image: CGImage) -> CGImage {
164+
switch image.alphaInfo {
165+
case .none, .noneSkipFirst, .noneSkipLast:
166+
return image
167+
default:
168+
break
169+
}
170+
171+
guard
172+
let context = CGContext(
173+
data: nil,
174+
width: image.width,
175+
height: image.height,
176+
bitsPerComponent: 8,
177+
bytesPerRow: 0,
178+
space: CGColorSpaceCreateDeviceRGB(),
179+
bitmapInfo: CGImageAlphaInfo.noneSkipLast.rawValue)
180+
else {
181+
return image
182+
}
183+
184+
let rect = CGRect(x: 0, y: 0, width: image.width, height: image.height)
185+
context.setFillColor(CGColor(red: 1, green: 1, blue: 1, alpha: 1))
186+
context.fill(rect)
187+
context.draw(image, in: rect)
188+
return context.makeImage() ?? image
189+
}
135190
}
Lines changed: 187 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,187 @@
1+
import CoreGraphics
2+
import Foundation
3+
import ImageIO
4+
import Testing
5+
import UniformTypeIdentifiers
6+
@testable import OpenClawKit
7+
8+
struct ChatImageProcessorTests {
9+
private func syntheticJPEG(width: Int, height: Int) throws -> Data {
10+
guard
11+
let context = CGContext(
12+
data: nil,
13+
width: width,
14+
height: height,
15+
bitsPerComponent: 8,
16+
bytesPerRow: width * 4,
17+
space: CGColorSpaceCreateDeviceRGB(),
18+
bitmapInfo: CGImageAlphaInfo.premultipliedLast.rawValue)
19+
else {
20+
throw NSError(domain: "ChatImageProcessorTests", code: 1)
21+
}
22+
23+
context.setFillColor(CGColor(red: 0.8, green: 0.2, blue: 0.4, alpha: 1))
24+
context.fill(CGRect(x: 0, y: 0, width: width, height: height))
25+
context.setFillColor(CGColor(red: 0.1, green: 0.7, blue: 0.3, alpha: 1))
26+
context.fill(CGRect(x: 0, y: 0, width: width / 2, height: height / 2))
27+
28+
guard let image = context.makeImage() else {
29+
throw NSError(domain: "ChatImageProcessorTests", code: 2)
30+
}
31+
32+
let data = NSMutableData()
33+
guard let destination = CGImageDestinationCreateWithData(data, UTType.jpeg.identifier as CFString, 1, nil)
34+
else {
35+
throw NSError(domain: "ChatImageProcessorTests", code: 3)
36+
}
37+
38+
let properties: [CFString: Any] = [
39+
kCGImageDestinationLossyCompressionQuality: 0.95,
40+
kCGImagePropertyExifDictionary: [
41+
kCGImagePropertyExifDateTimeOriginal: "2026:04:20 16:30:00",
42+
kCGImagePropertyExifLensModel: "Leaky Lens 50mm f/1.4",
43+
] as CFDictionary,
44+
kCGImagePropertyGPSDictionary: [
45+
kCGImagePropertyGPSLatitude: 60.02,
46+
kCGImagePropertyGPSLatitudeRef: "N",
47+
kCGImagePropertyGPSLongitude: 10.95,
48+
kCGImagePropertyGPSLongitudeRef: "E",
49+
] as CFDictionary,
50+
kCGImagePropertyTIFFDictionary: [
51+
kCGImagePropertyTIFFMake: "LeakCorp",
52+
kCGImagePropertyTIFFModel: "Privacy-Leaker-1",
53+
] as CFDictionary,
54+
]
55+
CGImageDestinationAddImage(destination, image, properties as CFDictionary)
56+
guard CGImageDestinationFinalize(destination) else {
57+
throw NSError(domain: "ChatImageProcessorTests", code: 4)
58+
}
59+
return data as Data
60+
}
61+
62+
private func syntheticPNGWithAlpha(width: Int, height: Int) throws -> Data {
63+
guard
64+
let context = CGContext(
65+
data: nil,
66+
width: width,
67+
height: height,
68+
bitsPerComponent: 8,
69+
bytesPerRow: width * 4,
70+
space: CGColorSpaceCreateDeviceRGB(),
71+
bitmapInfo: CGImageAlphaInfo.premultipliedLast.rawValue)
72+
else {
73+
throw NSError(domain: "ChatImageProcessorTests", code: 5)
74+
}
75+
76+
context.clear(CGRect(x: 0, y: 0, width: width, height: height))
77+
context.setFillColor(CGColor(red: 1, green: 0, blue: 0, alpha: 1))
78+
context.fill(CGRect(x: width / 4, y: height / 4, width: width / 2, height: height / 2))
79+
80+
guard let image = context.makeImage() else {
81+
throw NSError(domain: "ChatImageProcessorTests", code: 6)
82+
}
83+
84+
let data = NSMutableData()
85+
guard let destination = CGImageDestinationCreateWithData(data, UTType.png.identifier as CFString, 1, nil)
86+
else {
87+
throw NSError(domain: "ChatImageProcessorTests", code: 7)
88+
}
89+
CGImageDestinationAddImage(destination, image, nil)
90+
guard CGImageDestinationFinalize(destination) else {
91+
throw NSError(domain: "ChatImageProcessorTests", code: 8)
92+
}
93+
return data as Data
94+
}
95+
96+
private func properties(for data: Data) -> [CFString: Any] {
97+
guard
98+
let source = CGImageSourceCreateWithData(data as CFData, nil),
99+
let properties = CGImageSourceCopyPropertiesAtIndex(source, 0, nil) as? [CFString: Any]
100+
else {
101+
return [:]
102+
}
103+
return properties
104+
}
105+
106+
private func dimensions(for data: Data) -> (width: Int, height: Int)? {
107+
let properties = self.properties(for: data)
108+
guard
109+
let width = properties[kCGImagePropertyPixelWidth] as? NSNumber,
110+
let height = properties[kCGImagePropertyPixelHeight] as? NSNumber
111+
else {
112+
return nil
113+
}
114+
return (width.intValue, height.intValue)
115+
}
116+
117+
@Test func `resizes landscape long edge to upload limit`() throws {
118+
let source = try self.syntheticJPEG(width: 4000, height: 3000)
119+
let output = try ChatImageProcessor.processForUpload(data: source)
120+
let dimensions = try #require(self.dimensions(for: output))
121+
122+
#expect(max(dimensions.width, dimensions.height) <= ChatImageProcessor.maxLongEdgePx)
123+
#expect(abs((Double(dimensions.width) / Double(dimensions.height)) - (4000.0 / 3000.0)) <= 0.02)
124+
}
125+
126+
@Test func `resizes portrait long edge to upload limit`() throws {
127+
let source = try self.syntheticJPEG(width: 3000, height: 4000)
128+
let output = try ChatImageProcessor.processForUpload(data: source)
129+
let dimensions = try #require(self.dimensions(for: output))
130+
131+
#expect(max(dimensions.width, dimensions.height) <= ChatImageProcessor.maxLongEdgePx)
132+
#expect(abs((Double(dimensions.width) / Double(dimensions.height)) - (3000.0 / 4000.0)) <= 0.02)
133+
}
134+
135+
@Test func `resizes narrow tall long edge to upload limit`() throws {
136+
let source = try self.syntheticJPEG(width: 1080, height: 2400)
137+
let output = try ChatImageProcessor.processForUpload(data: source)
138+
let dimensions = try #require(self.dimensions(for: output))
139+
140+
#expect(max(dimensions.width, dimensions.height) <= ChatImageProcessor.maxLongEdgePx)
141+
#expect(abs((Double(dimensions.width) / Double(dimensions.height)) - (1080.0 / 2400.0)) <= 0.02)
142+
}
143+
144+
@Test func `small image is not upscaled`() throws {
145+
let source = try self.syntheticJPEG(width: 400, height: 300)
146+
let output = try ChatImageProcessor.processForUpload(data: source)
147+
let dimensions = try #require(self.dimensions(for: output))
148+
149+
#expect(max(dimensions.width, dimensions.height) <= 400)
150+
}
151+
152+
@Test func `output fits payload budget`() throws {
153+
let source = try self.syntheticJPEG(width: 4000, height: 3000)
154+
let output = try ChatImageProcessor.processForUpload(data: source)
155+
156+
#expect(output.count <= ChatImageProcessor.maxPayloadBytes)
157+
}
158+
159+
@Test func `rejects non image data`() {
160+
let garbage = Data("not an image".utf8)
161+
162+
#expect(throws: ChatImageProcessor.ProcessError.self) {
163+
_ = try ChatImageProcessor.processForUpload(data: garbage)
164+
}
165+
}
166+
167+
@Test func `strips source metadata from output`() throws {
168+
let source = try self.syntheticJPEG(width: 3000, height: 2000)
169+
let output = try ChatImageProcessor.processForUpload(data: source)
170+
let properties = self.properties(for: output)
171+
let gps = properties[kCGImagePropertyGPSDictionary] as? [CFString: Any] ?? [:]
172+
173+
#expect(gps.isEmpty)
174+
for needle in ["Leaky Lens", "LeakCorp", "Privacy-Leaker", "2026:04:20"] {
175+
#expect(output.range(of: Data(needle.utf8)) == nil)
176+
}
177+
}
178+
179+
@Test func `flattens transparent sources to opaque JPEG`() throws {
180+
let source = try self.syntheticPNGWithAlpha(width: 800, height: 600)
181+
let output = try ChatImageProcessor.processForUpload(data: source)
182+
let imageSource = try #require(CGImageSourceCreateWithData(output as CFData, nil))
183+
let image = try #require(CGImageSourceCreateImageAtIndex(imageSource, 0, nil))
184+
185+
#expect([.none, .noneSkipFirst, .noneSkipLast].contains(image.alphaInfo))
186+
}
187+
}

0 commit comments

Comments
 (0)