Skip to content

Commit 9ef0a25

Browse files
romtsnromtsn
authored
Merge cb424d1 into 6b019b7 
2 parents 6b019b7 + cb424d1 commit 9ef0a25

10 files changed

Lines changed: 589 additions & 25 deletions

File tree

‎CHANGELOG.md‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,11 @@
22

33
## 8.40.0
44

5+
### Features
6+
7+
- Session Replay: experimental support for capturing `SurfaceView` content (e.g. Unity, video players, maps) ([#5333](https://github.com/getsentry/sentry-java/pull/5333))
8+
- To enable, set `options.sessionReplay.isCaptureSurfaceViews = true`
9+
510
### Fixes
611

712
- Fix `NoSuchMethodError` for `LayoutCoordinates.localBoundingBoxOf$default` on Compose touch dispatch with AGP 8.13 and `minSdk < 24` ([#5302](https://github.com/getsentry/sentry-java/pull/5302))

‎sentry-android-replay/src/main/java/io/sentry/android/replay/ScreenshotRecorder.kt‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -70,7 +70,7 @@ internal class ScreenshotRecorder(
7070
)
7171
}
7272

73-
if (!contentChanged.get()) {
73+
if (!contentChanged.get() && !screenshotStrategy.hasSurfaceViews()) {
7474
screenshotStrategy.emitLastScreenshot()
7575
return
7676
}

‎sentry-android-replay/src/main/java/io/sentry/android/replay/screenshot/PixelCopyStrategy.kt‎

Lines changed: 190 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,13 @@ package io.sentry.android.replay.screenshot
22

33
import android.annotation.SuppressLint
44
import android.graphics.Bitmap
5+
import android.graphics.Canvas
56
import android.graphics.Matrix
7+
import android.graphics.Paint
8+
import android.graphics.PorterDuff
9+
import android.graphics.PorterDuffXfermode
10+
import android.graphics.Rect
11+
import android.graphics.RectF
612
import android.view.PixelCopy
713
import android.view.View
814
import io.sentry.SentryLevel.DEBUG
@@ -19,6 +25,7 @@ import io.sentry.android.replay.util.ReplayRunnable
1925
import io.sentry.android.replay.util.traverse
2026
import io.sentry.android.replay.viewhierarchy.ViewHierarchyNode
2127
import java.util.concurrent.atomic.AtomicBoolean
28+
import java.util.concurrent.atomic.AtomicInteger
2229
import kotlin.LazyThreadSafetyMode.NONE
2330

2431
@SuppressLint("UseKtx")
@@ -40,6 +47,16 @@ internal class PixelCopyStrategy(
4047
private val maskRenderer = MaskRenderer()
4148
private val contentChanged = AtomicBoolean(false)
4249
private val isClosed = AtomicBoolean(false)
50+
private val hasSurfaceViews = AtomicBoolean(false)
51+
private val dstOverPaint by
52+
lazy(NONE) { Paint().apply { xfermode = PorterDuffXfermode(PorterDuff.Mode.DST_OVER) } }
53+
private val screenshotCanvas by lazy(NONE) { Canvas(screenshot) }
54+
private val tmpSrcRect = Rect()
55+
private val tmpDstRect = RectF()
56+
private val windowLocation = IntArray(2)
57+
private val svLocation = IntArray(2)
58+
59+
private class SurfaceViewCapture(val bitmap: Bitmap, val x: Int, val y: Int)
4360

4461
@SuppressLint("NewApi")
4562
override fun capture(root: View) {
@@ -81,31 +98,22 @@ internal class PixelCopyStrategy(
8198

8299
// TODO: disableAllMasking here and dont traverse?
83100
val viewHierarchy = ViewHierarchyNode.fromView(root, null, 0, options.sessionReplay)
84-
root.traverse(viewHierarchy, options.sessionReplay, options.logger)
85-
86-
executor.submit(
87-
ReplayRunnable("screenshot_recorder.mask") {
88-
if (isClosed.get() || screenshot.isRecycled) {
89-
options.logger.log(DEBUG, "PixelCopyStrategy is closed, skipping masking")
90-
return@ReplayRunnable
91-
}
101+
val captureSurfaceViewsEnabled = options.sessionReplay.isCaptureSurfaceViews
102+
val surfaceViewNodes =
103+
if (captureSurfaceViewsEnabled) {
104+
mutableListOf<ViewHierarchyNode.SurfaceViewHierarchyNode>()
105+
} else {
106+
null
107+
}
108+
root.traverse(viewHierarchy, options.sessionReplay, options.logger, surfaceViewNodes)
92109

93-
val debugMasks = maskRenderer.renderMasks(screenshot, viewHierarchy, prescaledMatrix)
110+
hasSurfaceViews.set(surfaceViewNodes?.isNotEmpty() == true)
94111

95-
if (options.replayController.isDebugMaskingOverlayEnabled()) {
96-
mainLooperHandler.post {
97-
if (debugOverlayDrawable.callback == null) {
98-
root.overlay.add(debugOverlayDrawable)
99-
}
100-
debugOverlayDrawable.updateMasks(debugMasks)
101-
root.postInvalidate()
102-
}
103-
}
104-
screenshotRecorderCallback?.onScreenshotRecorded(screenshot)
105-
lastCaptureSuccessful.set(true)
106-
contentChanged.set(false)
107-
}
108-
)
112+
if (surfaceViewNodes.isNullOrEmpty()) {
113+
submitMaskingAndCallback(root, viewHierarchy)
114+
} else {
115+
captureSurfaceViews(root, surfaceViewNodes, viewHierarchy)
116+
}
109117
},
110118
mainLooperHandler.handler,
111119
)
@@ -115,6 +123,126 @@ internal class PixelCopyStrategy(
115123
}
116124
}
117125

126+
private fun submitMaskingAndCallback(root: View, viewHierarchy: ViewHierarchyNode) {
127+
executor.submit(
128+
ReplayRunnable("screenshot_recorder.mask") { applyMaskingAndNotify(root, viewHierarchy) }
129+
)
130+
}
131+
132+
private fun applyMaskingAndNotify(root: View, viewHierarchy: ViewHierarchyNode) {
133+
if (isClosed.get() || screenshot.isRecycled) {
134+
options.logger.log(DEBUG, "PixelCopyStrategy is closed, skipping masking")
135+
return
136+
}
137+
138+
val debugMasks = maskRenderer.renderMasks(screenshot, viewHierarchy, prescaledMatrix)
139+
140+
if (options.replayController.isDebugMaskingOverlayEnabled()) {
141+
mainLooperHandler.post {
142+
if (debugOverlayDrawable.callback == null) {
143+
root.overlay.add(debugOverlayDrawable)
144+
}
145+
debugOverlayDrawable.updateMasks(debugMasks)
146+
root.postInvalidate()
147+
}
148+
}
149+
screenshotRecorderCallback?.onScreenshotRecorded(screenshot)
150+
lastCaptureSuccessful.set(true)
151+
contentChanged.set(false)
152+
}
153+
154+
@SuppressLint("NewApi")
155+
private fun captureSurfaceViews(
156+
root: View,
157+
surfaceViewNodes: List<ViewHierarchyNode.SurfaceViewHierarchyNode>,
158+
viewHierarchy: ViewHierarchyNode,
159+
) {
160+
root.getLocationOnScreen(windowLocation)
161+
162+
val captures = arrayOfNulls<SurfaceViewCapture>(surfaceViewNodes.size)
163+
val remaining = AtomicInteger(surfaceViewNodes.size)
164+
165+
fun onCaptureComplete() {
166+
if (remaining.decrementAndGet() == 0) {
167+
compositeSurfaceViewsAndMask(root, captures, viewHierarchy)
168+
}
169+
}
170+
171+
for ((index, node) in surfaceViewNodes.withIndex()) {
172+
val surfaceView = node.surfaceViewRef.get()
173+
// holder.surface can be null before the surface is created — guard against NPE.
174+
val surface = surfaceView?.holder?.surface
175+
if (surfaceView == null || surface == null || !surface.isValid) {
176+
onCaptureComplete()
177+
continue
178+
}
179+
180+
try {
181+
val svBitmap =
182+
Bitmap.createBitmap(surfaceView.width, surfaceView.height, Bitmap.Config.ARGB_8888)
183+
184+
surfaceView.getLocationOnScreen(svLocation)
185+
val capturedX = svLocation[0]
186+
val capturedY = svLocation[1]
187+
188+
PixelCopy.request(
189+
surfaceView,
190+
svBitmap,
191+
{ copyResult: Int ->
192+
if (copyResult == PixelCopy.SUCCESS) {
193+
captures[index] = SurfaceViewCapture(svBitmap, capturedX, capturedY)
194+
} else {
195+
svBitmap.recycle()
196+
options.logger.log(INFO, "Failed to capture SurfaceView: %d", copyResult)
197+
}
198+
onCaptureComplete()
199+
},
200+
mainLooperHandler.handler,
201+
)
202+
} catch (e: Throwable) {
203+
options.logger.log(WARNING, "Failed to capture SurfaceView", e)
204+
onCaptureComplete()
205+
}
206+
}
207+
}
208+
209+
private fun compositeSurfaceViewsAndMask(
210+
root: View,
211+
captures: Array<SurfaceViewCapture?>,
212+
viewHierarchy: ViewHierarchyNode,
213+
) {
214+
executor.submit(
215+
ReplayRunnable("screenshot_recorder.composite") {
216+
if (isClosed.get() || screenshot.isRecycled) {
217+
options.logger.log(DEBUG, "PixelCopyStrategy is closed, skipping compositing")
218+
return@ReplayRunnable
219+
}
220+
221+
for (capture in captures) {
222+
if (capture == null) continue
223+
if (capture.bitmap.isRecycled) continue
224+
225+
compositeSurfaceViewInto(
226+
screenshotCanvas,
227+
dstOverPaint,
228+
tmpSrcRect,
229+
tmpDstRect,
230+
capture.bitmap,
231+
capture.x,
232+
capture.y,
233+
windowLocation[0],
234+
windowLocation[1],
235+
config.scaleFactorX,
236+
config.scaleFactorY,
237+
)
238+
capture.bitmap.recycle()
239+
}
240+
241+
applyMaskingAndNotify(root, viewHierarchy)
242+
}
243+
)
244+
}
245+
118246
override fun onContentChanged() {
119247
contentChanged.set(true)
120248
}
@@ -123,6 +251,10 @@ internal class PixelCopyStrategy(
123251
return lastCaptureSuccessful.get()
124252
}
125253

254+
override fun hasSurfaceViews(): Boolean {
255+
return hasSurfaceViews.get()
256+
}
257+
126258
override fun emitLastScreenshot() {
127259
if (lastCaptureSuccessful() && !screenshot.isRecycled) {
128260
screenshotRecorderCallback?.onScreenshotRecorded(screenshot)
@@ -148,3 +280,38 @@ internal class PixelCopyStrategy(
148280
)
149281
}
150282
}
283+
284+
/**
285+
* Composites [sourceBitmap] (a SurfaceView capture) onto [destCanvas] (wrapping the recording
286+
* screenshot) using [destPaint] (expected to have DST_OVER xfermode), so the SurfaceView content
287+
* draws _behind_ existing Window content — filling the transparent holes the Window PixelCopy
288+
* leaves where SurfaceViews are.
289+
*
290+
* Extracted for testability — the compositing is pure drawing logic that can be driven with
291+
* hand-built bitmaps, while the surrounding [PixelCopyStrategy.captureSurfaceViews] flow depends on
292+
* a real SurfaceView producer that Robolectric cannot provide.
293+
*/
294+
internal fun compositeSurfaceViewInto(
295+
destCanvas: Canvas,
296+
destPaint: Paint,
297+
tmpSrc: Rect,
298+
tmpDst: RectF,
299+
sourceBitmap: Bitmap,
300+
sourceX: Int,
301+
sourceY: Int,
302+
windowX: Int,
303+
windowY: Int,
304+
scaleFactorX: Float,
305+
scaleFactorY: Float,
306+
) {
307+
val left = (sourceX - windowX) * scaleFactorX
308+
val top = (sourceY - windowY) * scaleFactorY
309+
tmpSrc.set(0, 0, sourceBitmap.width, sourceBitmap.height)
310+
tmpDst.set(
311+
left,
312+
top,
313+
left + sourceBitmap.width * scaleFactorX,
314+
top + sourceBitmap.height * scaleFactorY,
315+
)
316+
destCanvas.drawBitmap(sourceBitmap, tmpSrc, tmpDst, destPaint)
317+
}

‎sentry-android-replay/src/main/java/io/sentry/android/replay/screenshot/ScreenshotStrategy.kt‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -12,4 +12,11 @@ internal interface ScreenshotStrategy {
1212
fun lastCaptureSuccessful(): Boolean
1313

1414
fun emitLastScreenshot()
15+
16+
/**
17+
* Whether the last capture detected SurfaceViews that render independently of the View tree. When
18+
* true, the recorder bypasses the contentChanged guard since SurfaceView redraws don't trigger
19+
* ViewTreeObserver.OnDrawListener.
20+
*/
21+
fun hasSurfaceViews(): Boolean = false
1522
}

‎sentry-android-replay/src/main/java/io/sentry/android/replay/util/Views.kt‎

Lines changed: 10 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -34,10 +34,12 @@ import java.lang.NullPointerException
3434
* @param logger Logger for error reporting during Compose traversal
3535
*/
3636
@SuppressLint("UseKtx")
37+
@JvmOverloads
3738
internal fun View.traverse(
3839
parentNode: ViewHierarchyNode,
3940
options: SentryMaskingOptions,
4041
logger: ILogger,
42+
surfaceViewNodes: MutableList<ViewHierarchyNode.SurfaceViewHierarchyNode>? = null,
4143
) {
4244
if (this !is ViewGroup) {
4345
return
@@ -59,7 +61,14 @@ internal fun View.traverse(
5961
if (child != null) {
6062
val childNode = ViewHierarchyNode.fromView(child, parentNode, indexOfChild(child), options)
6163
childNodes.add(childNode)
62-
child.traverse(childNode, options, logger)
64+
if (
65+
surfaceViewNodes != null &&
66+
childNode is ViewHierarchyNode.SurfaceViewHierarchyNode &&
67+
childNode.isVisible
68+
) {
69+
surfaceViewNodes.add(childNode)
70+
}
71+
child.traverse(childNode, options, logger, surfaceViewNodes)
6372
}
6473
}
6574
parentNode.children = childNodes

‎sentry-android-replay/src/main/java/io/sentry/android/replay/viewhierarchy/ViewHierarchyNode.kt‎

Lines changed: 48 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@ package io.sentry.android.replay.viewhierarchy
33
import android.annotation.SuppressLint
44
import android.annotation.TargetApi
55
import android.graphics.Rect
6+
import android.view.SurfaceView
67
import android.view.View
78
import android.view.ViewParent
89
import android.widget.ImageView
@@ -15,6 +16,7 @@ import io.sentry.android.replay.util.isMaskable
1516
import io.sentry.android.replay.util.isVisibleToUser
1617
import io.sentry.android.replay.util.toOpaque
1718
import io.sentry.android.replay.util.totalPaddingTopSafe
19+
import java.lang.ref.WeakReference
1820

1921
@SuppressLint("UseRequiresApi")
2022
@TargetApi(26)
@@ -121,6 +123,34 @@ internal sealed class ViewHierarchyNode(
121123
visibleRect,
122124
)
123125

126+
class SurfaceViewHierarchyNode(
127+
val surfaceViewRef: WeakReference<SurfaceView>,
128+
x: Float,
129+
y: Float,
130+
width: Int,
131+
height: Int,
132+
elevation: Float,
133+
distance: Int,
134+
parent: ViewHierarchyNode? = null,
135+
shouldMask: Boolean = false,
136+
isImportantForContentCapture: Boolean = false,
137+
isVisible: Boolean = false,
138+
visibleRect: Rect? = null,
139+
) :
140+
ViewHierarchyNode(
141+
x,
142+
y,
143+
width,
144+
height,
145+
elevation,
146+
distance,
147+
parent,
148+
shouldMask,
149+
isImportantForContentCapture,
150+
isVisible,
151+
visibleRect,
152+
)
153+
124154
/**
125155
* Basically replicating this:
126156
* https://developer.android.com/reference/android/view/View#isImportantForContentCapture() but
@@ -379,6 +409,24 @@ internal sealed class ViewHierarchyNode(
379409
visibleRect = visibleRect,
380410
)
381411
}
412+
413+
is SurfaceView -> {
414+
parent?.setImportantForCaptureToAncestors(true)
415+
return SurfaceViewHierarchyNode(
416+
surfaceViewRef = WeakReference(view),
417+
x = view.x,
418+
y = view.y,
419+
width = view.width,
420+
height = view.height,
421+
elevation = (parent?.elevation ?: 0f) + view.elevation,
422+
distance = distance,
423+
parent = parent,
424+
shouldMask = shouldMask,
425+
isImportantForContentCapture = true,
426+
isVisible = isVisible,
427+
visibleRect = visibleRect,
428+
)
429+
}
382430
}
383431

384432
return GenericViewHierarchyNode(

0 commit comments

Comments
 (0)