Skip to content

Commit 1acd813

Browse files
romtsnromtsn
authored
Merge df58de1 into 6b019b7 
2 parents 6b019b7 + df58de1 commit 1acd813

7 files changed

Lines changed: 251 additions & 25 deletions

File tree

‎CHANGELOG.md‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,11 @@
22

33
## 8.40.0
44

5+
### Features
6+
7+
- Session Replay: experimental support for capturing `SurfaceView` content (e.g. Unity, video players, maps) ([#5333](https://github.com/getsentry/sentry-java/pull/5333))
8+
- To enable, set `options.sessionReplay.isCaptureSurfaceViews = true`
9+
510
### Fixes
611

712
- Fix `NoSuchMethodError` for `LayoutCoordinates.localBoundingBoxOf$default` on Compose touch dispatch with AGP 8.13 and `minSdk < 24` ([#5302](https://github.com/getsentry/sentry-java/pull/5302))

‎sentry-android-replay/src/main/java/io/sentry/android/replay/ScreenshotRecorder.kt‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -70,7 +70,7 @@ internal class ScreenshotRecorder(
7070
)
7171
}
7272

73-
if (!contentChanged.get()) {
73+
if (!contentChanged.get() && !screenshotStrategy.hasSurfaceViews()) {
7474
screenshotStrategy.emitLastScreenshot()
7575
return
7676
}

‎sentry-android-replay/src/main/java/io/sentry/android/replay/screenshot/PixelCopyStrategy.kt‎

Lines changed: 152 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,13 @@ package io.sentry.android.replay.screenshot
22

33
import android.annotation.SuppressLint
44
import android.graphics.Bitmap
5+
import android.graphics.Canvas
56
import android.graphics.Matrix
7+
import android.graphics.Paint
8+
import android.graphics.PorterDuff
9+
import android.graphics.PorterDuffXfermode
10+
import android.graphics.Rect
11+
import android.graphics.RectF
612
import android.view.PixelCopy
713
import android.view.View
814
import io.sentry.SentryLevel.DEBUG
@@ -19,6 +25,7 @@ import io.sentry.android.replay.util.ReplayRunnable
1925
import io.sentry.android.replay.util.traverse
2026
import io.sentry.android.replay.viewhierarchy.ViewHierarchyNode
2127
import java.util.concurrent.atomic.AtomicBoolean
28+
import java.util.concurrent.atomic.AtomicInteger
2229
import kotlin.LazyThreadSafetyMode.NONE
2330

2431
@SuppressLint("UseKtx")
@@ -40,6 +47,16 @@ internal class PixelCopyStrategy(
4047
private val maskRenderer = MaskRenderer()
4148
private val contentChanged = AtomicBoolean(false)
4249
private val isClosed = AtomicBoolean(false)
50+
private val hasSurfaceViews = AtomicBoolean(false)
51+
private val dstOverPaint by
52+
lazy(NONE) { Paint().apply { xfermode = PorterDuffXfermode(PorterDuff.Mode.DST_OVER) } }
53+
private val screenshotCanvas by lazy(NONE) { Canvas(screenshot) }
54+
private val tmpSrcRect = Rect()
55+
private val tmpDstRect = RectF()
56+
private val windowLocation = IntArray(2)
57+
private val svLocation = IntArray(2)
58+
59+
private class SurfaceViewCapture(val bitmap: Bitmap, val x: Int, val y: Int)
4360

4461
@SuppressLint("NewApi")
4562
override fun capture(root: View) {
@@ -81,31 +98,22 @@ internal class PixelCopyStrategy(
8198

8299
// TODO: disableAllMasking here and dont traverse?
83100
val viewHierarchy = ViewHierarchyNode.fromView(root, null, 0, options.sessionReplay)
84-
root.traverse(viewHierarchy, options.sessionReplay, options.logger)
85-
86-
executor.submit(
87-
ReplayRunnable("screenshot_recorder.mask") {
88-
if (isClosed.get() || screenshot.isRecycled) {
89-
options.logger.log(DEBUG, "PixelCopyStrategy is closed, skipping masking")
90-
return@ReplayRunnable
91-
}
101+
val captureSurfaceViewsEnabled = options.sessionReplay.isCaptureSurfaceViews
102+
val surfaceViewNodes =
103+
if (captureSurfaceViewsEnabled) {
104+
mutableListOf<ViewHierarchyNode.SurfaceViewHierarchyNode>()
105+
} else {
106+
null
107+
}
108+
root.traverse(viewHierarchy, options.sessionReplay, options.logger, surfaceViewNodes)
92109

93-
val debugMasks = maskRenderer.renderMasks(screenshot, viewHierarchy, prescaledMatrix)
110+
hasSurfaceViews.set(surfaceViewNodes?.isNotEmpty() == true)
94111

95-
if (options.replayController.isDebugMaskingOverlayEnabled()) {
96-
mainLooperHandler.post {
97-
if (debugOverlayDrawable.callback == null) {
98-
root.overlay.add(debugOverlayDrawable)
99-
}
100-
debugOverlayDrawable.updateMasks(debugMasks)
101-
root.postInvalidate()
102-
}
103-
}
104-
screenshotRecorderCallback?.onScreenshotRecorded(screenshot)
105-
lastCaptureSuccessful.set(true)
106-
contentChanged.set(false)
107-
}
108-
)
112+
if (surfaceViewNodes.isNullOrEmpty()) {
113+
submitMaskingAndCallback(root, viewHierarchy)
114+
} else {
115+
captureSurfaceViews(root, surfaceViewNodes, viewHierarchy)
116+
}
109117
},
110118
mainLooperHandler.handler,
111119
)
@@ -115,6 +123,123 @@ internal class PixelCopyStrategy(
115123
}
116124
}
117125

126+
private fun submitMaskingAndCallback(root: View, viewHierarchy: ViewHierarchyNode) {
127+
executor.submit(
128+
ReplayRunnable("screenshot_recorder.mask") { applyMaskingAndNotify(root, viewHierarchy) }
129+
)
130+
}
131+
132+
private fun applyMaskingAndNotify(root: View, viewHierarchy: ViewHierarchyNode) {
133+
if (isClosed.get() || screenshot.isRecycled) {
134+
options.logger.log(DEBUG, "PixelCopyStrategy is closed, skipping masking")
135+
return
136+
}
137+
138+
val debugMasks = maskRenderer.renderMasks(screenshot, viewHierarchy, prescaledMatrix)
139+
140+
if (options.replayController.isDebugMaskingOverlayEnabled()) {
141+
mainLooperHandler.post {
142+
if (debugOverlayDrawable.callback == null) {
143+
root.overlay.add(debugOverlayDrawable)
144+
}
145+
debugOverlayDrawable.updateMasks(debugMasks)
146+
root.postInvalidate()
147+
}
148+
}
149+
screenshotRecorderCallback?.onScreenshotRecorded(screenshot)
150+
lastCaptureSuccessful.set(true)
151+
contentChanged.set(false)
152+
}
153+
154+
@SuppressLint("NewApi")
155+
private fun captureSurfaceViews(
156+
root: View,
157+
surfaceViewNodes: List<ViewHierarchyNode.SurfaceViewHierarchyNode>,
158+
viewHierarchy: ViewHierarchyNode,
159+
) {
160+
root.getLocationOnScreen(windowLocation)
161+
162+
val captures = arrayOfNulls<SurfaceViewCapture>(surfaceViewNodes.size)
163+
val remaining = AtomicInteger(surfaceViewNodes.size)
164+
165+
fun onCaptureComplete() {
166+
if (remaining.decrementAndGet() == 0) {
167+
compositeSurfaceViewsAndMask(root, captures, viewHierarchy)
168+
}
169+
}
170+
171+
for ((index, node) in surfaceViewNodes.withIndex()) {
172+
val surfaceView = node.surfaceViewRef.get()
173+
if (surfaceView == null || !surfaceView.holder.surface.isValid) {
174+
onCaptureComplete()
175+
continue
176+
}
177+
178+
try {
179+
val svBitmap =
180+
Bitmap.createBitmap(surfaceView.width, surfaceView.height, Bitmap.Config.ARGB_8888)
181+
182+
surfaceView.getLocationOnScreen(svLocation)
183+
val capturedX = svLocation[0]
184+
val capturedY = svLocation[1]
185+
186+
PixelCopy.request(
187+
surfaceView,
188+
svBitmap,
189+
{ copyResult: Int ->
190+
if (copyResult == PixelCopy.SUCCESS) {
191+
captures[index] = SurfaceViewCapture(svBitmap, capturedX, capturedY)
192+
} else {
193+
svBitmap.recycle()
194+
options.logger.log(INFO, "Failed to capture SurfaceView: %d", copyResult)
195+
}
196+
onCaptureComplete()
197+
},
198+
mainLooperHandler.handler,
199+
)
200+
} catch (e: Throwable) {
201+
options.logger.log(WARNING, "Failed to capture SurfaceView", e)
202+
onCaptureComplete()
203+
}
204+
}
205+
}
206+
207+
private fun compositeSurfaceViewsAndMask(
208+
root: View,
209+
captures: Array<SurfaceViewCapture?>,
210+
viewHierarchy: ViewHierarchyNode,
211+
) {
212+
executor.submit(
213+
ReplayRunnable("screenshot_recorder.composite") {
214+
if (isClosed.get() || screenshot.isRecycled) {
215+
options.logger.log(DEBUG, "PixelCopyStrategy is closed, skipping compositing")
216+
return@ReplayRunnable
217+
}
218+
219+
for (capture in captures) {
220+
if (capture == null) continue
221+
if (capture.bitmap.isRecycled) continue
222+
223+
val left = (capture.x - windowLocation[0]) * config.scaleFactorX
224+
val top = (capture.y - windowLocation[1]) * config.scaleFactorY
225+
tmpSrcRect.set(0, 0, capture.bitmap.width, capture.bitmap.height)
226+
tmpDstRect.set(
227+
left,
228+
top,
229+
left + capture.bitmap.width * config.scaleFactorX,
230+
top + capture.bitmap.height * config.scaleFactorY,
231+
)
232+
233+
// DST_OVER draws the SurfaceView content behind the existing Window content
234+
screenshotCanvas.drawBitmap(capture.bitmap, tmpSrcRect, tmpDstRect, dstOverPaint)
235+
capture.bitmap.recycle()
236+
}
237+
238+
applyMaskingAndNotify(root, viewHierarchy)
239+
}
240+
)
241+
}
242+
118243
override fun onContentChanged() {
119244
contentChanged.set(true)
120245
}
@@ -123,6 +248,10 @@ internal class PixelCopyStrategy(
123248
return lastCaptureSuccessful.get()
124249
}
125250

251+
override fun hasSurfaceViews(): Boolean {
252+
return hasSurfaceViews.get()
253+
}
254+
126255
override fun emitLastScreenshot() {
127256
if (lastCaptureSuccessful() && !screenshot.isRecycled) {
128257
screenshotRecorderCallback?.onScreenshotRecorded(screenshot)

‎sentry-android-replay/src/main/java/io/sentry/android/replay/screenshot/ScreenshotStrategy.kt‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -12,4 +12,11 @@ internal interface ScreenshotStrategy {
1212
fun lastCaptureSuccessful(): Boolean
1313

1414
fun emitLastScreenshot()
15+
16+
/**
17+
* Whether the last capture detected SurfaceViews that render independently of the View tree. When
18+
* true, the recorder bypasses the contentChanged guard since SurfaceView redraws don't trigger
19+
* ViewTreeObserver.OnDrawListener.
20+
*/
21+
fun hasSurfaceViews(): Boolean = false
1522
}

‎sentry-android-replay/src/main/java/io/sentry/android/replay/util/Views.kt‎

Lines changed: 10 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -34,10 +34,12 @@ import java.lang.NullPointerException
3434
* @param logger Logger for error reporting during Compose traversal
3535
*/
3636
@SuppressLint("UseKtx")
37+
@JvmOverloads
3738
internal fun View.traverse(
3839
parentNode: ViewHierarchyNode,
3940
options: SentryMaskingOptions,
4041
logger: ILogger,
42+
surfaceViewNodes: MutableList<ViewHierarchyNode.SurfaceViewHierarchyNode>? = null,
4143
) {
4244
if (this !is ViewGroup) {
4345
return
@@ -59,7 +61,14 @@ internal fun View.traverse(
5961
if (child != null) {
6062
val childNode = ViewHierarchyNode.fromView(child, parentNode, indexOfChild(child), options)
6163
childNodes.add(childNode)
62-
child.traverse(childNode, options, logger)
64+
if (
65+
surfaceViewNodes != null &&
66+
childNode is ViewHierarchyNode.SurfaceViewHierarchyNode &&
67+
childNode.isVisible
68+
) {
69+
surfaceViewNodes.add(childNode)
70+
}
71+
child.traverse(childNode, options, logger, surfaceViewNodes)
6372
}
6473
}
6574
parentNode.children = childNodes

‎sentry-android-replay/src/main/java/io/sentry/android/replay/viewhierarchy/ViewHierarchyNode.kt‎

Lines changed: 48 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@ package io.sentry.android.replay.viewhierarchy
33
import android.annotation.SuppressLint
44
import android.annotation.TargetApi
55
import android.graphics.Rect
6+
import android.view.SurfaceView
67
import android.view.View
78
import android.view.ViewParent
89
import android.widget.ImageView
@@ -15,6 +16,7 @@ import io.sentry.android.replay.util.isMaskable
1516
import io.sentry.android.replay.util.isVisibleToUser
1617
import io.sentry.android.replay.util.toOpaque
1718
import io.sentry.android.replay.util.totalPaddingTopSafe
19+
import java.lang.ref.WeakReference
1820

1921
@SuppressLint("UseRequiresApi")
2022
@TargetApi(26)
@@ -121,6 +123,34 @@ internal sealed class ViewHierarchyNode(
121123
visibleRect,
122124
)
123125

126+
class SurfaceViewHierarchyNode(
127+
val surfaceViewRef: WeakReference<SurfaceView>,
128+
x: Float,
129+
y: Float,
130+
width: Int,
131+
height: Int,
132+
elevation: Float,
133+
distance: Int,
134+
parent: ViewHierarchyNode? = null,
135+
shouldMask: Boolean = false,
136+
isImportantForContentCapture: Boolean = false,
137+
isVisible: Boolean = false,
138+
visibleRect: Rect? = null,
139+
) :
140+
ViewHierarchyNode(
141+
x,
142+
y,
143+
width,
144+
height,
145+
elevation,
146+
distance,
147+
parent,
148+
shouldMask,
149+
isImportantForContentCapture,
150+
isVisible,
151+
visibleRect,
152+
)
153+
124154
/**
125155
* Basically replicating this:
126156
* https://developer.android.com/reference/android/view/View#isImportantForContentCapture() but
@@ -379,6 +409,24 @@ internal sealed class ViewHierarchyNode(
379409
visibleRect = visibleRect,
380410
)
381411
}
412+
413+
is SurfaceView -> {
414+
parent?.setImportantForCaptureToAncestors(true)
415+
return SurfaceViewHierarchyNode(
416+
surfaceViewRef = WeakReference(view),
417+
x = view.x,
418+
y = view.y,
419+
width = view.width,
420+
height = view.height,
421+
elevation = (parent?.elevation ?: 0f) + view.elevation,
422+
distance = distance,
423+
parent = parent,
424+
shouldMask = shouldMask,
425+
isImportantForContentCapture = true,
426+
isVisible = isVisible,
427+
visibleRect = visibleRect,
428+
)
429+
}
382430
}
383431

384432
return GenericViewHierarchyNode(

‎sentry/src/main/java/io/sentry/SentryReplayOptions.java‎

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -146,6 +146,14 @@ public enum SentryReplayQuality {
146146
@ApiStatus.Experimental
147147
private @NotNull ScreenshotStrategyType screenshotStrategy = ScreenshotStrategyType.PIXEL_COPY;
148148

149+
/**
150+
* Whether to capture SurfaceView content (e.g. Unity, video players, maps) during replay
151+
* recording. When enabled, each SurfaceView in the view hierarchy will be captured separately via
152+
* PixelCopy and composited onto the screenshot. Only applies when {@link #screenshotStrategy} is
153+
* {@link ScreenshotStrategyType#PIXEL_COPY}. Default is disabled.
154+
*/
155+
@ApiStatus.Experimental private boolean captureSurfaceViews = false;
156+
149157
/**
150158
* Capture request and response details for XHR and fetch requests that match the given URLs.
151159
* Default is empty (network details not collected).
@@ -383,6 +391,26 @@ public void setScreenshotStrategy(final @NotNull ScreenshotStrategyType screensh
383391
this.screenshotStrategy = screenshotStrategy;
384392
}
385393

394+
/**
395+
* Whether SurfaceView capture is enabled. See {@link #captureSurfaceViews}.
396+
*
397+
* @return true if SurfaceView capture is enabled
398+
*/
399+
@ApiStatus.Experimental
400+
public boolean isCaptureSurfaceViews() {
401+
return captureSurfaceViews;
402+
}
403+
404+
/**
405+
* Enables or disables SurfaceView capture. See {@link #captureSurfaceViews}.
406+
*
407+
* @param captureSurfaceViews true to enable SurfaceView capture
408+
*/
409+
@ApiStatus.Experimental
410+
public void setCaptureSurfaceViews(final boolean captureSurfaceViews) {
411+
this.captureSurfaceViews = captureSurfaceViews;
412+
}
413+
386414
/**
387415
* Gets the list of URLs for which network request and response details should be captured.
388416
*

0 commit comments

Comments
 (0)