Skip to content

Commit 4c3ac81

Browse files
clainclycopybara-github
claincly
authored and
copybara-github
committed
Add a class wrapping GlTextureInfo and presentation time
The grouping is used in some places and will be used in dynamic effect update PiperOrigin-RevId: 729569353
1 parent 7c2e8c1 commit 4c3ac81

File tree

6 files changed

+87
-67
lines changed

6 files changed

+87
-67
lines changed

‎libraries/effect/src/main/java/androidx/media3/effect/DefaultVideoCompositor.java‎

Lines changed: 17 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -201,8 +201,7 @@ public synchronized void queueInputTexture(
201201
InputFrameInfo inputFrameInfo =
202202
new InputFrameInfo(
203203
textureProducer,
204-
inputTexture,
205-
presentationTimeUs,
204+
new TimedGlTextureInfo(inputTexture, presentationTimeUs),
206205
settings.getOverlaySettings(inputIndex, presentationTimeUs));
207206
inputSource.frameInfos.add(inputFrameInfo);
208207

@@ -260,13 +259,15 @@ private synchronized void releaseExcessFramesInSecondaryStream(InputSource secon
260259
// nextTimestampToComposite.
261260
@Nullable InputFrameInfo nextPrimaryFrame = primaryInputSource.frameInfos.peek();
262261
long nextTimestampToComposite =
263-
nextPrimaryFrame != null ? nextPrimaryFrame.presentationTimeUs : C.TIME_UNSET;
262+
nextPrimaryFrame != null
263+
? nextPrimaryFrame.timedGlTextureInfo.presentationTimeUs
264+
: C.TIME_UNSET;
264265

265266
int numberOfSecondaryFramesBeforeOrAtNextTargetTimestamp =
266267
Iterables.size(
267268
Iterables.filter(
268269
secondaryInputSource.frameInfos,
269-
frame -> frame.presentationTimeUs <= nextTimestampToComposite));
270+
frame -> frame.timedGlTextureInfo.presentationTimeUs <= nextTimestampToComposite));
270271
releaseFrames(
271272
secondaryInputSource,
272273
/* numberOfFramesToRelease= */ max(
@@ -277,7 +278,7 @@ private synchronized void releaseFrames(InputSource inputSource, int numberOfFra
277278
for (int i = 0; i < numberOfFramesToRelease; i++) {
278279
InputFrameInfo frameInfoToRelease = inputSource.frameInfos.remove();
279280
frameInfoToRelease.textureProducer.releaseOutputTexture(
280-
frameInfoToRelease.presentationTimeUs);
281+
frameInfoToRelease.timedGlTextureInfo.presentationTimeUs);
281282
}
282283
}
283284

@@ -302,15 +303,15 @@ private synchronized void maybeComposite()
302303

303304
ImmutableList.Builder<Size> inputSizes = new ImmutableList.Builder<>();
304305
for (int i = 0; i < framesToComposite.size(); i++) {
305-
GlTextureInfo texture = framesToComposite.get(i).texture;
306+
GlTextureInfo texture = framesToComposite.get(i).timedGlTextureInfo.glTextureInfo;
306307
inputSizes.add(new Size(texture.width, texture.height));
307308
}
308309
Size outputSize = settings.getOutputSize(inputSizes.build());
309310
outputTexturePool.ensureConfigured(
310311
glObjectsProvider, outputSize.getWidth(), outputSize.getHeight());
311312

312313
GlTextureInfo outputTexture = outputTexturePool.useTexture();
313-
long outputPresentationTimestampUs = primaryInputFrame.presentationTimeUs;
314+
long outputPresentationTimestampUs = primaryInputFrame.timedGlTextureInfo.presentationTimeUs;
314315
outputTextureTimestamps.add(outputPresentationTimestampUs);
315316

316317
compositorGlProgram.drawFrame(framesToComposite, outputTexture);
@@ -369,16 +370,18 @@ private synchronized ImmutableList<InputFrameInfo> getFramesToComposite() {
369370
Iterator<InputFrameInfo> frameInfosIterator = secondaryInputSource.frameInfos.iterator();
370371
while (frameInfosIterator.hasNext()) {
371372
InputFrameInfo candidateFrame = frameInfosIterator.next();
372-
long candidateTimestampUs = candidateFrame.presentationTimeUs;
373+
long candidateTimestampUs = candidateFrame.timedGlTextureInfo.presentationTimeUs;
373374
long candidateAbsDistance =
374-
abs(candidateTimestampUs - primaryFrameToComposite.presentationTimeUs);
375+
abs(
376+
candidateTimestampUs
377+
- primaryFrameToComposite.timedGlTextureInfo.presentationTimeUs);
375378

376379
if (candidateAbsDistance < minTimeDiffFromPrimaryUs) {
377380
minTimeDiffFromPrimaryUs = candidateAbsDistance;
378381
secondaryFrameToComposite = candidateFrame;
379382
}
380383

381-
if (candidateTimestampUs > primaryFrameToComposite.presentationTimeUs
384+
if (candidateTimestampUs > primaryFrameToComposite.timedGlTextureInfo.presentationTimeUs
382385
|| (!frameInfosIterator.hasNext() && secondaryInputSource.isInputEnded)) {
383386
framesToComposite.add(checkNotNull(secondaryFrameToComposite));
384387
break;
@@ -503,7 +506,7 @@ private void ensureConfigured() throws VideoFrameProcessingException, GlUtil.GlE
503506

504507
private void blendOntoFocusedTexture(InputFrameInfo inputFrameInfo) throws GlUtil.GlException {
505508
GlProgram glProgram = checkNotNull(this.glProgram);
506-
GlTextureInfo inputTexture = inputFrameInfo.texture;
509+
GlTextureInfo inputTexture = inputFrameInfo.timedGlTextureInfo.glTextureInfo;
507510
glProgram.setSamplerTexIdUniform("uTexSampler", inputTexture.texId, /* texUnitIndex= */ 0);
508511
float[] transformationMatrix =
509512
overlayMatrixProvider.getTransformationMatrix(
@@ -537,18 +540,15 @@ public InputSource() {
537540
/** Holds information on a frame and how to release it. */
538541
private static final class InputFrameInfo {
539542
public final GlTextureProducer textureProducer;
540-
public final GlTextureInfo texture;
541-
public final long presentationTimeUs;
543+
public final TimedGlTextureInfo timedGlTextureInfo;
542544
public final OverlaySettings overlaySettings;
543545

544546
public InputFrameInfo(
545547
GlTextureProducer textureProducer,
546-
GlTextureInfo texture,
547-
long presentationTimeUs,
548+
TimedGlTextureInfo timedGlTextureInfo,
548549
OverlaySettings overlaySettings) {
549550
this.textureProducer = textureProducer;
550-
this.texture = texture;
551-
this.presentationTimeUs = presentationTimeUs;
551+
this.timedGlTextureInfo = timedGlTextureInfo;
552552
this.overlaySettings = overlaySettings;
553553
}
554554
}

‎libraries/effect/src/main/java/androidx/media3/effect/FinalShaderProgramWrapper.java‎

Lines changed: 5 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -28,7 +28,6 @@
2828
import android.opengl.EGLDisplay;
2929
import android.opengl.EGLExt;
3030
import android.opengl.EGLSurface;
31-
import android.util.Pair;
3231
import android.view.Surface;
3332
import androidx.annotation.Nullable;
3433
import androidx.media3.common.C;
@@ -86,7 +85,7 @@ interface OnInputStreamProcessedListener {
8685
private final VideoFrameProcessingTaskExecutor videoFrameProcessingTaskExecutor;
8786
private final Executor videoFrameProcessorListenerExecutor;
8887
private final VideoFrameProcessor.Listener videoFrameProcessorListener;
89-
private final Queue<Pair<GlTextureInfo, Long>> availableFrames;
88+
private final Queue<TimedGlTextureInfo> availableFrames;
9089
private final TexturePool outputTexturePool;
9190
private final LongArrayQueue outputTextureTimestamps; // Synchronized with outputTexturePool.
9291
private final LongArrayQueue syncObjects;
@@ -221,7 +220,7 @@ public void queueInputFrame(
221220
presentationTimeUs,
222221
/* renderTimeNs= */ presentationTimeUs * 1000);
223222
} else {
224-
availableFrames.add(Pair.create(inputTexture, presentationTimeUs));
223+
availableFrames.add(new TimedGlTextureInfo(inputTexture, presentationTimeUs));
225224
}
226225
inputListener.onReadyToAcceptInputFrame();
227226
} else {
@@ -307,11 +306,11 @@ public void renderOutputFrame(GlObjectsProvider glObjectsProvider, long renderTi
307306
return;
308307
}
309308
checkState(!renderFramesAutomatically);
310-
Pair<GlTextureInfo, Long> oldestAvailableFrame = availableFrames.remove();
309+
TimedGlTextureInfo oldestAvailableFrame = availableFrames.remove();
311310
renderFrame(
312311
glObjectsProvider,
313-
/* inputTexture= */ oldestAvailableFrame.first,
314-
/* presentationTimeUs= */ oldestAvailableFrame.second,
312+
oldestAvailableFrame.glTextureInfo,
313+
oldestAvailableFrame.presentationTimeUs,
315314
renderTimeNs);
316315
if (availableFrames.isEmpty() && isInputStreamEndedWithPendingAvailableFrames) {
317316
checkNotNull(onInputStreamProcessedListener).onInputStreamProcessed();

‎libraries/effect/src/main/java/androidx/media3/effect/FrameConsumptionManager.java‎

Lines changed: 7 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,6 @@
1515
*/
1616
package androidx.media3.effect;
1717

18-
import android.util.Pair;
1918
import androidx.annotation.GuardedBy;
2019
import androidx.annotation.Nullable;
2120
import androidx.media3.common.C;
@@ -39,7 +38,7 @@
3938
private final VideoFrameProcessingTaskExecutor videoFrameProcessingTaskExecutor;
4039

4140
@GuardedBy("this")
42-
private final Queue<Pair<GlTextureInfo, Long>> availableFrames;
41+
private final Queue<TimedGlTextureInfo> availableFrames;
4342

4443
@GuardedBy("this")
4544
private int consumingGlShaderProgramInputCapacity;
@@ -63,7 +62,7 @@ public FrameConsumptionManager(
6362

6463
@Override
6564
public synchronized void onReadyToAcceptInputFrame() {
66-
@Nullable Pair<GlTextureInfo, Long> pendingFrame = availableFrames.poll();
65+
@Nullable TimedGlTextureInfo pendingFrame = availableFrames.poll();
6766
if (pendingFrame == null) {
6867
consumingGlShaderProgramInputCapacity++;
6968
return;
@@ -72,11 +71,9 @@ public synchronized void onReadyToAcceptInputFrame() {
7271
videoFrameProcessingTaskExecutor.submit(
7372
() ->
7473
consumingGlShaderProgram.queueInputFrame(
75-
glObjectsProvider,
76-
/* inputTexture= */ pendingFrame.first,
77-
/* presentationTimeUs= */ pendingFrame.second));
78-
@Nullable Pair<GlTextureInfo, Long> nextPendingFrame = availableFrames.peek();
79-
if (nextPendingFrame != null && nextPendingFrame.second == C.TIME_END_OF_SOURCE) {
74+
glObjectsProvider, pendingFrame.glTextureInfo, pendingFrame.presentationTimeUs));
75+
@Nullable TimedGlTextureInfo nextPendingFrame = availableFrames.peek();
76+
if (nextPendingFrame != null && nextPendingFrame.presentationTimeUs == C.TIME_END_OF_SOURCE) {
8077
videoFrameProcessingTaskExecutor.submit(
8178
consumingGlShaderProgram::signalEndOfCurrentInputStream);
8279
availableFrames.remove();
@@ -97,7 +94,7 @@ public synchronized void queueInputFrame(GlTextureInfo inputTexture, long presen
9794
glObjectsProvider, inputTexture, presentationTimeUs));
9895
consumingGlShaderProgramInputCapacity--;
9996
} else {
100-
availableFrames.add(Pair.create(inputTexture, presentationTimeUs));
97+
availableFrames.add(new TimedGlTextureInfo(inputTexture, presentationTimeUs));
10198
}
10299
}
103100

@@ -107,7 +104,7 @@ public synchronized void queueInputFrame(GlTextureInfo inputTexture, long presen
107104
*/
108105
public synchronized void signalEndOfCurrentStream() {
109106
if (!availableFrames.isEmpty()) {
110-
availableFrames.add(Pair.create(GlTextureInfo.UNSET, C.TIME_END_OF_SOURCE));
107+
availableFrames.add(new TimedGlTextureInfo(GlTextureInfo.UNSET, C.TIME_END_OF_SOURCE));
111108
} else {
112109
videoFrameProcessingTaskExecutor.submit(
113110
consumingGlShaderProgram::signalEndOfCurrentInputStream);

‎libraries/effect/src/main/java/androidx/media3/effect/MultipleInputVideoGraph.java‎

Lines changed: 3 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -87,7 +87,7 @@ public abstract class MultipleInputVideoGraph implements VideoGraph {
8787
private final ExecutorService sharedExecutorService;
8888

8989
private final DefaultVideoFrameProcessor.Factory videoFrameProcessorFactory;
90-
private final Queue<CompositorOutputTextureInfo> compositorOutputTextures;
90+
private final Queue<TimedGlTextureInfo> compositorOutputTextures;
9191
private final SparseArray<CompositorOutputTextureRelease> compositorOutputTextureReleases;
9292

9393
private final long initialTimestampOffsetUs;
@@ -363,8 +363,7 @@ private void processCompositorOutputTexture(
363363
DebugTraceUtil.logEvent(
364364
COMPONENT_COMPOSITOR, EVENT_OUTPUT_TEXTURE_RENDERED, presentationTimeUs);
365365

366-
compositorOutputTextures.add(
367-
new CompositorOutputTextureInfo(outputTexture, presentationTimeUs));
366+
compositorOutputTextures.add(new TimedGlTextureInfo(outputTexture, presentationTimeUs));
368367
compositorOutputTextureReleases.put(
369368
outputTexture.texId,
370369
new CompositorOutputTextureRelease(textureProducer, presentationTimeUs));
@@ -421,7 +420,7 @@ private void queueCompositionOutputInternal() {
421420
return;
422421
}
423422

424-
@Nullable CompositorOutputTextureInfo outputTexture = compositorOutputTextures.peek();
423+
@Nullable TimedGlTextureInfo outputTexture = compositorOutputTextures.peek();
425424
if (outputTexture == null) {
426425
return;
427426
}
@@ -446,16 +445,6 @@ private void handleVideoFrameProcessingException(Exception e) {
446445
: VideoFrameProcessingException.from(e)));
447446
}
448447

449-
private static final class CompositorOutputTextureInfo {
450-
public final GlTextureInfo glTextureInfo;
451-
public final long presentationTimeUs;
452-
453-
private CompositorOutputTextureInfo(GlTextureInfo glTextureInfo, long presentationTimeUs) {
454-
this.glTextureInfo = glTextureInfo;
455-
this.presentationTimeUs = presentationTimeUs;
456-
}
457-
}
458-
459448
private static final class CompositorOutputTextureRelease {
460449
private final GlTextureProducer textureProducer;
461450
private final long presentationTimeUs;

‎libraries/effect/src/main/java/androidx/media3/effect/QueuingGlShaderProgram.java‎

Lines changed: 22 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -142,7 +142,7 @@ void finishProcessingAndBlend(GlTextureInfo outputFrame, long presentationTimeUs
142142

143143
private final ConcurrentEffect<T> concurrentEffect;
144144
private final TexturePool outputTexturePool;
145-
private final Queue<TimedTextureInfo<T>> frameQueue;
145+
private final Queue<QueuedFrame<T>> frameQueue;
146146
private InputListener inputListener;
147147
private OutputListener outputListener;
148148
private ErrorListener errorListener;
@@ -226,7 +226,8 @@ public void queueInputFrame(
226226

227227
Future<T> task =
228228
concurrentEffect.queueInputFrame(glObjectsProvider, outputTexture, presentationTimeUs);
229-
frameQueue.add(new TimedTextureInfo<T>(outputTexture, presentationTimeUs, task));
229+
frameQueue.add(
230+
new QueuedFrame<T>(new TimedGlTextureInfo(outputTexture, presentationTimeUs), task));
230231

231232
inputListener.onInputFrameProcessed(inputTexture);
232233

@@ -297,25 +298,28 @@ public void release() throws VideoFrameProcessingException {
297298
* <p>Returns {@code false} if no more frames are available for output.
298299
*/
299300
private boolean outputOneFrame() {
300-
TimedTextureInfo<T> timedTextureInfo = frameQueue.poll();
301-
if (timedTextureInfo == null) {
301+
QueuedFrame<T> queuedFrame = frameQueue.poll();
302+
if (queuedFrame == null) {
302303
return false;
303304
}
304305
try {
305306
T result =
306307
Futures.getChecked(
307-
timedTextureInfo.task,
308+
queuedFrame.task,
308309
VideoFrameProcessingException.class,
309310
PROCESSING_TIMEOUT_MS,
310311
TimeUnit.MILLISECONDS);
311312
GlUtil.focusFramebufferUsingCurrentContext(
312-
timedTextureInfo.textureInfo.fboId,
313-
timedTextureInfo.textureInfo.width,
314-
timedTextureInfo.textureInfo.height);
313+
queuedFrame.timedGlTextureInfo.glTextureInfo.fboId,
314+
queuedFrame.timedGlTextureInfo.glTextureInfo.width,
315+
queuedFrame.timedGlTextureInfo.glTextureInfo.height);
315316
concurrentEffect.finishProcessingAndBlend(
316-
timedTextureInfo.textureInfo, timedTextureInfo.presentationTimeUs, result);
317+
queuedFrame.timedGlTextureInfo.glTextureInfo,
318+
queuedFrame.timedGlTextureInfo.presentationTimeUs,
319+
result);
317320
outputListener.onOutputFrameAvailable(
318-
timedTextureInfo.textureInfo, timedTextureInfo.presentationTimeUs);
321+
queuedFrame.timedGlTextureInfo.glTextureInfo,
322+
queuedFrame.timedGlTextureInfo.presentationTimeUs);
319323
return true;
320324
} catch (GlUtil.GlException | VideoFrameProcessingException e) {
321325
onError(e);
@@ -324,9 +328,9 @@ private boolean outputOneFrame() {
324328
}
325329

326330
private void cancelProcessingOfPendingFrames() {
327-
TimedTextureInfo<T> timedTextureInfo;
328-
while ((timedTextureInfo = frameQueue.poll()) != null) {
329-
timedTextureInfo.task.cancel(/* mayInterruptIfRunning= */ false);
331+
QueuedFrame<T> queuedFrame;
332+
while ((queuedFrame = frameQueue.poll()) != null) {
333+
queuedFrame.task.cancel(/* mayInterruptIfRunning= */ false);
330334
}
331335
}
332336

@@ -335,14 +339,12 @@ private void onError(Exception e) {
335339
() -> errorListener.onError(VideoFrameProcessingException.from(e)));
336340
}
337341

338-
private static class TimedTextureInfo<T> {
339-
final GlTextureInfo textureInfo;
340-
final long presentationTimeUs;
341-
final Future<T> task;
342+
private static final class QueuedFrame<T> {
343+
public final TimedGlTextureInfo timedGlTextureInfo;
344+
public final Future<T> task;
342345

343-
TimedTextureInfo(GlTextureInfo textureInfo, long presentationTimeUs, Future<T> task) {
344-
this.textureInfo = textureInfo;
345-
this.presentationTimeUs = presentationTimeUs;
346+
public QueuedFrame(TimedGlTextureInfo timedGlTextureInfo, Future<T> task) {
347+
this.timedGlTextureInfo = timedGlTextureInfo;
346348
this.task = task;
347349
}
348350
}

‎libraries/effect/src/main/java/androidx/media3/effect/TimedGlTextureInfo.java‎

Lines changed: 33 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,33 @@
1+
/*
2+
* Copyright 2025 The Android Open Source Project
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
package androidx.media3.effect;
17+
18+
import androidx.media3.common.GlTextureInfo;
19+
20+
/* package */ final class TimedGlTextureInfo {
21+
22+
/** The {@link GlTextureInfo}. */
23+
public final GlTextureInfo glTextureInfo;
24+
25+
/** The designated presentation time with the texture, in microseconds. */
26+
public final long presentationTimeUs;
27+
28+
/** Creates a new instance. */
29+
public TimedGlTextureInfo(GlTextureInfo glTextureInfo, long presentationTimeUs) {
30+
this.glTextureInfo = glTextureInfo;
31+
this.presentationTimeUs = presentationTimeUs;
32+
}
33+
}

0 commit comments

Comments
 (0)