From de07b0ab9edf2d895f917caa52505df4c0068f42 Mon Sep 17 00:00:00 2001 From: Angelika Serwa Date: Wed, 12 Oct 2022 13:50:44 +0000 Subject: [PATCH 1/3] Fix simulcast using hardware encoder on Android --- media/engine/simulcast_encoder_adapter.cc | 9 +-------- .../src/java/org/webrtc/HardwareVideoEncoder.java | 10 ---------- 2 files changed, 1 insertion(+), 18 deletions(-) diff --git a/media/engine/simulcast_encoder_adapter.cc b/media/engine/simulcast_encoder_adapter.cc index e1dfc946092..d74f8c265e6 100644 --- a/media/engine/simulcast_encoder_adapter.cc +++ b/media/engine/simulcast_encoder_adapter.cc @@ -872,10 +872,6 @@ VideoEncoder::EncoderInfo SimulcastEncoderAdapter::GetEncoderInfo() const { primary_info.apply_alignment_to_all_simulcast_layers || fallback_info.apply_alignment_to_all_simulcast_layers; - if (!primary_info.supports_simulcast || !fallback_info.supports_simulcast) { - encoder_info.apply_alignment_to_all_simulcast_layers = true; - } - cached_encoder_contexts_.emplace_back(std::move(encoder_context)); OverrideFromFieldTrial(&encoder_info); @@ -930,10 +926,7 @@ VideoEncoder::EncoderInfo SimulcastEncoderAdapter::GetEncoderInfo() const { encoder_impl_info.requested_resolution_alignment); // request alignment on all layers if any of the encoders may need it, or // if any non-top layer encoder requests a non-trivial alignment. - if (encoder_impl_info.apply_alignment_to_all_simulcast_layers || - (encoder_impl_info.requested_resolution_alignment > 1 && - (codec_.simulcastStream[i].height < codec_.height || - codec_.simulcastStream[i].width < codec_.width))) { + if (encoder_impl_info.apply_alignment_to_all_simulcast_layers) { encoder_info.apply_alignment_to_all_simulcast_layers = true; } } diff --git a/sdk/android/src/java/org/webrtc/HardwareVideoEncoder.java b/sdk/android/src/java/org/webrtc/HardwareVideoEncoder.java index 743edb0b858..68d036c6580 100644 --- a/sdk/android/src/java/org/webrtc/HardwareVideoEncoder.java +++ b/sdk/android/src/java/org/webrtc/HardwareVideoEncoder.java @@ -211,11 +211,6 @@ public VideoCodecStatus initEncode(Settings settings, Callback callback) { this.callback = callback; automaticResizeOn = settings.automaticResizeOn; - if (settings.width % REQUIRED_RESOLUTION_ALIGNMENT != 0 - || settings.height % REQUIRED_RESOLUTION_ALIGNMENT != 0) { - Logging.e(TAG, "MediaCodec is only tested with resolutions that are 16x16 aligned."); - return VideoCodecStatus.ERR_SIZE; - } this.width = settings.width; this.height = settings.height; useSurfaceMode = canUseSurface(); @@ -528,11 +523,6 @@ private VideoCodecStatus resetCodec(int newWidth, int newHeight, boolean newUseS return status; } - if (newWidth % REQUIRED_RESOLUTION_ALIGNMENT != 0 - || newHeight % REQUIRED_RESOLUTION_ALIGNMENT != 0) { - Logging.e(TAG, "MediaCodec is only tested with resolutions that are 16x16 aligned."); - return VideoCodecStatus.ERR_SIZE; - } width = newWidth; height = newHeight; useSurfaceMode = newUseSurfaceMode; From 0adde7eeb11978b8553a36e89bf4cc67e64e1495 Mon Sep 17 00:00:00 2001 From: Angelika Serwa Date: Thu, 17 Nov 2022 15:52:11 +0000 Subject: [PATCH 2/3] Revert disabling apply_alignment_to_all_simulcast_layers --- media/engine/simulcast_encoder_adapter.cc | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/media/engine/simulcast_encoder_adapter.cc b/media/engine/simulcast_encoder_adapter.cc index d74f8c265e6..e1dfc946092 100644 --- a/media/engine/simulcast_encoder_adapter.cc +++ b/media/engine/simulcast_encoder_adapter.cc @@ -872,6 +872,10 @@ VideoEncoder::EncoderInfo SimulcastEncoderAdapter::GetEncoderInfo() const { primary_info.apply_alignment_to_all_simulcast_layers || fallback_info.apply_alignment_to_all_simulcast_layers; + if (!primary_info.supports_simulcast || !fallback_info.supports_simulcast) { + encoder_info.apply_alignment_to_all_simulcast_layers = true; + } + cached_encoder_contexts_.emplace_back(std::move(encoder_context)); OverrideFromFieldTrial(&encoder_info); @@ -926,7 +930,10 @@ VideoEncoder::EncoderInfo SimulcastEncoderAdapter::GetEncoderInfo() const { encoder_impl_info.requested_resolution_alignment); // request alignment on all layers if any of the encoders may need it, or // if any non-top layer encoder requests a non-trivial alignment. - if (encoder_impl_info.apply_alignment_to_all_simulcast_layers) { + if (encoder_impl_info.apply_alignment_to_all_simulcast_layers || + (encoder_impl_info.requested_resolution_alignment > 1 && + (codec_.simulcastStream[i].height < codec_.height || + codec_.simulcastStream[i].width < codec_.width))) { encoder_info.apply_alignment_to_all_simulcast_layers = true; } } From bd222bf2510273da49cb1a9dc7a3a7d9509325bf Mon Sep 17 00:00:00 2001 From: Angelika Serwa Date: Mon, 21 Nov 2022 13:40:37 +0000 Subject: [PATCH 3/3] Update HardwareVideoEncoder.java --- .../src/java/org/webrtc/HardwareVideoEncoder.java | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/sdk/android/src/java/org/webrtc/HardwareVideoEncoder.java b/sdk/android/src/java/org/webrtc/HardwareVideoEncoder.java index 68d036c6580..b6a2f7fd004 100644 --- a/sdk/android/src/java/org/webrtc/HardwareVideoEncoder.java +++ b/sdk/android/src/java/org/webrtc/HardwareVideoEncoder.java @@ -50,8 +50,8 @@ class HardwareVideoEncoder implements VideoEncoder { private static final int MEDIA_CODEC_RELEASE_TIMEOUT_MS = 5000; private static final int DEQUEUE_OUTPUT_BUFFER_TIMEOUT_US = 100000; - // Size of the input frames should be multiple of 16 for the H/W encoder. - private static final int REQUIRED_RESOLUTION_ALIGNMENT = 16; + // Size of the input frames should be multiple of 2 for the H/W encoder. + private static final int REQUIRED_RESOLUTION_ALIGNMENT = 2; /** * Keeps track of the number of output buffers that have been passed down the pipeline and not yet @@ -211,6 +211,11 @@ public VideoCodecStatus initEncode(Settings settings, Callback callback) { this.callback = callback; automaticResizeOn = settings.automaticResizeOn; + if (settings.width % REQUIRED_RESOLUTION_ALIGNMENT != 0 + || settings.height % REQUIRED_RESOLUTION_ALIGNMENT != 0) { + Logging.e(TAG, "MediaCodec requires 2x2 alignment."); + return VideoCodecStatus.ERR_SIZE; + } this.width = settings.width; this.height = settings.height; useSurfaceMode = canUseSurface(); @@ -523,6 +528,11 @@ private VideoCodecStatus resetCodec(int newWidth, int newHeight, boolean newUseS return status; } + if (newWidth % REQUIRED_RESOLUTION_ALIGNMENT != 0 + || newHeight % REQUIRED_RESOLUTION_ALIGNMENT != 0) { + Logging.e(TAG, "MediaCodec requires 2x2 alignment."); + return VideoCodecStatus.ERR_SIZE; + } width = newWidth; height = newHeight; useSurfaceMode = newUseSurfaceMode;