Skip to content

Commit 167e343

Browse files
JackLau1222winlinvipyangrtccloudwebrtcduiniuluantanqin
committed
avformat/whip: Add WHIP muxer support for subsecond latency streaming
0. WHIP Version 3. 1. The WHIP muxer has been renamed and refined, with improved logging context and error messages for SSL, DTLS, and RTC. 2. Magic numbers have been replaced with macros and extracted to functions, and log levels have been altered for better clarity. 3. DTLS curve list has been updated, and SRTP profile names have been refined for FFmpeg and OpenSSL. 4. ICE STUN magic number has been refined, and RTP payload types have been updated based on Chrome's definition. 5. Fixed frame size has been refined to rtc->audio_par->frame_size, and h264_mp4toannexb is now used to convert MP4/ISOM to annexb. 6. OPUS timestamp issue has been addressed, and marker setting has been corrected after utilizing BSF. 7. DTLS handshake and ICE handling have been optimized for improved performance, with a single handshake timeout and server role to prevent ARQ. 8. Consolidated ICE request/response handling and DTLS handshake into a single function, and fixed OpenSSL build errors to work with Pion. 9. Merge TLS & DTLS implementation, shared BIO callbacks, read, write, print_ssl_error, openssl_init_ca_key_cert, init_bio_method function and shared same data structure 10. Modify configure that whip is enabled only dtls is enabled(just support openssl for now) to fix build error Co-authored-by: winlin <winlinvip@gmail.com> Co-authored-by: yangrtc <yangrtc@aliyun.com> Co-authored-by: cloudwebrtc <duanweiwei1982@gmail.com> Co-authored-by: Haibo Chen <495810242@qq.com> Co-authored-by: Steven Liu <lq@chinaffmpeg.org> Co-authored-by: Jun Zhao <barryjzhao@tencent.com> Signed-off-by: Jack Lau <jacklau1222@qq.com> Signed-off-by: Steven Liu <lq@chinaffmpeg.org>
1 parent d4556c9 commit 167e343

13 files changed

Lines changed: 2925 additions & 56 deletions

File tree

‎configure‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3749,6 +3749,7 @@ wav_demuxer_select="riffdec"
37493749
wav_muxer_select="riffenc"
37503750
webm_chunk_muxer_select="webm_muxer"
37513751
webm_dash_manifest_demuxer_select="matroska_demuxer"
3752+
whip_muxer_deps_any="dtls_protocol"
37523753
wtv_demuxer_select="mpegts_demuxer riffdec"
37533754
wtv_muxer_select="mpegts_muxer riffenc"
37543755
xmv_demuxer_select="riffdec"
@@ -3847,6 +3848,9 @@ srtp_protocol_select="rtp_protocol srtp"
38473848
tcp_protocol_select="network"
38483849
tls_protocol_deps_any="gnutls openssl schannel securetransport libtls mbedtls"
38493850
tls_protocol_select="tcp_protocol"
3851+
# TODO: Support libtls, mbedtls, and gnutls.
3852+
dtls_protocol_deps_any="openssl"
3853+
dtls_protocol_select="udp_protocol"
38503854
udp_protocol_select="network"
38513855
udplite_protocol_select="network"
38523856
unix_protocol_deps="sys_un_h"
@@ -7198,6 +7202,14 @@ enabled rkmpp && { require_pkg_config rkmpp rockchip_mpp rockchip/r
71987202
}
71997203
enabled vapoursynth && require_headers "vapoursynth/VSScript4.h vapoursynth/VapourSynth4.h"
72007204

7205+
enabled openssl && {
7206+
enabled whip_muxer && {
7207+
$pkg_config --exists --print-errors "openssl >= 1.0.1k" ||
7208+
require_pkg_config openssl "openssl >= 1.0.1k" openssl/ssl.h SSL_library_init ||
7209+
require_pkg_config openssl "openssl >= 1.0.1k" openssl/ssl.h OPENSSL_init_ssl
7210+
}
7211+
}
7212+
72017213

72027214
if enabled gcrypt; then
72037215
GCRYPT_CONFIG="${cross_prefix}libgcrypt-config"

‎doc/muxers.texi‎

Lines changed: 47 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3879,4 +3879,51 @@ ffmpeg -f webm_dash_manifest -i video1.webm \
38793879
manifest.xml
38803880
@end example
38813881

3882+
@anchor{whip}
3883+
@section whip
3884+
3885+
WebRTC (Real-Time Communication) muxer that supports sub-second latency streaming according to
3886+
the WHIP (WebRTC-HTTP ingestion protocol) specification.
3887+
3888+
It uses HTTP as a signaling protocol to exchange SDP capabilities and ICE lite candidates. Then,
3889+
it uses STUN binding requests and responses to establish a session over UDP. Subsequently, it
3890+
initiates a DTLS handshake to exchange the SRTP encryption keys. Lastly, it splits video and
3891+
audio frames into RTP packets and encrypts them using SRTP.
3892+
3893+
Ensure that you use H.264 without B frames and Opus for the audio codec. For example, to convert
3894+
an input file with @command{ffmpeg} to WebRTC:
3895+
@example
3896+
ffmpeg -re -i input.mp4 -acodec libopus -ar 48000 -ac 2 \
3897+
-vcodec libx264 -profile:v baseline -tune zerolatency -threads 1 -bf 0 \
3898+
-f whip "http://localhost:1985/rtc/v1/whip/?app=live&stream=livestream"
3899+
@end example
3900+
3901+
For this example, we have employed low latency options, resulting in an end-to-end latency of
3902+
approximately 150ms.
3903+
3904+
@subsection Options
3905+
3906+
This muxer supports the following options:
3907+
3908+
@table @option
3909+
3910+
@item handshake_timeout @var{integer}
3911+
Set the timeout in milliseconds for ICE and DTLS handshake.
3912+
Default value is 5000.
3913+
3914+
@item pkt_size @var{integer}
3915+
Set the maximum size, in bytes, of RTP packets that send out.
3916+
Default value is 1500.
3917+
3918+
@item authorization @var{string}
3919+
The optional Bearer token for WHIP Authorization.
3920+
3921+
@item cert_file @var{string}
3922+
The optional certificate file path for DTLS.
3923+
3924+
@item key_file @var{string}
3925+
The optional private key file path for DTLS.
3926+
3927+
@end table
3928+
38823929
@c man end MUXERS

‎libavformat/Makefile‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -638,6 +638,7 @@ OBJS-$(CONFIG_WEBM_CHUNK_MUXER) += webm_chunk.o
638638
OBJS-$(CONFIG_WEBP_MUXER) += webpenc.o
639639
OBJS-$(CONFIG_WEBVTT_DEMUXER) += webvttdec.o subtitles.o
640640
OBJS-$(CONFIG_WEBVTT_MUXER) += webvttenc.o
641+
OBJS-$(CONFIG_WHIP_MUXER) += whip.o avc.o http.o srtp.o tls_openssl.o
641642
OBJS-$(CONFIG_WSAUD_DEMUXER) += westwood_aud.o
642643
OBJS-$(CONFIG_WSAUD_MUXER) += westwood_audenc.o
643644
OBJS-$(CONFIG_WSD_DEMUXER) += wsddec.o rawdec.o

‎libavformat/allformats.c‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -515,6 +515,7 @@ extern const FFOutputFormat ff_webp_muxer;
515515
extern const FFInputFormat ff_webvtt_demuxer;
516516
extern const FFOutputFormat ff_webvtt_muxer;
517517
extern const FFInputFormat ff_wsaud_demuxer;
518+
extern const FFOutputFormat ff_whip_muxer;
518519
extern const FFOutputFormat ff_wsaud_muxer;
519520
extern const FFInputFormat ff_wsd_demuxer;
520521
extern const FFInputFormat ff_wsvqa_demuxer;

‎libavformat/avio.c‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -339,8 +339,9 @@ static const struct URLProtocol *url_find_protocol(const char *filename)
339339
}
340340
}
341341
av_freep(&protocols);
342-
if (av_strstart(filename, "https:", NULL) || av_strstart(filename, "tls:", NULL))
343-
av_log(NULL, AV_LOG_WARNING, "https protocol not found, recompile FFmpeg with "
342+
if (av_strstart(filename, "https:", NULL) || av_strstart(filename, "tls:", NULL) ||
343+
av_strstart(filename, "dtls:", NULL))
344+
av_log(NULL, AV_LOG_WARNING, "https or dtls protocol not found, recompile FFmpeg with "
344345
"openssl, gnutls or securetransport enabled.\n");
345346

346347
return NULL;

‎libavformat/http.c‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -562,6 +562,12 @@ int ff_http_averror(int status_code, int default_averror)
562562
return default_averror;
563563
}
564564

565+
const char* ff_http_get_new_location(URLContext *h)
566+
{
567+
HTTPContext *s = h->priv_data;
568+
return s->new_location;
569+
}
570+
565571
static int http_write_reply(URLContext* h, int status_code)
566572
{
567573
int ret, body = 0, reply_code, message_len;

‎libavformat/http.h‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -62,4 +62,6 @@ int ff_http_do_new_request2(URLContext *h, const char *uri, AVDictionary **optio
6262

6363
int ff_http_averror(int status_code, int default_averror);
6464

65+
const char* ff_http_get_new_location(URLContext *h);
66+
6567
#endif /* AVFORMAT_HTTP_H */

‎libavformat/protocols.c‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -62,6 +62,7 @@ extern const URLProtocol ff_subfile_protocol;
6262
extern const URLProtocol ff_tee_protocol;
6363
extern const URLProtocol ff_tcp_protocol;
6464
extern const URLProtocol ff_tls_protocol;
65+
extern const URLProtocol ff_dtls_protocol;
6566
extern const URLProtocol ff_udp_protocol;
6667
extern const URLProtocol ff_udplite_protocol;
6768
extern const URLProtocol ff_unix_protocol;

‎libavformat/srtp.h‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -27,7 +27,7 @@
2727
struct AVAES;
2828
struct AVHMAC;
2929

30-
struct SRTPContext {
30+
typedef struct SRTPContext {
3131
struct AVAES *aes;
3232
struct AVHMAC *hmac;
3333
int rtp_hmac_size, rtcp_hmac_size;
@@ -40,7 +40,7 @@ struct SRTPContext {
4040
uint32_t roc;
4141

4242
uint32_t rtcp_index;
43-
};
43+
} SRTPContext;
4444

4545
int ff_srtp_set_crypto(struct SRTPContext *s, const char *suite,
4646
const char *params);

‎libavformat/tls.c‎

Lines changed: 65 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
/*
2-
* TLS/SSL Protocol
2+
* TLS/DTLS/SSL Protocol
33
* Copyright (c) 2011 Martin Storsjo
4+
* Copyright (c) 2025 Jack Lau
45
*
56
* This file is part of FFmpeg.
67
*
@@ -20,6 +21,7 @@
2021
*/
2122

2223
#include "avformat.h"
24+
#include "internal.h"
2325
#include "network.h"
2426
#include "os_support.h"
2527
#include "url.h"
@@ -93,7 +95,7 @@ int ff_tls_open_underlying(TLSShared *c, URLContext *parent, const char *uri, AV
9395
c->listen = 1;
9496
}
9597

96-
ff_url_join(buf, sizeof(buf), "tcp", NULL, c->underlying_host, port, "%s", p);
98+
ff_url_join(buf, sizeof(buf), c->is_dtls ? "udp" : "tcp", NULL, c->underlying_host, port, "%s", p);
9799

98100
hints.ai_flags = AI_NUMERICHOST;
99101
if (!getaddrinfo(c->underlying_host, NULL, &hints, &ai)) {
@@ -124,7 +126,65 @@ int ff_tls_open_underlying(TLSShared *c, URLContext *parent, const char *uri, AV
124126
}
125127

126128
freeenv_utf8(env_http_proxy);
127-
return ffurl_open_whitelist(&c->tcp, buf, AVIO_FLAG_READ_WRITE,
128-
&parent->interrupt_callback, options,
129-
parent->protocol_whitelist, parent->protocol_blacklist, parent);
129+
if (c->is_dtls) {
130+
av_dict_set_int(options, "connect", 1, 0);
131+
av_dict_set_int(options, "fifo_size", 0, 0);
132+
/* Set the max packet size to the buffer size. */
133+
av_dict_set_int(options, "pkt_size", c->mtu, 0);
134+
}
135+
ret = ffurl_open_whitelist(c->is_dtls ? &c->udp : &c->tcp, buf, AVIO_FLAG_READ_WRITE,
136+
&parent->interrupt_callback, options,
137+
parent->protocol_whitelist, parent->protocol_blacklist, parent);
138+
if (c->is_dtls) {
139+
if (ret < 0) {
140+
av_log(c, AV_LOG_ERROR, "WHIP: Failed to connect udp://%s:%d\n", c->underlying_host, port);
141+
return ret;
142+
}
143+
/* Make the socket non-blocking, set to READ and WRITE mode after connected */
144+
ff_socket_nonblock(ffurl_get_file_handle(c->udp), 1);
145+
c->udp->flags |= AVIO_FLAG_READ | AVIO_FLAG_NONBLOCK;
146+
}
147+
return ret;
148+
}
149+
150+
/**
151+
* Read all data from the given URL url and store it in the given buffer bp.
152+
*/
153+
int ff_url_read_all(const char *url, AVBPrint *bp)
154+
{
155+
int ret = 0;
156+
AVDictionary *opts = NULL;
157+
URLContext *uc = NULL;
158+
char buf[MAX_URL_SIZE];
159+
160+
ret = ffurl_open_whitelist(&uc, url, AVIO_FLAG_READ, NULL, &opts, NULL, NULL, NULL);
161+
if (ret < 0) {
162+
av_log(NULL, AV_LOG_ERROR, "TLS: Failed to open url %s\n", url);
163+
goto end;
164+
}
165+
166+
while (1) {
167+
ret = ffurl_read(uc, buf, sizeof(buf));
168+
if (ret == AVERROR_EOF) {
169+
/* Reset the error because we read all response as answer util EOF. */
170+
ret = 0;
171+
break;
172+
}
173+
if (ret <= 0) {
174+
av_log(NULL, AV_LOG_ERROR, "TLS: Failed to read from url=%s, key is %s\n", url, bp->str);
175+
goto end;
176+
}
177+
178+
av_bprintf(bp, "%.*s", ret, buf);
179+
if (!av_bprint_is_complete(bp)) {
180+
av_log(NULL, AV_LOG_ERROR, "TLS: Exceed max size %.*s, %s\n", ret, buf, bp->str);
181+
ret = AVERROR(EIO);
182+
goto end;
183+
}
184+
}
185+
186+
end:
187+
ffurl_closep(&uc);
188+
av_dict_free(&opts);
189+
return ret;
130190
}

0 commit comments

Comments
 (0)