ttipo/comrelay
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Squashed 'livekit-server/' changes from 154b4d26..46c43095

Browse Source 
46c43095 fix goreleaser workflow, version 1.13.1 (#4577)
e0815be2 chore: improve docker test shutdown reliability (#4576)
bfd9deff expose TCPFallbackRTTThreshold and AllowUDPUnstableFallback via config (#4556)
b93c1e16 Release v1.13.0. (#4573)
fd452212 Update mediatransportutil to get ICE candidate timeout config (#4572)
8be8c74a Update github workflows (#4463)
c4e41872 Update go deps to v1.17.2 (#4462)
dc8e0310 Update go deps to v4 (#4482)
20fd1ad2 turn: allow for providing secret via file (#4564)
6590570d Pin pion/dtls to v3.1.2 (#4570)
cdbbee1f deps: bump protocol + psrpc to latest tips (#4565)
d290de81 Correct config comment (#4563)
77ecf920 rtc: report participant session end time on room move (#4561)
63be96f6 Prevent panic from nil(illegal) syncState.Subscriptions message (#4560)
835ef1b3 Metrics for participant active, i. e. fully established. (#4557)
5bd42534 Document of advertise_internal_ip and external_ip_only (#4554)
356ae211 Config documentation for advertise_internal_ip and skip_external_ip_validation (#4552)
7c319a67 rtc: prevent duration reporting for inactive participants (#4550)
2dd5e632 telemetry: split webhook-processed hook out of NewTelemetryService (#4548)
222177a9 service: prevent nil deref in validate with wrapped join request (#4547)
dd7580b4 Protect against nil clientInfo (#4546)
145689e6 Start tracking Twirp method request latency in prometheus too, not just in logs (#4545)
cde89627 rtc: emit per-data-track bytes via BytesTrackStats (#4540)
2e22911d Remove backwards compatibility support for TURN auth. (#4539)
062d1219 Use NACKQuueInterface type. (#4538)
7f08b04c Add IsIntentionalDisconnect helper (#4537)
1ab2bf04 Clean up packet size logging (#4536)
8ab92a80 Don't require media sections when joining (#4535)
019a6640 rtc: report participant kind code and details (#4534)
77595d38 TEL-336: fix sip error categorization (#4528)
f303f499 Always enable rtx codec (#4533)
e4a8a55c Check Less and LessEq in version compare. (#4532)
37eb7a32 Release v1.12.0 (#4529)
4a7b1e85 Create NACK tracker only once. (#4527)
89faaeba Apply ttl check only when authenticate allocation creating (#4526)
b32933b0 Log details of RTCP packets. (#4525)
8b79ec9e Support SIP auth realm for inbound. (#4522)
4b8db3cf Add integration test for TURN auth failures (#4524)
ef2e5efe Log large packets receive/send. (#4521)
d1236750 feat: auto create rooms for tokens with the RoomCreate grant (#4320)
7a3e595b apply room tags from JWT grant room configuration (#4518)
ab7fdeab add AssignmentHook to AssignJob; propagate websocket write errors (#4516)
cf20c9cd Add expiry to TURN password. (#4515)
20d4a3a1 Populate data track loggers with context (#4514)
12fff29a allow setting agent job assignment url (#4512)
ba366fc7 Fix SIP media config upgrade. (#4511)
8fbc5adf update protocol for protojson (#4510)
3de6f517 Add TURN permission handler. (#4505)
8ffcef93 Update protocol to support SIP media config. (#4509)
1ab1e072 test: verify upstream and downstream connection stats end-to-end (#4508)
c4fd71a5 Fix sense check in DeltaInfo gathering (#4507)
803999ef rename agent environment to deployment (#4506)
bacc21e6 add helper to check for agent worker endpoint (#4503)
253f977d add duration seconds reporting (#4500)
ffab3bd3 add agent environment (#4498)
ccdf23c8 Use mediatransportutil/codec package, no functional change (#4497)
680703f2 Include reception reoprts in receiver report callback. (#4496)
f51798bc Fix publish-only limitations being incorrectly applied to receivers (#4495)
a002337d Legacy TrackInfo.Simulcast flag. (#4493)
af1dcc88 Add CloseWithReason to agent SignalConn interface (#4492)
d7c2daf1 report all simulcast layers (#4491)
c1ad2b22 Misc optimisations. (#4490)
19b9e8c0 Additional data tracks logging (#4489)
743d9c8b add support for client capabilities (#4461)
fc47e478 Close peer connection unconditionally to unblock set local/remote (#4485)
639406eb Update module github.com/pion/ice/v4 to v4.2.3 (#4481)
dc6b7505 reduce some heap use in packet path (#4478)
f3b80b28 fix: wrap IPv6 addresses in brackets in UDP TURN URLs (RFC 3986) (#4476)
3a7f2628 Turn off transceiver re-use on Safari. (#4474)
d84f3d7a add more types to signum (#4473)
701a37c2 Convert sort.Slice -> slices.SortFunc (#4472)
85be9d70 Avoid stream allocator event data cast to interface and back. (#4471)
b43685e8 Keep a shadow copy of tracks for use by different stream allocator state (#4470)
27c2b149 Consolidate RTCP packets and do RTCP callback outside lock. (#4469)
31083307 do not log data track stats if not started (#4468)
9ee06635 feat(pion/ice): replace deprecated NAT1To1 with SetAddressRewriteRules (#4466)
8ccad68d Release v1.11.0 (#4459)
dbf5cf61 Store concrete ICE candidate for remote candidates. (#4458)
2a04bc3c fix publisher frame count reporting for simulcast streams (#4457)
1d804737 fix: limit join request and WHIP request body to http.DefaultMaxHeaderBytes (#4450)
3cfb71e7 Use Muted in TrackInfo to propagated published track muted. (#4453)
69aa9479 Some drive-by clean up (#4452)
6c81f678 Add subscriber stream start event notification (#4449)
ce1bf47b Revert "fix: ensure num_participants is accurate in webhook events (#4265) (#…" (#4448)
cdb0769c fix: ensure num_participants is accurate in webhook events (#4265) (#4422)
c91e79af Switch to stdlib maps, slices (#4445)
ea7b9c6f Update module github.com/livekit/protocol to v1.45.3 (#4435)
97378368 Update go deps (major) (#3179)
d6aef547 Update go deps (#3862)
afc9feae Update github workflows (#4331)
4b385612 chore: pin GH commits and switch to golangci-lint (#4444)
2974ba87 Unsubscribe from data track on close (#4443)
5dc4e90d Apply IPFilter when get local ip (#4440)
88c77dc6 compute agent dispatch affinity from target load (#4442)
8fe99377 Log join duration. (#4433)
0a503a57 Add `Close` method for UpDataTrackManager and call it on participant (#4432)
55912dff Add some simple data track stats (#4431)
050909e6 Enable data tracks by default. (#4429)
72c7e65c chore: log API key during worker registration (#4428)
8a67dd1b Do not close publisher peer connection to aid migration. (#4427)
91e90c10 Add some more logging around migration. (#4426)
c6ddc879 isExpectedToResume is based on whether flushing or not. (#4425)
7d06cfca Keep subscription synchronous when publisher is expected to resume. (#4424)
934f8598 Clean up data track observers on unsubscribe. (#4421)
9674ac48 Cleaning up some logs and standardising log frequency. (#4420)
7b925304 Drop time inverted packets in RED -> Opus conversion. (#4418)
4d8d232a ensure participant init is correctly serialized for logging (#4417)
4fe80877 Log time inversion between incoming packets (#4415)
248d7394 Guard against timestamp inversion in RED -> Opus conversion. (#4414)
9ab8c1d5 clear track notifier observers on subscription teardown (#4413)
397cd09a Embedded turn test (#4412)
56326654 Prepare release v1.10.1 (#4408)
e9b113c8 Make the TURN bind address configurable and allow for multiple addresses. (#4315)
4bc5e6bb Address malformed H264/H265 parsing issues. (#4407)
77a0a4fc AV1 parser overflow fix. (#4405)
ff7fd7ed feat(agent-dispatch): add job restart policy (#4401)
34bd1e08 do not log roll over for padding only packets (#4396)
13d02ee9 add deadline to dtls connect context (#4395)
9055a349 Path check helpers (#4392)
1f1eeb68 Fallback to servicestore if rpc is unavailable (#4391)
59e9bb41 Fix TURN server URL (#4389)
9e0a7e54 Close both peer connections to aid migration. (#4382)
9474c807 route participant reads through PSRPC instead of Redis (#4387)
a5333a86 add packet trailer stripping support (#4361)
0e3d765d Release v1.10.0 (#4380)
bc3aeaf3 Update grpc to address CVE-2026-33186 (#4381)
8cdd6f4c Replace deprecated io/ioutil with io in whipservice (#4375)
89410df7 handle AGENT_ERROR disconnect reason (#4339)
8f984c77 Fix repair stream ID reporting for RTX pairing. (#4369)
cdfaacfc Restart nacker on OOB sequence number restart. (#4368)
4a9e0045 Update protocol. (#4367)
750d5904 Add API to restart lite stats. (#4366)
c8bb2578 Rename log field "pID" to "participantID" for consistency (#4365)
77fc74a7 Do not block all ext ID determination on stream allocator listener (#4364)
90a46fab Do not kick off migration of closed participant (#4363)
5dc2e7b1 Switch data track extension to 1-byte ID/length. (#4362)
7323ad02 Sample data send error logging. (#4358)
0d34e455 Add option to not re-use transceiver in e2ee. (#4356)
95225ff2 don't require media section for dual peerconnection mode (#4354)
e9639538 Refine ipv6 support (#4352)
b34b0472 Add StopEgress function to the EgressLauncher interface (#4353)
69235ed2 update readme (#4340)
db1a8046 defensive check for peer connection instance (#4350)
cb7dc2d0 TEL-405: support originating calls from custom domains (#4349)
7eaaaada Mark last run of grow bucket outside goroutine. (#4348)
caa47522 Add option to require media sections when participant joining (#4347)
087050d1 Release v1.9.12 (#4346)
493e87df Fix SIP client timeout. (#4345)
52c28a93 Log a bit more details of out-of-order TWCC feedback report. (#4343)
516aeabf Use ParticipantTelemetryListener of LocalParticipant. (#4342)
b3510565 Exclude ice restart case from offer answer id mismatch warning (#4341)
303657bc feat: make INSTALL_PATH overridable in install script (#3954)
9d418689 Send participant left event after track unpublished for moved (#4334)
bab91868 do not discount packets lost on duplicate packets (#4333)
939794cf mark + restart (#4329)
75f9c462 Add debug for receiver restart. (#4328)
74891f30 Protect against incorrect temporal layer. (#4327)
f51b2732 Update pion/webrtc and deps to update dtls (#4326)
b81bac0e Key telemetry stats worker using combination of roomID, participantID  (#4323)
2d40449f Update self-hosting deployment documentation link (#4312)
03e90dd7 Fix for some CodeQL reported issues (#4314)
77c858f0 Log stats worker when it not closable. (#4313)
a6035212 ESP32 client info (#4267)
478e486a Copilot suggested improvement to Github Actions permissions (#4310)
cbd2f82d Copilot suggested improvement to Github Actions permissions (#4311)
a9b8d40d Publish is always on publisher peer connection. (#4307)
8ae56406 generate & log egressID for start egress request (#4303)
35d7ef88 Avoid alloc in RTPStatsReceiver.Update (#4302)
bb744916 More optimisation in RTPStats module. (#4298)
cefd5da9 Optimise some bits in rtpstats_receiver (#4297)
52a4b848 Read client protocol from query param (#4294)
195b17f6 Populate client_protocol field in ParticipantInfo (#4293)
370e0a4d Set up audio config in audio level module when config is updated. (#4290)
f3e9b688 Do not increase max expected layer on track info update. (#4285)
a9849340 Avoid logger data race. (#4284)
97016627 Do not hold lock when creating buffer (#4283)
6b68e3d5 Create buffer if needed when a PLI is requested. (#4282)
3cca7180 use separate allocation for signal stats telemetry guard (#4281)
d2bae34d refresh telemetry guard on participant move (#4280)
333f0349 clear reference guard when resetting signal stats (#4279)
d1bab17b Add session duration and participant kind to closing log. (#4277)
1e689e1a Reducing some info level logs. (#4274)
88facc02 adds a test to ensure agent worker errors cause disconnection (#4273)
76a41a7a Generate config flags (#4268)
700e1788 require participant broadcast when metadata/attributes are set in token (#4266)
b61799ec Ignore parse addr error when add remote candidate (#4264)
01bd966f Add silent frame for pcmu/a (#4258)
0c33b8c6 chore: move codecs/mime stuff to protocol (#4255)
165c1735 Update livekit protocol to v1.44.0 (#4254)
343f12b8 Wrapping SIP errors for invalid argument and not found (#4253)
aea044c5 Defer setting clock rate in RTPStats module till codec is bound. (#4250)
d9f716c1 FIx receiver restart race (#4248)
0508ee9f remove copy/paste left over line (#4246)
40408407 Release v1.9.11 (#4245)
f8675507 Do not remove participant from cache on disconnect. (#4241)
a35a6ae7 Add participant option for data track auto-subscribe. (#4240)
07572511 Wrapping the invalid request errors for CreateSipParticipant (#4239)
843d8c3e Update Pion transport package. (#4237)
a05690d2 Changing field naming of data track packet (#4235)
c4015008 Clear participant version cache on disconnect (#4234)
18db4ec1 Log modified timeout of API context. (#4232)
ac20ccda Log timeout in API. (#4231)
541a7d6c Change some logs to debugw (#4229)
52ab3374 Return on SDP fragment read error. (#4228)
7fae5ac9 Do not restart receiver on codec change mid-session. (#4225)
dafdc36e Add option to force simuclast codec. (#4226)
0a7dd40b Use only layer 0 for SVC codecs. (#4224)
4ec0f8f4 Support OpenTelemetry tracing. Add Jaeger support. (#4222)
80ba93fa Do NACK updates as soon as flow state is processed. (#4221)
4405afe2 Use atomic pointer and return interface from RED transformer constructors (#4220)
b649c2fe Remove method not needed from REDTransformer. (#4219)
f0080f35 Remove enable arrival time forwarding method. (#4217)
335f4c33 Swap result sink atomically rather than closing and setting. (#4216)
46651c19 Release v1.9.10 (#4214)
08ac4ecd Support preserving external supplied time. (#4212)
f6efccce report video size from media data for whip (#4211)
d92f6a79 return iceservers for whip (#4210)
1a4758ed Skip restart callback if external. (#4208)
dde4fb49 configurable dependency descriptor restart (#4207)
08793bea Use active at time to check for track not bound timeout. (#4206)
3606ce54 Do not warn about track not bound if participant is not ready. (#4205)
b8ddd0f9 Taking interface{} -> any modernize bits (#4204)
b91cd2e4 Rework receiver restart. (#4202)
bb00c864 Restart API on receiver. (#4200)
25ece1e9 Minor refactor in buffer base and audio level (#4198)
599002f8 ignore PLI requests for non-video (#4196)
2510b946 Taking a bunch of go modernize suggestions. (#4194)
ed8e6afc Handle repair SSRC of simulcast tracks during migration. (#4193)
c6bf7a27 Fix logging key and other clean up around stream restart. (#4192)
3cb9abb6 Update pion/webrtc to v4.2.1 (#4191)
8b0efb8c Resolve RTX pair via OnTrack also. (#4190)
381bce03 Return extended sequence number only and not packet. (#4189)
6bcbf54e Always instantiate nacker when using out-of-band sequence numbers. (#4187)
e71184de Store buffer after creating it.  (#4186)
7c8ea115 Refactor receiver and buffer into Base and higher layer.  (#4185)
4104b827 update protocol (#4183)
cd99fec2 Make new path for signalling v1.5 support. (#4180)
32cd0370 Flush the ext packets on restart/close and release packets. (#4179)
1df1316b Move OnDataTrackMessage to participant listener to support replay. (#4178)
e7601251 Make data message naming a bit more consistent. (#4177)
a04e566d Use published track for model access in data down track. (#4176)
47c86be1 Add support for TURN static auth secret credentials (#3796)
24559a28 chore(deps): update github workflows to v6 (#3866)
0a824386 add explicit room exists servicestore op (#4175)
39bd077d Release v1.9.9 (#4174)
cbb2c617 Publish/Unpublish counter match. (#4173)
fb849edc Minor clean up (#4172)
c28e5e45 fix(kindToProto): update protocol (#4171)
47324abd Drop run away receiver reports. (#4170)
462ec324 prevent uint overflow setting packet not found count (#4169)
5c841b8e Some logging changes. (#4168)
2f2d0a57 skip lost sequence number ranges in getIntervalStats (#4166)
898ebe05 clean up manual roomservice log redaction (#4165)
3e417253 move delete to oss service store (#4164)
5964efbb Ensure subscribe data track handles are unique (#4162)
a26c4830 Add support for RTP stream restart. (#4161)
386f0b38 fix typo in clearing index when removing track from room track manager (#4158)
0abfb251 deregister observability function when participant is closed (#4157)
97aba5e7 Consistently undo update to sequence number and timestamp when the (#4156)
2317c295 Fix panic while removing track from room track manager. (#4153)
a0a28ac5 Avoid duplicate track add to room track manager.  (#4152)
f01008f8 Revert telemetry stats worker wait configuration. (#4151)
ca4b56d2 Handle case of sequence number jump just after start. (#4150)
97099cae Configurable telemetry stats worker clean up wait. (#4148)
d7db7cb3 chore: fix a large number of spelling issues (#4147)
cadf2ad9 Release v1.9.8 (#4145)
498304cd defensive nil check (#4144)
20f6a497 Store ddParser in atomic.Pointer (#4143)
037cb906 release ext packet if patching fails (#4142)
dd598ef2 Release ExtPacket if dependency descriptor or other parsing fails (#4141)
35b0e2bc update webrtc to 4.1.8 to pick up DTLS fingerprint check during handshake (#4140)
1c1a836c Mark RTCP buffer Write as noinline. (#4138)
ea9b2177 protocol deps to get inactive file adjusted memory usage. (#4137)
64f3d1e9 switch participant callbacks to room to listener interface (#4136)
c6e6c021 add debug metric for tracking references (#4134)
a30c79fa Use isEnding to indicate if down track could be resumed. (#4132)
8c241ecf Fix RTCP reader leak in DownTrack. (#4131)
3eef869a Do not pause rid in SDP (#4129)
8e01e595 Release 1.9.7 (#4128)
7c1a0fab Fix concurrent map access. (#4127)
14446b1c Let participant close remove the published tracks. (#4125)
fa0633aa move utils.WrapAround to mediatransportutil (#4124)
f8706cd4 Update pion/ice to stop gather first on close (#4123)
7954748d Data tracks (#4089)
7158d983 log bucket growth (#4122)
04b35eb6 Release v1.9.6 (#4121)
ebdcead5 Update mediatransportutil to get bucket packet size limit. (#4120)
411b09f6 Release v1.9.5 (#4119)
8dcf235a Update pion/ice - attempt to address tcp packet conn close hang (#4116)
64c65143 Update mediatransportutil (#4115)
0a2943bb Clean up bits added to debug peer connection close hang. (#4114)
9c483a69 Use released version v1.8.41 of pion/sctp (#4113)
35c79a57 Update SCTP hash, had the wrong one in previous PR (#4111)
e0fbbef1 Update pion/sctp with RFC9260 revert (#4110)
f3c80917 Try SCTP with read deadline to unblock abort. (#4109)
bd5382da Splitting transport close timeout logs. (#4108)
6d4154b8 Update pion/ice. (#4107)
a6418ae2 Log more peer conenction state on close timeout. (#4105)
06d99974 Check for cancel on unsubscription/source track going away. (#4104)
7f10e18b Record join/publish/subscribe cancellations. (#4102)
40293632 Clear stereo=1 if stereo is not enabled. (#4101)
70f6def3 Add checks for participant and sub-components close. (#4100)
ffbabcc7 Switch forwarding latency log to Debugw (#4098)
27d82a72 Fix "address" typo in transport logs (addddress → address) (#4097)
37a06821 logger proto redaction. (#4090)
54cf7d46 Control latency of lossy data channel (#4088)
5175c1af Lock x/tools at 0.37.0 (#4085)
d510fff1 Downgrade x/tools to be able to make a release (#4084)
c3ea5890 Prepare release v1.9.4. (#4083)
3a128e61 protocol bump for SIP error mapping and validation (#4081)
c3964ba2 Use sync.Pool for objects in packet path. (#4066)
f8b994d4 Forwarding latency measurement tweaks. (#4080)
f4929f09 Revert "Revert pion/transpor to v3.0.8 (#4073)" (#4074)
a04d9c48 Revert pion/transpor to v3.0.8 (#4073)
2d5054ad kind details for connector (#4072)
a272e28a Log raeson for subscriber not being to determine codec. (#4071)
b9b4eec9 Update pion/transport to v3.1.1 (#4070)
b23d093c update protocol (#4069)
4ce07bed Higher resolution forwarding latency histogram. (#4067)
858db7ab fix(deps): update module github.com/livekit/protocol to v1.43.0 (#4015)
1dc9b8fc Use buffered indicator to exclude from forwarding latency.  (#4062)
f117ee51 Track start up delay.  (#4061)
4872f205 Return write count from WriteRTP. (#4059)
d0ba46b4 Log write count atomic. (#4057)
ae5fb7e8 Add packet to forwarding stats only if packet is forwarded. (#4056)
f6909192 Update PsRPC to get redis pipeliner implementation. (#4055)
ca3c507b Prevent invalid track access while peer connection is shutting down. (#4054)
9ca6ee00 Use replace so that x/tools does not get overridden (#4048)
b9323eab chore(deps): downgrade x/tools for counterfeiter (#4047)
2f1e6c36 Prep release v1.9.3 (#4046)
9d5c351d Fix prom units for forwarding latency/jitter. (#4045)
e183657c Add prom histogram for forwarding latency and jitter. (#4044)
1eefeb30 Enable AbsCaptureTimeURI in RTC configuration (#4043)
075a7576 Use simulcast codec as default policy for audio track (#4040)
c264b504 Don't warn 0 payload type for PCMU (#4039)
32fc3525 Broadcast cond var on RTX write. (#4038)
061eb8b4 AddDownTrack to regressed codec after restarting forwarder. (#4037)
c87eb8ed fix: add missing Unlock() in AddReceiver (#4036)
70444924 if RingingTimeout is provided, deadline should be set to that timeout. (#4018)
ab906d71 Prevent leakage of previous codec after codec regression. (#4035)
79b03f97 Log queueing latency when encountering high forwarding latency (#4034)
29117b14 set max layer in allocation (#4033)
15b19ccd Remove ~ from rid which indicates disabled layer to get the actual rid (#4032)
34e16a87 Check more conditions for opportunistic alloc. (#4031)
81fbd355 Use the optimal allocation function for opportunistic allocation. (#4030)
a2ce73e0 Do not bind buffer if codec is invalid. (#4028)
cef6fdb7 Correct direction for request/response for prom counters. (#4027)
5042c06c Use rtp converter from protocol/utils/rtputil (#4020)
5a426d15 Use rtp converter from protocol/utils (#4019)
35fb8877 feat: use env var for GOARCH (#4012)
c0397696 Issue #1 only: Fix spatial layer initialization in Forwarder (#4003)
2afbf0e8 Some golang modernisation bits. (#4016)
484f784a Prepare release v1.9.2 (#4011)
ad074ed2 counterfeiter needs an older version of x/tools (#4009)
e63e8b6f Include mid -> trackID in both SDP offer and answer. (#4007)
781dfede Do not call receiver methods under settings lock. (#4006)
69ff25a0 Use answer with mid -> trackID mapping when in single peer connection (#4005)
fe912acf Update pion/webrtc to prevent GetStats panic. (#4004)
7930dcde Do not try to read stats from peer connection after close. (#4002)
ca0d5ee9 Count request/response packets on both client and server side. (#4001)
dd62eb00 Resort to full search for requested quality is not available. (#4000)
f6ca82d1 Revert to using silence packets for audio dummy start. (#3999)
0e2c59c8 Sort codec layers when adding track (#3998)
100bb46a Adding ProviderInfo to GetSIPTrunkAuthenticationResponse (#3993)
a8d4df66 "Power of Two Random Choices" option for node selection  (#3785)
a20bbe34 Log RPC details. (#3991)
158496bc Increment RTP timestamp on padding when using dummy start. (#3989)
4f6ed65d Limit check to red + opus when looking for primary codec match. (#3988)
a87f6c4b Allow passing inline trunk for outbound calls. (#3987)
bf06596f Support Opus mixed with RED when encrypted. (#3986)
ea208a1c Add encryption datapacket type (#3869)
2a6adbe8 Use padding only packets for dummy start of audio. (#3984)
be018f97 Provide the InputVideo/AudioState to Ingress in WHIPRTCConnectionNotify (#3982)
146bd969 Do not panic of redis is not configured (#3981)
01337ba7 Do not start forawarding on out-of-order packet. (#3985)
c7f625d6 Do not force codec regression between opus and red. (#3980)
3bd20ddb Revert unintentional change to not handle transport fallback on (#3970)
89a2f46c Update deps to fix redis issue when 1 cluster address is provided (#3969)
060719d1 add config for user data recording (#3966)
b3ee219c fix stats worker closed condition (#3965)
3d737031 add idempotent reference count to telemetry stats worker (#3964)
735c663a Update protocol for EventKey helper. (#3963)
646b9de8 Add node_ip to config-sample.yaml (#3960)
0bf7b178 avoid logging on small values (#3958)
00ff2ab9 Adjust for hold time when fowarding RTCP report. (#3956)
e180be06 short circuit participant broadcast filter in livestream mode (#3955)
bfba6fee Adjust stream allocator ping interval based on state. (#3951)
3837006b Revert "Switch ops queue a singly linked list. (#3949)" (#3950)
990c5faf feat: server rpc apis (#3904)
80b11662 Switch ops queue a singly linked list. (#3949)
56ee2328 handle terminated job requests (#3948)
49f9b9c8 Flush stats when there are no packets. (#3947)
e6a3df1e ForwarStats.GetStats needs to be public (#3946)
824d116b Tweaks tresholds for logging high forwarding latency/jitter. (#3945)
408492e0 Log some information around high forwarding latency. (#3944)
6a41fae5 Use microseconds for forwarding stats. (#3943)
856e0871 mediatransportutil to log local address when validating external IP (#3942)
40101cf7 Update protocol for SipCreateParticipant (#3939)
b07e7a38 Use difference in key frame counter to stop seeder. (#3936)
d7f92878 Avoid matching on empty track id. (#3937)
56fb2885 Do DD restart only if DD structure is present. (#3935)
86facce9 More debugging of DD jump (#3934)
6058a3f6 Add debugging from DD frame number wrap around. (#3933)
dc6825c0 mediatransportutil crash fix for logging local address (#3930)
d6f0588f Update mediatransportutil to log external IP found via STUN. (#3929)
2c30a064 Fix dynacast subscriber node clearing on move participant. (#3926)
6489237e Simulcast audio fixes (#3925)
9f0ab870 Wait for `SetRemoteDescription` before configuring senders. (#3924)
df6c26db Subscrbed audio codecs - update from remote nodes. (#3921)
798fa761 Support simulcasting of audio (#3920)
f4a06cf0 Clean code as there is no oss sweeper for ingress (#3918)
5f561b4f Include agent_name as a participant attribute (#3914)
782a35e8 update protocol for psrpc (#3915)
eee2001a Set publisher codec preferences after setting remote description (#3913)
fc995533 add incoming request id to request response message (#3912)
76645fad Rpcs for ingress proxy WHIP (#3911)
991a4a4f Refactor subscribedTrack + mediaTrackSubscriptions. (#3908)
e16b3ba9 Use gzip reader pool (#3903)
17c34921 update protocol for sip api change (#3902)
2f43a575 Release candidate for v1.9.1 (#3899)
07c40cf3 Use `RequestResponse` to report protocol handling errors (#3895)
98352fd0 Prevent race in determining BWE type. (#3891)
f7291fda Do not send both asb-send-time and twcc. (#3890)
21b42fa6 Do not advertise NACK for RED. (#3889)
6633bf93 Use departure timeout from room preset. (#3888)
38f7906e Handle migration better in single peer connection case. (#3886)
5026de2b handle frame number wrap back in svc (#3885)
091e3c13 Revert to using answer for migration case. (#3884)
2aeadf14 init ua parser once (#3883)
998a9f94 Switch known rids from 012 -> 210, used by OBS. (#3882)
890fd942 Single peer connection mode (#3873)
bfe98eaa fix: ensure the participant kind is set on refresh tokens (#3881)
8d270e2a chunk room updates (#3880)
b4e146c5 update mediatransport util for ice port 3478 (#3877)
dc3a7753 Fix timeout handing in StopEgress (#3876)
d62336e1 Remove unnecessary check (#3870)
c58e5d23 Update golang Docker tag to v1.25 (#3864)
98d577ee Update module github.com/livekit/protocol to v1.40.0 (#3865)
afbf541e Update pion deps (#3863)
b660c3b5 Extract video size from media stream (#3856)
456b8709 Fix missed unlock (#3861)
d500806e Handle no codecs in track info. (#3859)
11b240d6 Log track settings more. (#3853)
1aa0f963 Log signal messages on media node. (#3852)
b182d07b Log signal messages as debug. (#3851)
a370bb20 Support G.711 A-law and U-law (#3849)
fa5f4ef3 Populate SDP cid in track info when available. (#3845)
eed27885 Send `participant_connection_aborted` when participant session is closed (#3848)
61e59346 Update go deps (#3439)
1b228913 Support video layer mode from client and make most of the code mime aware (#3843)
f2da4444 Support per simulcast codec layers. (#3840)
f275f592 handle SyncState in join request (#3839)
5d44cf6d Use wrapped join request to be able to support compressed and uncompressed. (#3838)
5ca16264 Support join request as proto + base64 encoded query param (#3836)
7dea1012 Clean up missed v2 pieces (#3837)
34a49130 Delete v2 signalling (#3835)
1fe33716 Fix: RingingTimeout was being skipped for transferParticipant (#3831)
5751692a deps (#3829)
db4bc127 Get to the point of connecting publisher PC and using it for async signalling (#3822)
5e483e75 update readme (#3809)
e3155b14 Get to the point of establishing subscriber peer connection. (#3821)
a7ce1382 HTTP DELETE of participant session (#3819)
01de0e36 Do not send leave if nil (to older clients) (#3817)
10103449 Add country label to edge prom stats. (#3816)
68387b41 Minor tweak to keep RPC type at service level. (#3815)
a75295fc More v2 signalling changes (#3814)
b20db94d Validation end point for v2 signalling. (#3811)
f2f595f4 update readme (#3808)
fffc2ac0 Use signalling utils from protocol (#3807)
f5fc82d3 Filling out messages unlikely to change in v2. (#3806)
1c99b9ad Split signal segmenter and reassembler. (#3805)
0a1bfd30 Signal handling interfaces and participant specific HTTP PATCH. (#3804)
7837c8e5 starting signaller interface (#3802)
18ce5244 Handle Metadata field from RoomConfig (#3798)
2a6a9b8a Grouping all signal messages into participant_signal. (#3801)
078c01fa Signal v2: envelope and fragments as wire message format. (#3800)
b9a44c3f Signalling V2 protocol implementation start (#3794)
ba702a53 forward agent id to job state (#3786)
1f31d430 Map ErrNoResponse to ErrRequestTimedOut in StopEgress to avoid returning 503 (#3788)
51bbe8c5 Set participant active when peerconnection connected (#3790)
40028dc3 Normalize known rids. (#3779)
ddd92329 Return default layer for invalid rid + track info combination. (#3778)
8c033ce9 Enable H265 by default (#3773)
7678e087 Set rids for all codecs. (#3772)
5d636acf Limit taking rids from SDP only in WHIP path. (#3771)
4d09e5b5 Log SDP rids to understand the mapping better. (#3770)
c69f1aae Revert "Temporary change: use pre-defined rids" (#3769)
8197438e bounds check layer index (#3768)
d11da5f5 Temporary change: use pre-defined rids (#3767)
cb4da533 fix(deps): update module github.com/livekit/protocol to v1.39.3 (#3733)
d6d2b6d8 feat(cli-flags): add option for cpu profiling (#3765)
9fc4ddbe ClearAllReceivers interface is used to pause relay tracks. (#3761)
1216113b Do not need to just clean up receivers. Remove that interface. (#3760)
ef6c38ce Log previous allocation to see changes. (#3759)
01bf9685 SVC with RID -> spatial layer mapping (#3754)
c481396f offer could be nil when migrating. (#3752)
8c2fc0bc Fix svc encoding for chrome mobile on iOS (#3751)
e467daa0 move egress roomID load to launcher (#3748)
3783ebb3 feat(cli): update to urfave/cli/v3 (#3745)
03d3fcab Fix data packet ParticipantIdentity override logic in participant.go (#3735)
068b4366 reuse compiled client config scripts (#3743)
e754a860 return error when moving egree/agent participant (#3741)
7542cf07 remove unused code (#3740)
9d569e2f Take ClientInfo from request. (#3738)
80774263 chore: set workerid on job creation (#3737)
5549ab55 Revert clearing RIDs. (#3732)
ae967313 Clear rids if not present in SDP. (#3731)
0e033907 Return highest available layer if requested quality is higher than (#3729)
9ce737db Add log for dropping out of order reliable message (#3728)
1b95e818 Don't check bindState on downtrack.Bind (#3726)
670f927f Set and use rid/spatial layer in TrackInfo. (#3724)
a9e29116 Add Id to SDP signalling messages. (#3722)
4ec828ce Fix bug with SDP rid, clear only overflow. (#3723)
8f6c3a9b Clear rids from default for layers not published. (#3721)
ce07740e Add simulcast support for WHIP. (#3719)
e98fb94f Create client config manager in room manager constructor. (#3718)
fdf9b852 e2e reliability for data channel (#3716)
35dda8ea swap pub/sub track metrics (#3717)
1d9a4366 Do not require create permission for WHIP participant. (#3715)
e0aea17a Flush stats on close (#3713)
630aa7d9 implement observability for room metrics (#3712)
e7f0294e remove unused ws signal read loop (#3709)
77f70b18 for real, pick up protocol change for webhooks queue length bucnkets (#3707)
7b180646 protocol dep for webhook stats buckets (#3706)
b0ab95ba warn about credentials when used in tokens (#3705)
a72ce30f Small changes to add/use helper functions for length checks. (#3704)
425f6bb3 Allow passing extra attributes to RTC endpoint. (#3693)
758e1762 Add a trend check before declaring joint queuing region. (#3701)
fe81e411 Adds Devin to readme so it auto updates DeepWiki weekly (#3699)
09dede35 version bump v1.9.0 (#3698)
fc867c5b Webhook prom stats (#3697)
0e17916f Do not use Redis pipeline for SIP delete. Fixes Redis clustering support. (#3694)
1b760393 WHIP support. (#3692)
e4f7d81b add client ip to agent worker registration (#3675)
6b849a87 update mediatransportutil for sctp congestion control (#3673)
83b189b0 Add ServerInfo to ReconnectResponse (#3671)
5f87a35b Prevent operating on swapped out map. (#3670)
13b55a80 move agent token (#3669)
c9385edd handle agent worker jwt (#3668)
3b359d8b Use logger resolver reset to reset contexts. (#3665)
dbb70e0f Fix dynacast quality for moving out tracks (#3664)
0a5f3c2a resolve new room name logger earlier when moving participant (#3662)
2df05517 Revert unbound transceiver stop. (#3661)
5172af15 ~Send initial participant update only after a participant becomes active.~ - General clean up (#3655)
5d4d86f8 protocol to pick resolver values replace (#3659)
7f8e6323 Send self participant update immediately. (#3656)
11630878 Use unordered for lossy data channel. (#3653)
aee34ffe log request for agent dispatch api (#3650)
793b383a Add Moving participant to another room (#3648)
2fff36cb Stub MoveParticipant so that cloud can include the latest protocol. (#3646)
d4ab1142 Redact address (#3643)
b83190a3 protocol update to fix memory stats path (#3642)
a1f4e88e Update protocol to latest, got bit by tag (#3641)
8d3902af Protocol to pick up cgroups v2 memory path fix (#3640)
2f002388 Use participant close reason in remove. (#3639)
58822c26 Include clientInfo in connectivity logs. (#3638)
6d6393a6 Take AudioFeatures from AddTrack. (#3635)
08670412 Limit buffer queue before Bind. (#3634)
9f5bc9b9 Avoid synthesising duplicate feature. (#3632)
847239c3 Disable vp9 for safari 18.4 (#3631)
f69ab680 Populate the sender identity when translating to user packet. (#3628)
e1490558 Forward data between WHIP client and non-WHIP client (#3627)
6739e7bc Broadcast inside lock (#3626)
f24152b4 Call Broadcast in lock scope. (#3625)
b760918a Use logger from request context. (#3623)
34a2e2c1 Check for multiple layers for managed track. (#3622)
4955ebe4 Forward transfer headers to internal request (#3615)
d9ee9214 Set up RTX for WHIP publish (#3619)
d8cf5439 Determine TURN connection type and no fallback for TURN/TLS. (#3612)
d0d212fd Fix WHIP ICE restart. (#3616)
5e7f8a12 Update mediatransportutil for max sctp message (65K) (#3611)
28dfac14 Use exported GetEgressNotifyOptions (#3604)
75236bef protocol update to fix IPv6 SDP fragment parsing (#3603)
2130980d Add basic video support to WHIP. (#3602)
e5cbb227 Allow specifying extra webhooks with egress requests (#3597)
7e16106a Add OnSubscirberReady callback on LocalParticipant. (#3600)
5c2d96b9 Check DestinationRoom of VideoGrant for participant forwarding (#3599)
2e236a19 Revert participant state ACTIVE change. (#3598)
35ac5f56 Add support for WHIP ICE Trickle/Restart. (#3596)
ec2dff96 Fix SIP updates when replacing slices. (#3592)
e24fe77b map PEER_CONNECTION_DISCONNECTED -> CONNECTION_TIMEOUT (#3591)
6ee6eb43 Do not drop audio codecs (#3590)
68357ba6 List audio codecs after video codecs. (#3589)
05a891ff Fix rule (had an extra bracket) (#3588)
d7c41091 Exclude RED from enabled codecs for Flutter + 2.4.2 + Android. (#3587)
ee08aede skip out of order participant state updates (#3583)
15a8d9a2 Break track published fuse when there are no tracks (#3581)
35f83c51 Replace Promise with Fuse. (#3580)
07fe9b72 Prevent migration race. (#3579)
ac8082ff Use older SDP module to accommodate bad SDP. (#3578)
1c8307c7 Use cgroup for memstats. (#3573)
e9be0fca log SDP offer on error (#3577)
3238ab8d Calculate rates for memory used and total. (#3570)
d08487bf Unlabeled (pass through) data channels. (#3567)
52ce18d5 fix: revert recent changes to determine simulcast from sdp (#3565)
cdfbb106 Audio uses signal SignalCid and SdpCid. (#3564)
ed5e2f16 Keep simulcast information tied to receiver. (#3563)
ad010cfc chore(logs): log VLS type for VP9/AV1 (#3561)
8cc17f8f Rework node stats a bit. (#3555)
15f56551 fix(video): determine svc/simulcast from SDP for advanced codecs (#3549)
2b6a46f4 Handle `prefer_regression` for backup codec (#3554)
b0abb0ae Add option to use different pacer with send side bwe. (#3552)
26822b6b ParseUsername utility for TURN user name. (#3547)
55909ed7 log the initial join response (#3546)
97fcb82a Fix: Return NotFoundErr instead of Unavailable when the participant does not exist in UpdateParticipant. (#3543)
75d0e18e Implement SIP update API. (#3141)
e118aff1 Fire track subscribed when the subscriber connected (#3540)
13417c01 Send mute event only on change (#3537)
7f4c4597 Stubs for SIP update API. (#3533)
fe673bb2 Send regressed codec upstream stats to analytics. (#3532)
8eb81388 Use a generation to counter to stop key frame seeder on codec change (#3531)
188470a2 Do not accept unsupported track type in AddTrack (#3530)
507fc9cf Do not instantiate 0 sized sequencer. (#3529)
20bddfea Clean up published track on participant removal. (#3527)
65d8aa28 Handle subscribe race with track close better. (#3526)
a6cb00b3 Reduce seeder duration to 30s and also do not force send PLI. (#3525)
c8233205 Add a key frame seeder in up track. (#3524)
0f61ff3a Remove redundant log (#3523)
7685cd25 Log ParticipantInit on signal start to get a picture of join params (#3522)
ac9e62ef add server agent load threshold config (#3520)
cd5d32f0 Add pID and connID to log context to make it easier to search using pID. (#3518)
2d9aa6dd Update api call info method (#3515)
b3779a90 WebHookConfig (#3517)
6121b9af Check ForwardParticipant room name (#3514)
9a7c9442 mediatransportutil update (#3511)
50ab47c1 Log packet drops/forward. (#3510)
139d1b13 Add ForwardParticipant method to room service (#3507)
6c04909f Use atomic to store codec. (#3505)
7f6afe05 Prevent bind lock deadlock on muted. (#3504)
48063df5 load mime type before calling writeBlankFrameRTP (#3502)
d2e6cd15 Do not bind lock across flush which could take time (#3501)
47896f50 Update protocol and IO service. (#3499)
1dc42eef Bump github.com/go-jose/go-jose/v3 from 3.0.3 to 3.0.4 (#3497)
3a35cbc4 Log migration complete only when coming from sync (#3496)
c2f17a10 refactor: using slices.Contains to simplify the code (#3495)
01e51dbd fix: fix the wrong error return value (#3493)
ff9115b2 Disable dd parser for vp8 if extension is not found (#3492)
c3e06f05 Do not attempt to create objects for URL ingresses as the ingress service will do so (#3491)
f0edfbba Fix receiver rtt/jitter. (#3487)
05dfd30d Take RTT and jitter from receiver view while reporting track stats for (#3483)
04ed5683 Don't issue TrackPublished/Unpublished event on migrated track (#3482)
1cffe30c Use a RED transformer to consolidate both RED -> Opus OR Opus -> RED (#3481)
591888f7 Fix missing RTCP sender report when forwarding RED as Opus. (#3480)

git-subtree-dir: livekit-server
git-subtree-split: 46c4309554d37d23ee8da88a8a7e02a68fba09c1
This commit is contained in:
ttipo
2026-06-25 23:54:33 +09:00
parent 0da97ebd21
commit 6bd7fac875
291 changed files with 37631 additions and 15381 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pkg/sfu/audio/audiolevel.go
+53 -12
View File
@@ -51,12 +51,15 @@ var (
// --------------------------------------
type AudioLevelParams struct {
Config AudioLevelConfig
ClockRate uint32
}
// keeps track of audio level for a participant
type AudioLevel struct {
params AudioLevelParams
config AudioLevelConfig
// min duration within an observe duration window to be considered active
minActiveDuration uint32
smoothFactor float64
@@ -69,23 +72,32 @@ type AudioLevel struct {
activeDuration uint32 // ms
observedDuration uint32 // ms
lastObservedAt int64
highestRTPTimestamp uint32
highestRTPTimestampInitialized bool
}
func NewAudioLevel(params AudioLevelParams) *AudioLevel {
l := &AudioLevel{
return &AudioLevel{
params: params,
minActiveDuration: uint32(params.Config.MinPercentile) * params.Config.UpdateInterval / 100,
smoothFactor: 1,
activeThreshold: ConvertAudioLevel(float64(params.Config.ActiveLevel)),
loudestObservedLevel: silentAudioLevel,
}
}
if l.params.Config.SmoothIntervals > 0 {
func (l *AudioLevel) SetConfig(config AudioLevelConfig) {
l.lock.Lock()
defer l.lock.Unlock()
l.config = config
l.minActiveDuration = uint32(l.config.MinPercentile) * l.config.UpdateInterval / 100
if l.config.SmoothIntervals > 0 {
// exponential moving average (EMA), same center of mass with simple moving average (SMA)
l.smoothFactor = float64(2) / (float64(l.params.Config.SmoothIntervals + 1))
l.smoothFactor = float64(2) / (float64(l.config.SmoothIntervals + 1))
} else {
l.smoothFactor = 1
}
return l
l.activeThreshold = ConvertAudioLevel(float64(l.config.ActiveLevel))
}
// Observes a new frame
@@ -93,18 +105,26 @@ func (l *AudioLevel) Observe(level uint8, durationMs uint32, arrivalTime int64)
l.lock.Lock()
defer l.lock.Unlock()
l.observeLocked(level, durationMs, arrivalTime)
}
func (l *AudioLevel) observeLocked(level uint8, durationMs uint32, arrivalTime int64) {
if l.config.UpdateInterval == 0 {
return
}
l.lastObservedAt = arrivalTime
l.observedDuration += durationMs
if level <= l.params.Config.ActiveLevel {
if level <= l.config.ActiveLevel {
l.activeDuration += durationMs
if l.loudestObservedLevel > level {
l.loudestObservedLevel = level
}
}
if l.observedDuration >= l.params.Config.UpdateInterval {
if l.observedDuration >= l.config.UpdateInterval {
smoothedLevel := float64(0.0)
// compute and reset
if l.activeDuration >= l.minActiveDuration {
@@ -112,7 +132,7 @@ func (l *AudioLevel) Observe(level uint8, durationMs uint32, arrivalTime int64)
// Weight will be 0 if active the entire duration
// > 0 if active for longer than observe duration
// < 0 if active for less than observe duration
activityWeight := 20 * math.Log10(float64(l.activeDuration)/float64(l.params.Config.UpdateInterval))
activityWeight := 20 * math.Log10(float64(l.activeDuration)/float64(l.config.UpdateInterval))
adjustedLevel := float64(l.loudestObservedLevel) - activityWeight
linearLevel := ConvertAudioLevel(adjustedLevel)
@@ -123,18 +143,39 @@ func (l *AudioLevel) Observe(level uint8, durationMs uint32, arrivalTime int64)
}
}
func (l *AudioLevel) ObserveWithRTPTimestamp(level uint8, ts uint32, arrivalTime int64) {
l.lock.Lock()
defer l.lock.Unlock()
if !l.highestRTPTimestampInitialized {
l.highestRTPTimestampInitialized = true
l.highestRTPTimestamp = ts
}
if (ts - l.highestRTPTimestamp) < (1 << 31) {
durationMs := (ts - l.highestRTPTimestamp) * 1e3 / l.params.ClockRate
l.observeLocked(level, durationMs, arrivalTime)
l.highestRTPTimestamp = ts
}
}
// returns current smoothed audio level
func (l *AudioLevel) GetLevel(now int64) (float64, bool) {
l.lock.Lock()
defer l.lock.Unlock()
if l.config.UpdateInterval == 0 {
return 0.0, false
}
l.resetIfStaleLocked(now)
return l.smoothedLevel, l.smoothedLevel >= l.activeThreshold
}
func (l *AudioLevel) resetIfStaleLocked(arrivalTime int64) {
if (arrivalTime-l.lastObservedAt)/1e6 < int64(2*l.params.Config.UpdateInterval) {
if (arrivalTime-l.lastObservedAt)/1e6 < int64(2*l.config.UpdateInterval) {
return
}
pkg/sfu/audio/audiolevel_test.go
+45 -10
View File
@@ -15,6 +15,7 @@
package audio
import (
"math/rand"
"testing"
"time"
@@ -104,20 +105,54 @@ func TestAudioLevel(t *testing.T) {
require.Equal(t, float64(0.0), level)
require.False(t, noisy)
})
}
func createAudioLevel(activeLevel uint8, minPercentile uint8, observeDuration uint32) *AudioLevel {
return NewAudioLevel(AudioLevelParams{
Config: AudioLevelConfig{
ActiveLevel: activeLevel,
MinPercentile: minPercentile,
UpdateInterval: observeDuration,
},
t.Run("not noisy when samples are stale - with RTP timestamp", func(t *testing.T) {
clock := time.Now()
a := createAudioLevel(defaultActiveLevel, defaultPercentile, defaultObserveDuration)
observeSamplesWithRTPTimestamp(a, 25, 100, clock)
clock = clock.Add(100 * 20 * time.Millisecond)
level, noisy := a.GetLevel(clock.UnixNano())
require.True(t, noisy)
require.Greater(t, level, ConvertAudioLevel(float64(defaultActiveLevel)))
require.Less(t, level, ConvertAudioLevel(float64(20)))
// let enough time pass to make the samples stale
clock = clock.Add(1500 * time.Millisecond)
level, noisy = a.GetLevel(clock.UnixNano())
require.Equal(t, float64(0.0), level)
require.False(t, noisy)
})
}
func createAudioLevel(activeLevel uint8, minPercentile uint8, observeDuration uint32) *AudioLevel {
al := NewAudioLevel(AudioLevelParams{
ClockRate: 48000,
})
al.SetConfig(AudioLevelConfig{
ActiveLevel: activeLevel,
MinPercentile: minPercentile,
UpdateInterval: observeDuration,
})
return al
}
func observeSamples(a *AudioLevel, level uint8, count int, baseTime time.Time) {
for i := 0; i < count; i++ {
a.Observe(level, 20, baseTime.Add(+time.Duration(i*20)*time.Millisecond).UnixNano())
for i := range count {
a.Observe(level, 20, baseTime.Add(time.Duration(i*20)*time.Millisecond).UnixNano())
}
}
func observeSamplesWithRTPTimestamp(a *AudioLevel, level uint8, count int, baseTime time.Time) {
sampleTS := uint32(rand.Intn(1 << 20))
sampleTime := baseTime
for i := range count {
if (i % 5) == 0 {
// out-of-order sample
a.ObserveWithRTPTimestamp(level, sampleTS-1920, sampleTime.UnixNano())
}
a.ObserveWithRTPTimestamp(level, sampleTS, sampleTime.UnixNano())
sampleTS += 960 // 20 ms at 48 kHz
sampleTime = sampleTime.Add(20 * time.Millisecond)
}
}
pkg/sfu/buffer/buffer.go
+150 -918
View File
File diff suppressed because it is too large Load Diff
pkg/sfu/buffer/buffer_base.go
+1597
View File
File diff suppressed because it is too large Load Diff
pkg/sfu/buffer/buffer_test.go
+58 -14
View File
@@ -15,6 +15,7 @@
package buffer
import (
"fmt"
"math"
"sync"
"testing"
@@ -82,7 +83,7 @@ func TestNack(t *testing.T) {
}, vp8Codec.RTPCodecCapability, 0)
rtt := uint32(20)
buff.nacker.SetRTT(rtt)
for i := 0; i < 15; i++ {
for i := range 15 {
if i == 1 {
continue
}
@@ -147,7 +148,7 @@ func TestNack(t *testing.T) {
}, vp8Codec.RTPCodecCapability, 0)
rtt := uint32(30)
buff.nacker.SetRTT(rtt)
for i := 0; i < 15; i++ {
for i := range 15 {
if i > 0 && i < 5 {
continue
}
@@ -265,7 +266,7 @@ func TestFractionLostReport(t *testing.T) {
HeaderExtensions: nil,
Codecs: []webrtc.RTPCodecParameters{opusCodec},
}, opusCodec.RTPCodecCapability, 0)
for i := 0; i < 15; i++ {
for i := range 15 {
pkt := rtp.Packet{
Header: rtp.Header{
Version: 2,
@@ -303,7 +304,7 @@ func TestFractionLostReport(t *testing.T) {
HeaderExtensions: nil,
Codecs: []webrtc.RTPCodecParameters{opusCodec},
}, opusCodec.RTPCodecCapability, 0)
for i := 0; i < 15; i++ {
for i := range 15 {
pkt := rtp.Packet{
Header: rtp.Header{
Version: 2,
@@ -337,6 +338,15 @@ func TestCodecChange(t *testing.T) {
t.Fatalf("codec change not consumed")
}
})
buff.OnStreamRestart(func(reason string) {
require.Equal(t, "codec-change", reason)
// read once to clear pending restart
var buf [1500]byte
extPkt, err := buff.ReadExtended(buf[:])
require.NoError(t, err)
require.Nil(t, extPkt)
})
h265Pkt := rtp.Packet{
Header: rtp.Header{
@@ -359,20 +369,27 @@ func TestCodecChange(t *testing.T) {
case <-time.After(100 * time.Millisecond):
}
buff.Bind(webrtc.RTPParameters{
HeaderExtensions: nil,
Codecs: []webrtc.RTPCodecParameters{vp8Codec, h265Codec},
}, vp8Codec.RTPCodecCapability, 0)
// Bind sets up VP8 as expected codec,
// packet written to the buffer above before Bind is H.265,
// that should trigger a codec change and a stream restart
// when the queued packets from Write before Bind get flushed
buff.Bind(
webrtc.RTPParameters{
HeaderExtensions: nil,
Codecs: []webrtc.RTPCodecParameters{vp8Codec, h265Codec},
},
vp8Codec.RTPCodecCapability,
0,
)
select {
case c := <-changedCodec:
require.Equal(t, h265Codec, c)
case <-time.After(1 * time.Second):
t.Fatalf("expected codec change")
}
// codec change after bind
// second codec change - writing VP8 packet after Bind should trigger another codec change
vp8Pkt := rtp.Packet{
Header: rtp.Header{
Version: 2,
@@ -391,12 +408,16 @@ func TestCodecChange(t *testing.T) {
select {
case c := <-changedCodec:
require.Equal(t, vp8Codec, c)
case <-time.After(1 * time.Second):
t.Fatalf("expected codec change")
}
fmt.Printf("done second codec change\n") // REMOVE
// out of order pkts can't cause codec change
// rewrite the VP8 packet to start the sequence after a stream restart
_, err = buff.Write(buf)
require.NoError(t, err)
h265Pkt.SequenceNumber = 2
h265Pkt.Timestamp = 2
buf, err = h265Pkt.Marshal()
@@ -409,7 +430,7 @@ func TestCodecChange(t *testing.T) {
case <-time.After(100 * time.Millisecond):
}
// unknown codec should not cause change
// unknown codec should not cause change even if it is in order
h265Pkt.SequenceNumber = 4
h265Pkt.Timestamp = 4
h265Pkt.PayloadType = 117
@@ -422,14 +443,37 @@ func TestCodecChange(t *testing.T) {
t.Fatalf("unexpected codec change")
case <-time.After(100 * time.Millisecond):
}
// an in-order packet should change codec again
h265Pkt.SequenceNumber = 5
h265Pkt.Timestamp = 5
h265Pkt.PayloadType = 116
buf, err = h265Pkt.Marshal()
require.NoError(t, err)
_, err = buff.Write(buf)
require.NoError(t, err)
select {
case c := <-changedCodec:
require.Equal(t, h265Codec, c)
case <-time.After(1 * time.Second):
t.Fatalf("expected codec change")
}
}
func BenchmarkMemcpu(b *testing.B) {
buf := make([]byte, 1500*1500*10)
buf2 := make([]byte, 1500*1500*20)
b.ResetTimer()
for i := 0; i < b.N; i++ {
for b.Loop() {
copy(buf2, buf)
}
}
func BenchmarkExtPacketFactory(b *testing.B) {
for b.Loop() {
extPkt := ExtPacketFactory.Get().(*ExtPacket)
*extPkt = ExtPacket{}
ExtPacketFactory.Put(extPkt)
}
}
pkg/sfu/buffer/dependencydescriptorparser.go
+139 -17
View File
@@ -16,20 +16,41 @@ package buffer
import (
"fmt"
"sort"
"slices"
"sync"
"time"
"github.com/pion/rtp"
"go.uber.org/atomic"
dd "github.com/livekit/livekit-server/pkg/sfu/rtpextension/dependencydescriptor"
"github.com/livekit/livekit-server/pkg/sfu/utils"
"github.com/livekit/mediatransportutil/pkg/codec"
"github.com/livekit/mediatransportutil/pkg/utils"
"github.com/livekit/protocol/logger"
)
var (
ExtDependencyDescriptorFactory = &sync.Pool{
New: func() any {
return &ExtDependencyDescriptor{}
},
}
)
// --------------------------------------
const (
ddRestartThreshold = 30 * time.Second
// frame integrity check 2 seconds for L3T3 30fps video
integrityCheckFrame = 180
integrityCheckPkt = 1024
)
var (
ErrFrameEarlierThanKeyFrame = fmt.Errorf("frame is earlier than current keyframe")
ErrDDStructureAttachedToNonFirstPacket = fmt.Errorf("dependency descriptor structure is attached to non-first packet of a frame")
ErrDDExtentionNotFound = fmt.Errorf("dependency descriptor extension not found")
)
type DependencyDescriptorParser struct {
@@ -47,16 +68,22 @@ type DependencyDescriptorParser struct {
frameChecker *FrameIntegrityChecker
ddNotFoundCount atomic.Uint32
// restart detection
restartGeneration int
enableRestart bool
lastPacketAt time.Time
}
func NewDependencyDescriptorParser(ddExtID uint8, logger logger.Logger, onMaxLayerChanged func(int32, int32)) *DependencyDescriptorParser {
func NewDependencyDescriptorParser(ddExtID uint8, logger logger.Logger, onMaxLayerChanged func(int32, int32), enableRestart bool) *DependencyDescriptorParser {
return &DependencyDescriptorParser{
ddExtID: ddExtID,
logger: logger,
onMaxLayerChanged: onMaxLayerChanged,
seqWrapAround: utils.NewWrapAround[uint16, uint64](utils.WrapAroundParams{IsRestartAllowed: false}),
frameWrapAround: utils.NewWrapAround[uint16, uint64](utils.WrapAroundParams{IsRestartAllowed: false}),
frameChecker: NewFrameIntegrityChecker(180, 1024), // 2seconds for L3T3 30fps video
frameChecker: NewFrameIntegrityChecker(integrityCheckFrame, integrityCheckPkt),
enableRestart: enableRestart,
}
}
@@ -70,6 +97,10 @@ type ExtDependencyDescriptor struct {
ExtFrameNum uint64
// the frame number of the keyframe which the current frame depends on
ExtKeyFrameNum uint64
// increase when the stream restarts, clear and reinitialize all dd state includes
// attached structure, frame chain, decode target.
RestartGeneration int
}
func (r *DependencyDescriptorParser) Parse(pkt *rtp.Packet) (*ExtDependencyDescriptor, VideoLayer, error) {
@@ -80,7 +111,7 @@ func (r *DependencyDescriptorParser) Parse(pkt *rtp.Packet) (*ExtDependencyDescr
if ddNotFoundCount%100 == 0 {
r.logger.Warnw("dependency descriptor extension is not present", nil, "seq", pkt.SequenceNumber, "count", ddNotFoundCount)
}
return nil, videoLayer, nil
return nil, videoLayer, ErrDDExtentionNotFound
}
var ddVal dd.DependencyDescriptor
@@ -96,41 +127,92 @@ func (r *DependencyDescriptorParser) Parse(pkt *rtp.Packet) (*ExtDependencyDescr
return nil, videoLayer, err
}
var restart bool
if r.enableRestart {
if !r.lastPacketAt.IsZero() && time.Since(r.lastPacketAt) > ddRestartThreshold {
r.restart()
restart = true
r.logger.Debugw(
"dependency descriptor parser restart stream",
"generation", r.restartGeneration,
"lastPacketAt", r.lastPacketAt,
"sinceLast", time.Since(r.lastPacketAt),
"frameWrapAround", r.frameWrapAround,
)
}
r.lastPacketAt = time.Now()
}
extSeq := r.seqWrapAround.Update(pkt.SequenceNumber).ExtendedVal
if ddVal.FrameDependencies != nil {
videoLayer.Spatial, videoLayer.Temporal = int32(ddVal.FrameDependencies.SpatialId), int32(ddVal.FrameDependencies.TemporalId)
}
unwrapped := r.frameWrapAround.Update(ddVal.FrameNumber)
// assume the packet is in-order when stream restarting
unwrapped := r.frameWrapAround.UpdateWithOrderKnown(ddVal.FrameNumber, restart)
extFN := unwrapped.ExtendedVal
if extFN < r.structureExtFrameNum {
r.logger.Debugw("drop frame which is earlier than current structure", "frameNum", extFN, "structureFrameNum", r.structureExtFrameNum)
r.logger.Debugw(
"drop frame which is earlier than current structure",
"fn", ddVal.FrameNumber,
"extFN", extFN,
"structureExtFrameNum", r.structureExtFrameNum,
"unwrappedFN", unwrapped,
"frameWrapAround", r.frameWrapAround,
)
return nil, videoLayer, ErrFrameEarlierThanKeyFrame
}
r.frameChecker.AddPacket(extSeq, extFN, &ddVal)
extDD := &ExtDependencyDescriptor{
Descriptor: &ddVal,
ExtFrameNum: extFN,
Integrity: r.frameChecker.FrameIntegrity(extFN),
extDD := ExtDependencyDescriptorFactory.Get().(*ExtDependencyDescriptor)
*extDD = ExtDependencyDescriptor{
Descriptor: &ddVal,
ExtFrameNum: extFN,
Integrity: r.frameChecker.FrameIntegrity(extFN),
RestartGeneration: r.restartGeneration,
}
if ddVal.AttachedStructure != nil {
if !ddVal.FirstPacketInFrame {
r.logger.Warnw("attached structure is not the first packet in frame", nil, "extSeq", extSeq, "extFN", extFN)
r.logger.Warnw(
"attached structure is not the first packet in frame", nil,
"sn", pkt.SequenceNumber,
"extSeq", extSeq,
"fn", ddVal.FrameNumber,
"extFN", extFN,
)
ReleaseExtDependencyDescriptor(extDD)
return nil, videoLayer, ErrDDStructureAttachedToNonFirstPacket
}
if r.structure == nil || ddVal.AttachedStructure.StructureId != r.structure.StructureId {
r.logger.Debugw("structure updated", "structureID", ddVal.AttachedStructure.StructureId, "extSeq", extSeq, "extFN", extFN, "descriptor", ddVal.String())
r.logger.Debugw(
"structure updated",
"structureID", ddVal.AttachedStructure.StructureId,
"sn", pkt.SequenceNumber,
"extSeq", extSeq,
"fn", ddVal.FrameNumber,
"extFN", extFN,
"descriptor", ddVal.String(),
"unwrappedFN", unwrapped,
"frameWrapAround", r.frameWrapAround,
)
}
r.structure = ddVal.AttachedStructure
r.decodeTargets = ProcessFrameDependencyStructure(ddVal.AttachedStructure)
if extFN > unwrapped.PreExtendedHighest && extFN-unwrapped.PreExtendedHighest > 1000 {
r.logger.Debugw("large frame number jump on structure updating", "extFN", extFN, "preExtendedHighest", unwrapped.PreExtendedHighest, "structureExtFrameNum", r.structureExtFrameNum)
r.logger.Debugw(
"large frame number jump on structure updating",
"fn", ddVal.FrameNumber,
"extFN", extFN,
"preExtendedHighest", unwrapped.PreExtendedHighest,
"structureExtFrameNum", r.structureExtFrameNum,
"unwrappedFN", unwrapped,
"frameWrapAround", r.frameWrapAround,
)
}
r.structureExtFrameNum = extFN
extDD.StructureUpdated = true
@@ -166,6 +248,16 @@ func (r *DependencyDescriptorParser) Parse(pkt *rtp.Packet) (*ExtDependencyDescr
return extDD, videoLayer, nil
}
func (r *DependencyDescriptorParser) restart() {
r.frameChecker = NewFrameIntegrityChecker(integrityCheckFrame, integrityCheckPkt)
r.structure = nil
r.structureExtFrameNum = 0
r.activeDecodeTargetsExtSeq = 0
r.activeDecodeTargetsMask = 0
r.decodeTargets = r.decodeTargets[:0]
r.restartGeneration++
}
// ------------------------------------------------------------------------------
type DependencyDescriptorDecodeTarget struct {
@@ -197,8 +289,14 @@ func ProcessFrameDependencyStructure(structure *dd.FrameDependencyStructure) []D
}
// sort decode target layer by spatial and temporal from high to low
sort.Slice(decodeTargets, func(i, j int) bool {
return decodeTargets[i].Layer.GreaterThan(decodeTargets[j].Layer)
slices.SortFunc(decodeTargets, func(a, b DependencyDescriptorDecodeTarget) int {
if a.Layer.GreaterThan(b.Layer) {
return -1
}
if b.Layer.GreaterThan(a.Layer) {
return 1
}
return 0
})
return decodeTargets
@@ -216,3 +314,27 @@ func GetActiveDecodeTargetBitmask(layer VideoLayer, decodeTargets []DependencyDe
}
// ------------------------------------------------------------------------------
func ExtractDependencyDescriptorVideoSize(dd *dd.DependencyDescriptor) []codec.VideoSize {
if dd.AttachedStructure == nil {
return nil
}
videoSizes := make([]codec.VideoSize, 0, len(dd.AttachedStructure.Resolutions))
for _, res := range dd.AttachedStructure.Resolutions {
videoSizes = append(videoSizes, codec.VideoSize{Width: uint32(res.Width), Height: uint32(res.Height)})
}
return videoSizes
}
// ------------------------------------------------------------------------------
func ReleaseExtDependencyDescriptor(extDD *ExtDependencyDescriptor) {
if extDD == nil {
return
}
*extDD = ExtDependencyDescriptor{}
ExtDependencyDescriptorFactory.Put(extDD)
}
pkg/sfu/buffer/factory.go
+5 -2
View File
@@ -18,7 +18,7 @@ import (
"io"
"sync"
"github.com/pion/transport/v3/packetio"
"github.com/pion/transport/v4/packetio"
)
type FactoryOfBufferFactory struct {
@@ -118,7 +118,7 @@ func (f *Factory) GetRTCPReader(ssrc uint32) *RTCPReader {
return f.rtcpReaders[ssrc]
}
func (f *Factory) SetRTXPair(repair, base uint32) {
func (f *Factory) SetRTXPair(repair, base uint32, rsid string) {
f.Lock()
repairBuffer, baseBuffer := f.rtpBuffers[repair], f.rtpBuffers[base]
if repairBuffer == nil || baseBuffer == nil {
@@ -127,5 +127,8 @@ func (f *Factory) SetRTXPair(repair, base uint32) {
f.Unlock()
if repairBuffer != nil && baseBuffer != nil {
repairBuffer.SetPrimaryBufferForRTX(baseBuffer)
if rsid != "" {
baseBuffer.NotifyRTX(base, repair, rsid)
}
}
}
pkg/sfu/buffer/fps.go
+2 -1
View File
@@ -19,6 +19,7 @@ import (
"github.com/pion/rtp/codecs"
"github.com/livekit/mediatransportutil/pkg/codec"
"github.com/livekit/protocol/logger"
)
@@ -209,7 +210,7 @@ func (f *FrameRateCalculatorVP8) RecvPacket(ep *ExtPacket) bool {
return true
}
vp8, ok := ep.Payload.(VP8)
vp8, ok := ep.Payload.(codec.VP8)
if !ok {
f.logger.Debugw("no vp8 payload", "sn", ep.Packet.SequenceNumber)
return false
pkg/sfu/buffer/fps_test.go
+7 -6
View File
@@ -21,6 +21,7 @@ import (
"github.com/stretchr/testify/require"
dd "github.com/livekit/livekit-server/pkg/sfu/rtpextension/dependencydescriptor"
"github.com/livekit/mediatransportutil/pkg/codec"
"github.com/livekit/protocol/logger"
)
@@ -35,7 +36,7 @@ type testFrameInfo struct {
func (f *testFrameInfo) toVP8() *ExtPacket {
return &ExtPacket{
Packet: &rtp.Packet{Header: f.header},
Payload: VP8{
Payload: codec.VP8{
PictureID: f.framenumber,
},
VideoLayer: VideoLayer{Spatial: InvalidLayerSpatial, Temporal: int32(f.temporal)},
@@ -75,7 +76,7 @@ func createFrames(startFrameNumber uint16, startTs uint32, startSeq uint16, tota
nextTs := make([][]uint32, spatials)
tsStep := make([][]uint32, spatials)
for i := 0; i < spatials; i++ {
for i := range spatials {
nextTs[i] = make([]uint32, temporals)
tsStep[i] = make([]uint32, temporals)
for j := 0; j < temporals; j++ {
@@ -85,11 +86,11 @@ func createFrames(startFrameNumber uint16, startTs uint32, startSeq uint16, tota
}
currentTs := make([]uint32, spatials)
for i := 0; i < spatials; i++ {
for i := range spatials {
currentTs[i] = startTs
}
for i := 0; i < totalFramesPerSpatial; i++ {
for s := 0; s < spatials; s++ {
for range totalFramesPerSpatial {
for s := range spatials {
frame := &testFrameInfo{
header: rtp.Header{Timestamp: currentTs[s], SequenceNumber: startSeq},
framenumber: fn,
@@ -134,7 +135,7 @@ func createFrames(startFrameNumber uint16, startTs uint32, startSeq uint16, tota
func verifyFps(t *testing.T, expect, got []float32) {
require.Equal(t, len(expect), len(got))
for i := 0; i < len(expect); i++ {
for i := range expect {
require.GreaterOrEqual(t, got[i], expect[i]*0.9, "expect %v, got %v", expect, got)
require.LessOrEqual(t, got[i], expect[i]*1.1, "expect %v, got %v", expect, got)
}
pkg/sfu/buffer/helpers.go
-455
View File
@@ -1,455 +0,0 @@
// Copyright 2023 LiveKit, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package buffer
import (
"encoding/binary"
"errors"
"github.com/pion/rtp/codecs"
"github.com/livekit/protocol/logger"
)
var (
errShortPacket = errors.New("packet is not large enough")
errNilPacket = errors.New("invalid nil packet")
errInvalidPacket = errors.New("invalid packet")
)
// VP8 is a helper to get temporal data from VP8 packet header
/*
VP8 Payload Descriptor
0 1 2 3 4 5 6 7 0 1 2 3 4 5 6 7
+-+-+-+-+-+-+-+-+ +-+-+-+-+-+-+-+-+
|X|R|N|S|R| PID | (REQUIRED) |X|R|N|S|R| PID | (REQUIRED)
+-+-+-+-+-+-+-+-+ +-+-+-+-+-+-+-+-+
X: |I|L|T|K| RSV | (OPTIONAL) X: |I|L|T|K| RSV | (OPTIONAL)
+-+-+-+-+-+-+-+-+ +-+-+-+-+-+-+-+-+
I: |M| PictureID | (OPTIONAL) I: |M| PictureID | (OPTIONAL)
+-+-+-+-+-+-+-+-+ +-+-+-+-+-+-+-+-+
L: | TL0PICIDX | (OPTIONAL) | PictureID |
+-+-+-+-+-+-+-+-+ +-+-+-+-+-+-+-+-+
T/K:|TID|Y| KEYIDX | (OPTIONAL) L: | TL0PICIDX | (OPTIONAL)
+-+-+-+-+-+-+-+-+ +-+-+-+-+-+-+-+-+
T/K:|TID|Y| KEYIDX | (OPTIONAL)
+-+-+-+-+-+-+-+-+
*/
type VP8 struct {
FirstByte byte
S bool
I bool
M bool
PictureID uint16 /* 7 or 15 bits, picture ID */
L bool
TL0PICIDX uint8 /* 8 bits temporal level zero index */
// Optional Header If either of the T or K bits are set to 1,
// the TID/Y/KEYIDX extension field MUST be present.
T bool
TID uint8 /* 2 bits temporal layer idx */
Y bool
K bool
KEYIDX uint8 /* 5 bits of key frame idx */
HeaderSize int
// IsKeyFrame is a helper to detect if current packet is a keyframe
IsKeyFrame bool
}
// Unmarshal parses the passed byte slice and stores the result in the VP8 this method is called upon
func (v *VP8) Unmarshal(payload []byte) error {
if payload == nil {
return errNilPacket
}
payloadLen := len(payload)
if payloadLen < 1 {
return errShortPacket
}
idx := 0
v.FirstByte = payload[idx]
v.S = payload[idx]&0x10 > 0
// Check for extended bit control
if payload[idx]&0x80 > 0 {
idx++
if payloadLen < idx+1 {
return errShortPacket
}
v.I = payload[idx]&0x80 > 0
v.L = payload[idx]&0x40 > 0
v.T = payload[idx]&0x20 > 0
v.K = payload[idx]&0x10 > 0
if v.L && !v.T {
return errInvalidPacket
}
if v.I {
idx++
if payloadLen < idx+1 {
return errShortPacket
}
pid := payload[idx] & 0x7f
// if m is 1, then Picture ID is 15 bits
v.M = payload[idx]&0x80 > 0
if v.M {
idx++
if payloadLen < idx+1 {
return errShortPacket
}
v.PictureID = binary.BigEndian.Uint16([]byte{pid, payload[idx]})
} else {
v.PictureID = uint16(pid)
}
}
if v.L {
idx++
if payloadLen < idx+1 {
return errShortPacket
}
v.TL0PICIDX = payload[idx]
}
if v.T || v.K {
idx++
if payloadLen < idx+1 {
return errShortPacket
}
if v.T {
v.TID = (payload[idx] & 0xc0) >> 6
v.Y = (payload[idx] & 0x20) > 0
}
if v.K {
v.KEYIDX = payload[idx] & 0x1f
}
}
idx++
if payloadLen < idx+1 {
return errShortPacket
}
// Check is packet is a keyframe by looking at P bit in vp8 payload
v.IsKeyFrame = payload[idx]&0x01 == 0 && v.S
} else {
idx++
if payloadLen < idx+1 {
return errShortPacket
}
// Check is packet is a keyframe by looking at P bit in vp8 payload
v.IsKeyFrame = payload[idx]&0x01 == 0 && v.S
}
v.HeaderSize = idx
return nil
}
func (v *VP8) Marshal() ([]byte, error) {
buf := make([]byte, v.HeaderSize)
n, err := v.MarshalTo(buf)
if err != nil {
return nil, err
}
return buf[:n], err
}
func (v *VP8) MarshalTo(buf []byte) (int, error) {
if len(buf) < v.HeaderSize {
return 0, errShortPacket
}
idx := 0
buf[idx] = v.FirstByte
if v.I || v.L || v.T || v.K {
buf[idx] |= 0x80 // X bit
idx++
xpos := idx
xval := byte(0)
idx++
if v.I {
xval |= (1 << 7)
if v.M {
buf[idx] = 0x80 | byte((v.PictureID>>8)&0x7f)
buf[idx+1] = byte(v.PictureID & 0xff)
idx += 2
} else {
buf[idx] = byte(v.PictureID)
idx++
}
}
if v.L {
xval |= (1 << 6)
buf[idx] = v.TL0PICIDX
idx++
}
if v.T || v.K {
buf[idx] = 0
if v.T {
xval |= (1 << 5)
buf[idx] = v.TID << 6
if v.Y {
buf[idx] |= (1 << 5)
}
}
if v.K {
xval |= (1 << 4)
buf[idx] |= v.KEYIDX & 0x1f
}
idx++
}
buf[xpos] = xval
} else {
buf[idx] &^= 0x80 // X bit
idx++
}
return idx, nil
}
// -------------------------------------
func VPxPictureIdSizeDiff(mBit1 bool, mBit2 bool) int {
if mBit1 == mBit2 {
return 0
}
if mBit1 {
return 1
}
return -1
}
// -------------------------------------
// IsH264KeyFrame detects if h264 payload is a keyframe
// this code was taken from https://github.com/jech/galene/blob/codecs/rtpconn/rtpreader.go#L45
// all credits belongs to Juliusz Chroboczek @jech and the awesome Galene SFU
func IsH264KeyFrame(payload []byte) bool {
if len(payload) < 1 {
return false
}
nalu := payload[0] & 0x1F
if nalu == 0 {
// reserved
return false
} else if nalu <= 23 {
// simple NALU
return nalu == 7
} else if nalu == 24 || nalu == 25 || nalu == 26 || nalu == 27 {
// STAP-A, STAP-B, MTAP16 or MTAP24
i := 1
if nalu == 25 || nalu == 26 || nalu == 27 {
// skip DON
i += 2
}
for i < len(payload) {
if i+2 > len(payload) {
return false
}
length := uint16(payload[i])<<8 |
uint16(payload[i+1])
i += 2
if i+int(length) > len(payload) {
return false
}
offset := 0
if nalu == 26 {
offset = 3
} else if nalu == 27 {
offset = 4
}
if offset >= int(length) {
return false
}
n := payload[i+offset] & 0x1F
if n == 7 {
return true
} else if n >= 24 {
// is this legal?
logger.Debugw("Non-simple NALU within a STAP")
}
i += int(length)
}
if i == len(payload) {
return false
}
return false
} else if nalu == 28 || nalu == 29 {
// FU-A or FU-B
if len(payload) < 2 {
return false
}
if (payload[1] & 0x80) == 0 {
// not a starting fragment
return false
}
return payload[1]&0x1F == 7
}
return false
}
// -------------------------------------
// IsVP9KeyFrame detects if vp9 payload is a keyframe
// taken from https://github.com/jech/galene/blob/master/codecs/codecs.go
// all credits belongs to Juliusz Chroboczek @jech and the awesome Galene SFU
func IsVP9KeyFrame(payload []byte) bool {
var vp9 codecs.VP9Packet
_, err := vp9.Unmarshal(payload)
if err != nil || len(vp9.Payload) < 1 {
return false
}
if !vp9.B {
return false
}
if (vp9.Payload[0] & 0xc0) != 0x80 {
return false
}
profile := (vp9.Payload[0] >> 4) & 0x3
if profile != 3 {
return (vp9.Payload[0] & 0xC) == 0
}
return (vp9.Payload[0] & 0x6) == 0
}
// -------------------------------------
// IsAV1KeyFrame detects if av1 payload is a keyframe
// taken from https://github.com/jech/galene/blob/master/codecs/codecs.go
// all credits belongs to Juliusz Chroboczek @jech and the awesome Galene SFU
func IsAV1KeyFrame(payload []byte) bool {
if len(payload) < 2 {
return false
}
// Z=0, N=1
if (payload[0] & 0x88) != 0x08 {
return false
}
w := (payload[0] & 0x30) >> 4
getObu := func(data []byte, last bool) ([]byte, int, bool) {
if last {
return data, len(data), false
}
offset := 0
length := 0
for {
if len(data) <= offset {
return nil, offset, offset > 0
}
l := data[offset]
length |= int(l&0x7f) << (offset * 7)
offset++
if (l & 0x80) == 0 {
break
}
}
if len(data) < offset+length {
return data[offset:], len(data), true
}
return data[offset : offset+length],
offset + length, false
}
offset := 1
i := 0
for {
obu, length, truncated :=
getObu(payload[offset:], int(w) == i+1)
if len(obu) < 1 {
return false
}
tpe := (obu[0] & 0x38) >> 3
switch i {
case 0:
// OBU_SEQUENCE_HEADER
if tpe != 1 {
return false
}
default:
// OBU_FRAME_HEADER or OBU_FRAME
if tpe == 3 || tpe == 6 {
if len(obu) < 2 {
return false
}
// show_existing_frame == 0
if (obu[1] & 0x80) != 0 {
return false
}
// frame_type == KEY_FRAME
return (obu[1] & 0x60) == 0
}
}
if truncated || i >= int(w) {
// the first frame header is in a second
// packet, give up.
return false
}
offset += length
i++
}
}
func IsH265KeyFrame(payload []byte) (kf bool) {
if len(payload) < 2 {
return false
}
naluType := (payload[0] & 0x7E) >> 1
switch {
case naluType == 33 || naluType == 34:
return true
case naluType == 48: // AP
idx := 2
for idx < len(payload)-2 {
// TODO: check the DONL field (controled by sprop-max-don-diff)
size := binary.BigEndian.Uint16(payload[idx:])
idx += 2
if idx >= len(payload) {
return false
}
naluType = (payload[idx] & 0x7E) >> 1
if naluType == 33 || naluType == 34 {
return true
}
idx += int(size)
}
return false
case naluType == 49: // FU
if len(payload) < 3 {
return false
}
naluType = (payload[2] & 0x7E) >> 1
return naluType == 33 || naluType == 34
default:
return false
}
}
// -------------------------------------
pkg/sfu/buffer/helpers_test.go
-110
View File
@@ -1,110 +0,0 @@
// Copyright 2023 LiveKit, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package buffer
import (
"testing"
"github.com/stretchr/testify/require"
)
func TestVP8Helper_Unmarshal(t *testing.T) {
type args struct {
payload []byte
}
tests := []struct {
name string
args args
wantErr bool
checkTemporal bool
temporalSupport bool
checkKeyFrame bool
keyFrame bool
checkPictureID bool
pictureID uint16
checkTlzIdx bool
tlzIdx uint8
checkTempID bool
temporalID uint8
}{
{
name: "Empty or nil payload must return error",
args: args{payload: []byte{}},
wantErr: true,
},
{
name: "Temporal must be supported by setting T bit to 1",
args: args{payload: []byte{0xff, 0x20, 0x1, 0x2, 0x3, 0x4}},
checkTemporal: true,
temporalSupport: true,
},
{
name: "Picture must be ID 7 bits by setting M bit to 0 and present by I bit set to 1",
args: args{payload: []byte{0xff, 0xff, 0x11, 0x2, 0x3, 0x4}},
checkPictureID: true,
pictureID: 17,
},
{
name: "Picture ID must be 15 bits by setting M bit to 1 and present by I bit set to 1",
args: args{payload: []byte{0xff, 0xff, 0x92, 0x67, 0x3, 0x4, 0x5}},
checkPictureID: true,
pictureID: 4711,
},
{
name: "Temporal level zero index must be present if L set to 1",
args: args{payload: []byte{0xff, 0xff, 0xff, 0xfd, 0xb4, 0x4, 0x5}},
checkTlzIdx: true,
tlzIdx: 180,
},
{
name: "Temporal index must be present and used if T bit set to 1",
args: args{payload: []byte{0xff, 0xff, 0xff, 0xfd, 0xb4, 0x9f, 0x5, 0x6}},
checkTempID: true,
temporalID: 2,
},
{
name: "Check if packet is a keyframe by looking at P bit set to 0",
args: args{payload: []byte{0xff, 0xff, 0xff, 0xfd, 0xb4, 0x9f, 0x94, 0x1}},
checkKeyFrame: true,
keyFrame: true,
},
}
for _, tt := range tests {
tt := tt
t.Run(tt.name, func(t *testing.T) {
p := &VP8{}
if err := p.Unmarshal(tt.args.payload); (err != nil) != tt.wantErr {
t.Errorf("Unmarshal() error = %v, wantErr %v", err, tt.wantErr)
}
if tt.checkTemporal {
require.Equal(t, tt.temporalSupport, p.T)
}
if tt.checkKeyFrame {
require.Equal(t, tt.keyFrame, p.IsKeyFrame)
}
if tt.checkPictureID {
require.Equal(t, tt.pictureID, p.PictureID)
}
if tt.checkTlzIdx {
require.Equal(t, tt.tlzIdx, p.TL0PICIDX)
}
if tt.checkTempID {
require.Equal(t, tt.temporalID, p.TID)
}
})
}
}
// ------------------------------------------
pkg/sfu/buffer/rtcpreader.go
+1
View File
@@ -31,6 +31,7 @@ func NewRTCPReader(ssrc uint32) *RTCPReader {
return &RTCPReader{ssrc: ssrc}
}
//go:noinline
func (r *RTCPReader) Write(p []byte) (n int, err error) {
if r.closed.Load() {
err = io.EOF
pkg/sfu/buffer/videolayerutils.go
+234 -40
View File
@@ -15,24 +15,43 @@
package buffer
import (
"slices"
"github.com/livekit/protocol/codecs/mime"
"github.com/livekit/protocol/livekit"
"github.com/livekit/protocol/logger"
)
const (
QuarterResolution = "q"
HalfResolution = "h"
FullResolution = "f"
quarterResolutionQ = "q"
halfResolutionH = "h"
fullResolutionF = "f"
quarterResolution2 = "2"
halfResolution1 = "1"
fullResolution0 = "0"
)
// SIMULCAST-CODEC-TODO: these need to be codec mime aware if and when each codec suppports different layers
func LayerPresenceFromTrackInfo(trackInfo *livekit.TrackInfo) *[livekit.VideoQuality_HIGH + 1]bool {
if trackInfo == nil || len(trackInfo.Layers) == 0 {
type VideoLayersRid [DefaultMaxLayerSpatial + 1]string
var (
videoLayersRidQHF = VideoLayersRid{quarterResolutionQ, halfResolutionH, fullResolutionF}
videoLayersRid210 = VideoLayersRid{quarterResolution2, halfResolution1, fullResolution0}
DefaultVideoLayersRid = videoLayersRidQHF
)
func LayerPresenceFromTrackInfo(mimeType mime.MimeType, trackInfo *livekit.TrackInfo) *[livekit.VideoQuality_HIGH + 1]bool {
if trackInfo == nil {
return nil
}
layers := GetVideoLayersForMimeType(mimeType, trackInfo)
if len(layers) == 0 {
return nil
}
var layerPresence [livekit.VideoQuality_HIGH + 1]bool
for _, layer := range trackInfo.Layers {
for _, layer := range layers {
// WARNING: comparing protobuf enum
if layer.Quality <= livekit.VideoQuality_HIGH {
layerPresence[layer.Quality] = true
@@ -44,15 +63,15 @@ func LayerPresenceFromTrackInfo(trackInfo *livekit.TrackInfo) *[livekit.VideoQua
return &layerPresence
}
func RidToSpatialLayer(rid string, trackInfo *livekit.TrackInfo) int32 {
lp := LayerPresenceFromTrackInfo(trackInfo)
func RidToSpatialLayer(mimeType mime.MimeType, rid string, trackInfo *livekit.TrackInfo, ridSpace VideoLayersRid) int32 {
lp := LayerPresenceFromTrackInfo(mimeType, trackInfo)
if lp == nil {
switch rid {
case QuarterResolution:
case quarterResolutionQ:
return 0
case HalfResolution:
case halfResolutionH:
return 1
case FullResolution:
case fullResolutionF:
return 2
default:
return 0
@@ -60,7 +79,7 @@ func RidToSpatialLayer(rid string, trackInfo *livekit.TrackInfo) int32 {
}
switch rid {
case QuarterResolution:
case ridSpace[0]:
switch {
case lp[livekit.VideoQuality_LOW] && lp[livekit.VideoQuality_MEDIUM] && lp[livekit.VideoQuality_HIGH]:
fallthrough
@@ -76,7 +95,7 @@ func RidToSpatialLayer(rid string, trackInfo *livekit.TrackInfo) int32 {
return 0
}
case HalfResolution:
case ridSpace[1]:
switch {
case lp[livekit.VideoQuality_LOW] && lp[livekit.VideoQuality_MEDIUM] && lp[livekit.VideoQuality_HIGH]:
fallthrough
@@ -92,19 +111,19 @@ func RidToSpatialLayer(rid string, trackInfo *livekit.TrackInfo) int32 {
return 0
}
case FullResolution:
case ridSpace[2]:
switch {
case lp[livekit.VideoQuality_LOW] && lp[livekit.VideoQuality_MEDIUM] && lp[livekit.VideoQuality_HIGH]:
return 2
case lp[livekit.VideoQuality_LOW] && lp[livekit.VideoQuality_MEDIUM]:
logger.Warnw("unexpected rid f with only two qualities, low and medium", nil, "trackID", trackInfo.Sid, "trackInfo", logger.Proto(trackInfo))
logger.Warnw("unexpected rid with only two qualities, low and medium", nil, "trackID", trackInfo.Sid, "trackInfo", logger.Proto(trackInfo), "rid", ridSpace[2])
return 1
case lp[livekit.VideoQuality_LOW] && lp[livekit.VideoQuality_HIGH]:
logger.Warnw("unexpected rid f with only two qualities, low and high", nil, "trackID", trackInfo.Sid, "trackInfo", logger.Proto(trackInfo))
logger.Warnw("unexpected rid with only two qualities, low and high", nil, "trackID", trackInfo.Sid, "trackInfo", logger.Proto(trackInfo), "rid", ridSpace[2])
return 1
case lp[livekit.VideoQuality_MEDIUM] && lp[livekit.VideoQuality_HIGH]:
logger.Warnw("unexpected rid f with only two qualities, medium and high", nil, "trackID", trackInfo.Sid, "trackInfo", logger.Proto(trackInfo))
logger.Warnw("unexpected rid with only two qualities, medium and high", nil, "trackID", trackInfo.Sid, "trackInfo", logger.Proto(trackInfo), "rid", ridSpace[2])
return 1
default:
@@ -118,18 +137,18 @@ func RidToSpatialLayer(rid string, trackInfo *livekit.TrackInfo) int32 {
}
}
func SpatialLayerToRid(layer int32, trackInfo *livekit.TrackInfo) string {
lp := LayerPresenceFromTrackInfo(trackInfo)
func SpatialLayerToRid(mimeType mime.MimeType, layer int32, trackInfo *livekit.TrackInfo, ridSpace VideoLayersRid) string {
lp := LayerPresenceFromTrackInfo(mimeType, trackInfo)
if lp == nil {
switch layer {
case 0:
return QuarterResolution
return quarterResolutionQ
case 1:
return HalfResolution
return halfResolutionH
case 2:
return FullResolution
return fullResolutionF
default:
return QuarterResolution
return quarterResolutionQ
}
}
@@ -143,10 +162,10 @@ func SpatialLayerToRid(layer int32, trackInfo *livekit.TrackInfo) string {
case lp[livekit.VideoQuality_LOW] && lp[livekit.VideoQuality_HIGH]:
fallthrough
case lp[livekit.VideoQuality_MEDIUM] && lp[livekit.VideoQuality_HIGH]:
return QuarterResolution
return ridSpace[0]
default:
return QuarterResolution
return ridSpace[0]
}
case 1:
@@ -158,42 +177,42 @@ func SpatialLayerToRid(layer int32, trackInfo *livekit.TrackInfo) string {
case lp[livekit.VideoQuality_LOW] && lp[livekit.VideoQuality_HIGH]:
fallthrough
case lp[livekit.VideoQuality_MEDIUM] && lp[livekit.VideoQuality_HIGH]:
return HalfResolution
return ridSpace[1]
default:
return QuarterResolution
return ridSpace[0]
}
case 2:
switch {
case lp[livekit.VideoQuality_LOW] && lp[livekit.VideoQuality_MEDIUM] && lp[livekit.VideoQuality_HIGH]:
return FullResolution
return ridSpace[2]
case lp[livekit.VideoQuality_LOW] && lp[livekit.VideoQuality_MEDIUM]:
logger.Warnw("unexpected layer 2 with only two qualities, low and medium", nil, "trackID", trackInfo.Sid, "trackInfo", logger.Proto(trackInfo))
return HalfResolution
return ridSpace[1]
case lp[livekit.VideoQuality_LOW] && lp[livekit.VideoQuality_HIGH]:
logger.Warnw("unexpected layer 2 with only two qualities, low and high", nil, "trackID", trackInfo.Sid, "trackInfo", logger.Proto(trackInfo))
return HalfResolution
return ridSpace[1]
case lp[livekit.VideoQuality_MEDIUM] && lp[livekit.VideoQuality_HIGH]:
logger.Warnw("unexpected layer 2 with only two qualities, medium and high", nil, "trackID", trackInfo.Sid, "trackInfo", logger.Proto(trackInfo))
return HalfResolution
return ridSpace[1]
default:
return QuarterResolution
return ridSpace[0]
}
default:
return QuarterResolution
return ridSpace[0]
}
}
func VideoQualityToRid(quality livekit.VideoQuality, trackInfo *livekit.TrackInfo) string {
return SpatialLayerToRid(VideoQualityToSpatialLayer(quality, trackInfo), trackInfo)
func VideoQualityToRid(mimeType mime.MimeType, quality livekit.VideoQuality, trackInfo *livekit.TrackInfo, ridSpace VideoLayersRid) string {
return SpatialLayerToRid(mimeType, VideoQualityToSpatialLayer(mimeType, quality, trackInfo), trackInfo, ridSpace)
}
func SpatialLayerToVideoQuality(layer int32, trackInfo *livekit.TrackInfo) livekit.VideoQuality {
lp := LayerPresenceFromTrackInfo(trackInfo)
func SpatialLayerToVideoQuality(mimeType mime.MimeType, layer int32, trackInfo *livekit.TrackInfo) livekit.VideoQuality {
lp := LayerPresenceFromTrackInfo(mimeType, trackInfo)
if lp == nil {
switch layer {
case 0:
@@ -259,8 +278,8 @@ func SpatialLayerToVideoQuality(layer int32, trackInfo *livekit.TrackInfo) livek
return livekit.VideoQuality_OFF
}
func VideoQualityToSpatialLayer(quality livekit.VideoQuality, trackInfo *livekit.TrackInfo) int32 {
lp := LayerPresenceFromTrackInfo(trackInfo)
func VideoQualityToSpatialLayer(mimeType mime.MimeType, quality livekit.VideoQuality, trackInfo *livekit.TrackInfo) int32 {
lp := LayerPresenceFromTrackInfo(mimeType, trackInfo)
if lp == nil {
switch quality {
case livekit.VideoQuality_LOW:
@@ -324,3 +343,178 @@ func VideoQualityToSpatialLayer(quality livekit.VideoQuality, trackInfo *livekit
return InvalidLayerSpatial
}
func GetVideoLayerModeForMimeType(mimeType mime.MimeType, ti *livekit.TrackInfo) livekit.VideoLayer_Mode {
if ti != nil {
for _, codec := range ti.Codecs {
if mime.NormalizeMimeType(codec.MimeType) == mimeType {
return codec.VideoLayerMode
}
}
}
return livekit.VideoLayer_MODE_UNUSED
}
func GetVideoLayersForMimeType(mimeType mime.MimeType, ti *livekit.TrackInfo) []*livekit.VideoLayer {
var layers []*livekit.VideoLayer
if ti != nil {
for _, codec := range ti.Codecs {
if mime.NormalizeMimeType(codec.MimeType) == mimeType {
layers = codec.Layers
break
}
}
if len(layers) == 0 {
layers = ti.Layers
}
}
return layers
}
func GetSpatialLayerForRid(mimeType mime.MimeType, rid string, ti *livekit.TrackInfo) int32 {
if ti == nil {
return InvalidLayerSpatial
}
if rid == "" {
// single layer without RID
return 0
}
layers := GetVideoLayersForMimeType(mimeType, ti)
for _, layer := range layers {
if layer.Rid == rid {
return layer.SpatialLayer
}
}
if len(layers) != 0 {
// RID present in codec, but may not be specified via signalling
// (happens with older browsers setting a rid for SVC codecs)
hasRid := false
for _, layer := range layers {
if layer.Rid != "" {
hasRid = true
break
}
}
if !hasRid {
return 0
}
}
// SIMULCAST-CODEC-TODO - ideally should return invalid, but there are
// VP9 publishers using rid = f, if there are only two layers
// in TrackInfo, that will be q;h and f will become invalid.
//
// Actually, there should be no rids for VP9 in SDP and hence
// the above check should take effect. However, as simulcast
// codec/back up codec does not update rids from SDP,
// the default rids are used when vp9 (primary codec)
// is published. Due to that the above check gets bypassed.
//
// The full proper sequence would be
// 1. For primary codec using SVC, there will be no rids.
// The above check should take effect and it should
// return 0 even if some publisher uses a rid like `f`.
// 2. When secondary codec is published, rids for the codec
// corresponding to the back up codec mime type should
// be updated in `TrackInfo`. This is a bit tricky
// for a couple of cases
// a. Browsers like Firefox use a different CID everytime.
// So, it cannot be matched between `AddTrack` and SDP.
// One option is to look for a published track with
// back up codec and apply it there. But, that becomes
// a challenge if there are multiple published tracks
// with pending back up codec.
// b. The back up codec publish SDP will have the full
// codec list. It should be okay to assume that the
// codec that will be published is the back up codec,
// but just something to be aware of.
// 3. Use of this function with proper mime so that proper
// codec section can be looked up in `TrackInfo`.
// return InvalidLayerSpatial
logger.Infow(
"invalid layer for rid, returning default",
"trackID", ti.Sid,
"rid", rid,
"mimeType", mimeType,
"trackInfo", logger.Proto(ti),
)
return 0
}
func GetSpatialLayerForVideoQuality(mimeType mime.MimeType, quality livekit.VideoQuality, ti *livekit.TrackInfo) int32 {
if ti == nil || quality == livekit.VideoQuality_OFF {
return InvalidLayerSpatial
}
layers := GetVideoLayersForMimeType(mimeType, ti)
for _, layer := range layers {
if layer.Quality == quality {
return layer.SpatialLayer
}
}
if len(layers) == 0 {
// single layer
return 0
}
// requested quality is higher than available layers, return the highest available layer
return VideoQualityToSpatialLayer(mimeType, quality, ti)
}
func GetVideoQualityForSpatialLayer(mimeType mime.MimeType, spatialLayer int32, ti *livekit.TrackInfo) livekit.VideoQuality {
if spatialLayer == InvalidLayerSpatial || ti == nil {
return livekit.VideoQuality_OFF
}
layers := GetVideoLayersForMimeType(mimeType, ti)
for _, layer := range layers {
if layer.SpatialLayer == spatialLayer {
return layer.Quality
}
}
return livekit.VideoQuality_OFF
}
func isVideoLayersRidKnown(rids VideoLayersRid, knownRids VideoLayersRid) bool {
for _, rid := range rids {
if rid == "" {
continue
}
if !slices.Contains(knownRids[:], rid) {
return false
}
}
return true
}
func NormalizeVideoLayersRid(rids VideoLayersRid) VideoLayersRid {
out := rids
normalize := func(knownRids VideoLayersRid) {
idx := 0
for _, known := range knownRids {
if slices.Contains(rids[:], known) {
out[idx] = known
idx++
}
}
}
if isVideoLayersRidKnown(rids, videoLayersRidQHF) {
normalize(videoLayersRidQHF)
}
if isVideoLayersRidKnown(rids, videoLayersRid210) {
normalize(videoLayersRid210)
}
return out
}
pkg/sfu/buffer/videolayerutils_test.go
+575 -125
View File
@@ -19,6 +19,7 @@ import (
"github.com/stretchr/testify/require"
"github.com/livekit/protocol/codecs/mime"
"github.com/livekit/protocol/livekit"
)
@@ -30,129 +31,174 @@ func TestRidConversion(t *testing.T) {
tests := []struct {
name string
trackInfo *livekit.TrackInfo
mimeType mime.MimeType
ridToLayer map[string]RidAndLayer
}{
{
"no track info",
nil,
mime.MimeTypeVP8,
map[string]RidAndLayer{
"": {rid: QuarterResolution, layer: 0},
QuarterResolution: {rid: QuarterResolution, layer: 0},
HalfResolution: {rid: HalfResolution, layer: 1},
FullResolution: {rid: FullResolution, layer: 2},
"": {rid: quarterResolutionQ, layer: 0},
quarterResolutionQ: {rid: quarterResolutionQ, layer: 0},
halfResolutionH: {rid: halfResolutionH, layer: 1},
fullResolutionF: {rid: fullResolutionF, layer: 2},
},
},
{
"no layers",
&livekit.TrackInfo{},
mime.MimeTypeVP8,
map[string]RidAndLayer{
"": {rid: QuarterResolution, layer: 0},
QuarterResolution: {rid: QuarterResolution, layer: 0},
HalfResolution: {rid: HalfResolution, layer: 1},
FullResolution: {rid: FullResolution, layer: 2},
"": {rid: quarterResolutionQ, layer: 0},
quarterResolutionQ: {rid: quarterResolutionQ, layer: 0},
halfResolutionH: {rid: halfResolutionH, layer: 1},
fullResolutionF: {rid: fullResolutionF, layer: 2},
},
},
{
"single layer, low",
&livekit.TrackInfo{
Layers: []*livekit.VideoLayer{
{Quality: livekit.VideoQuality_LOW},
Codecs: []*livekit.SimulcastCodecInfo{
{
MimeType: mime.MimeTypeVP8.String(),
Layers: []*livekit.VideoLayer{
{Quality: livekit.VideoQuality_LOW},
},
},
},
},
mime.MimeTypeVP8,
map[string]RidAndLayer{
"": {rid: QuarterResolution, layer: 0},
QuarterResolution: {rid: QuarterResolution, layer: 0},
HalfResolution: {rid: QuarterResolution, layer: 0},
FullResolution: {rid: QuarterResolution, layer: 0},
"": {rid: quarterResolutionQ, layer: 0},
quarterResolutionQ: {rid: quarterResolutionQ, layer: 0},
halfResolutionH: {rid: quarterResolutionQ, layer: 0},
fullResolutionF: {rid: quarterResolutionQ, layer: 0},
},
},
{
"single layer, medium",
&livekit.TrackInfo{
Layers: []*livekit.VideoLayer{
{Quality: livekit.VideoQuality_MEDIUM},
Codecs: []*livekit.SimulcastCodecInfo{
{
MimeType: mime.MimeTypeVP8.String(),
Layers: []*livekit.VideoLayer{
{Quality: livekit.VideoQuality_MEDIUM},
},
},
},
},
mime.MimeTypeVP8,
map[string]RidAndLayer{
"": {rid: QuarterResolution, layer: 0},
QuarterResolution: {rid: QuarterResolution, layer: 0},
HalfResolution: {rid: QuarterResolution, layer: 0},
FullResolution: {rid: QuarterResolution, layer: 0},
"": {rid: quarterResolutionQ, layer: 0},
quarterResolutionQ: {rid: quarterResolutionQ, layer: 0},
halfResolutionH: {rid: quarterResolutionQ, layer: 0},
fullResolutionF: {rid: quarterResolutionQ, layer: 0},
},
},
{
"single layer, high",
&livekit.TrackInfo{
Layers: []*livekit.VideoLayer{
{Quality: livekit.VideoQuality_HIGH},
Codecs: []*livekit.SimulcastCodecInfo{
{
MimeType: mime.MimeTypeVP8.String(),
Layers: []*livekit.VideoLayer{
{Quality: livekit.VideoQuality_MEDIUM},
},
},
},
},
mime.MimeTypeVP8,
map[string]RidAndLayer{
"": {rid: QuarterResolution, layer: 0},
QuarterResolution: {rid: QuarterResolution, layer: 0},
HalfResolution: {rid: QuarterResolution, layer: 0},
FullResolution: {rid: QuarterResolution, layer: 0},
"": {rid: quarterResolutionQ, layer: 0},
quarterResolutionQ: {rid: quarterResolutionQ, layer: 0},
halfResolutionH: {rid: quarterResolutionQ, layer: 0},
fullResolutionF: {rid: quarterResolutionQ, layer: 0},
},
},
{
"two layers, low and medium",
&livekit.TrackInfo{
Layers: []*livekit.VideoLayer{
{Quality: livekit.VideoQuality_LOW},
{Quality: livekit.VideoQuality_MEDIUM},
Codecs: []*livekit.SimulcastCodecInfo{
{
MimeType: mime.MimeTypeVP8.String(),
Layers: []*livekit.VideoLayer{
{Quality: livekit.VideoQuality_LOW},
{Quality: livekit.VideoQuality_MEDIUM},
},
},
},
},
mime.MimeTypeVP8,
map[string]RidAndLayer{
"": {rid: QuarterResolution, layer: 0},
QuarterResolution: {rid: QuarterResolution, layer: 0},
HalfResolution: {rid: HalfResolution, layer: 1},
FullResolution: {rid: HalfResolution, layer: 1},
"": {rid: quarterResolutionQ, layer: 0},
quarterResolutionQ: {rid: quarterResolutionQ, layer: 0},
halfResolutionH: {rid: halfResolutionH, layer: 1},
fullResolutionF: {rid: halfResolutionH, layer: 1},
},
},
{
"two layers, low and high",
&livekit.TrackInfo{
Layers: []*livekit.VideoLayer{
{Quality: livekit.VideoQuality_LOW},
{Quality: livekit.VideoQuality_HIGH},
Codecs: []*livekit.SimulcastCodecInfo{
{
MimeType: mime.MimeTypeVP8.String(),
Layers: []*livekit.VideoLayer{
{Quality: livekit.VideoQuality_LOW},
{Quality: livekit.VideoQuality_HIGH},
},
},
},
},
mime.MimeTypeVP8,
map[string]RidAndLayer{
"": {rid: QuarterResolution, layer: 0},
QuarterResolution: {rid: QuarterResolution, layer: 0},
HalfResolution: {rid: HalfResolution, layer: 1},
FullResolution: {rid: HalfResolution, layer: 1},
"": {rid: quarterResolutionQ, layer: 0},
quarterResolutionQ: {rid: quarterResolutionQ, layer: 0},
halfResolutionH: {rid: halfResolutionH, layer: 1},
fullResolutionF: {rid: halfResolutionH, layer: 1},
},
},
{
"two layers, medium and high",
&livekit.TrackInfo{
Layers: []*livekit.VideoLayer{
{Quality: livekit.VideoQuality_MEDIUM},
{Quality: livekit.VideoQuality_HIGH},
Codecs: []*livekit.SimulcastCodecInfo{
{
MimeType: mime.MimeTypeVP8.String(),
Layers: []*livekit.VideoLayer{
{Quality: livekit.VideoQuality_MEDIUM},
{Quality: livekit.VideoQuality_HIGH},
},
},
},
},
mime.MimeTypeVP8,
map[string]RidAndLayer{
"": {rid: QuarterResolution, layer: 0},
QuarterResolution: {rid: QuarterResolution, layer: 0},
HalfResolution: {rid: HalfResolution, layer: 1},
FullResolution: {rid: HalfResolution, layer: 1},
"": {rid: quarterResolutionQ, layer: 0},
quarterResolutionQ: {rid: quarterResolutionQ, layer: 0},
halfResolutionH: {rid: halfResolutionH, layer: 1},
fullResolutionF: {rid: halfResolutionH, layer: 1},
},
},
{
"three layers",
&livekit.TrackInfo{
Layers: []*livekit.VideoLayer{
{Quality: livekit.VideoQuality_LOW},
{Quality: livekit.VideoQuality_MEDIUM},
{Quality: livekit.VideoQuality_HIGH},
Codecs: []*livekit.SimulcastCodecInfo{
{
MimeType: mime.MimeTypeVP8.String(),
Layers: []*livekit.VideoLayer{
{Quality: livekit.VideoQuality_LOW},
{Quality: livekit.VideoQuality_MEDIUM},
{Quality: livekit.VideoQuality_HIGH},
},
},
},
},
mime.MimeTypeVP8,
map[string]RidAndLayer{
"": {rid: QuarterResolution, layer: 0},
QuarterResolution: {rid: QuarterResolution, layer: 0},
HalfResolution: {rid: HalfResolution, layer: 1},
FullResolution: {rid: FullResolution, layer: 2},
"": {rid: quarterResolutionQ, layer: 0},
quarterResolutionQ: {rid: quarterResolutionQ, layer: 0},
halfResolutionH: {rid: halfResolutionH, layer: 1},
fullResolutionF: {rid: fullResolutionF, layer: 2},
},
},
}
@@ -160,10 +206,10 @@ func TestRidConversion(t *testing.T) {
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
for testRid, expectedResult := range test.ridToLayer {
actualLayer := RidToSpatialLayer(testRid, test.trackInfo)
actualLayer := RidToSpatialLayer(test.mimeType, testRid, test.trackInfo, DefaultVideoLayersRid)
require.Equal(t, expectedResult.layer, actualLayer)
actualRid := SpatialLayerToRid(actualLayer, test.trackInfo)
actualRid := SpatialLayerToRid(test.mimeType, actualLayer, test.trackInfo, DefaultVideoLayersRid)
require.Equal(t, expectedResult.rid, actualRid)
}
})
@@ -178,11 +224,13 @@ func TestQualityConversion(t *testing.T) {
tests := []struct {
name string
trackInfo *livekit.TrackInfo
mimeType mime.MimeType
qualityToLayer map[livekit.VideoQuality]QualityAndLayer
}{
{
"no track info",
nil,
mime.MimeTypeVP8,
map[livekit.VideoQuality]QualityAndLayer{
livekit.VideoQuality_LOW: {quality: livekit.VideoQuality_LOW, layer: 0},
livekit.VideoQuality_MEDIUM: {quality: livekit.VideoQuality_MEDIUM, layer: 1},
@@ -192,6 +240,7 @@ func TestQualityConversion(t *testing.T) {
{
"no layers",
&livekit.TrackInfo{},
mime.MimeTypeVP8,
map[livekit.VideoQuality]QualityAndLayer{
livekit.VideoQuality_LOW: {quality: livekit.VideoQuality_LOW, layer: 0},
livekit.VideoQuality_MEDIUM: {quality: livekit.VideoQuality_MEDIUM, layer: 1},
@@ -201,10 +250,16 @@ func TestQualityConversion(t *testing.T) {
{
"single layer, low",
&livekit.TrackInfo{
Layers: []*livekit.VideoLayer{
{Quality: livekit.VideoQuality_LOW},
Codecs: []*livekit.SimulcastCodecInfo{
{
MimeType: mime.MimeTypeVP8.String(),
Layers: []*livekit.VideoLayer{
{Quality: livekit.VideoQuality_LOW},
},
},
},
},
mime.MimeTypeVP8,
map[livekit.VideoQuality]QualityAndLayer{
livekit.VideoQuality_LOW: {quality: livekit.VideoQuality_LOW, layer: 0},
livekit.VideoQuality_MEDIUM: {quality: livekit.VideoQuality_LOW, layer: 0},
@@ -214,10 +269,16 @@ func TestQualityConversion(t *testing.T) {
{
"single layer, medium",
&livekit.TrackInfo{
Layers: []*livekit.VideoLayer{
{Quality: livekit.VideoQuality_MEDIUM},
Codecs: []*livekit.SimulcastCodecInfo{
{
MimeType: mime.MimeTypeVP8.String(),
Layers: []*livekit.VideoLayer{
{Quality: livekit.VideoQuality_MEDIUM},
},
},
},
},
mime.MimeTypeVP8,
map[livekit.VideoQuality]QualityAndLayer{
livekit.VideoQuality_LOW: {quality: livekit.VideoQuality_MEDIUM, layer: 0},
livekit.VideoQuality_MEDIUM: {quality: livekit.VideoQuality_MEDIUM, layer: 0},
@@ -227,10 +288,16 @@ func TestQualityConversion(t *testing.T) {
{
"single layer, high",
&livekit.TrackInfo{
Layers: []*livekit.VideoLayer{
{Quality: livekit.VideoQuality_HIGH},
Codecs: []*livekit.SimulcastCodecInfo{
{
MimeType: mime.MimeTypeVP8.String(),
Layers: []*livekit.VideoLayer{
{Quality: livekit.VideoQuality_HIGH},
},
},
},
},
mime.MimeTypeVP8,
map[livekit.VideoQuality]QualityAndLayer{
livekit.VideoQuality_LOW: {quality: livekit.VideoQuality_HIGH, layer: 0},
livekit.VideoQuality_MEDIUM: {quality: livekit.VideoQuality_HIGH, layer: 0},
@@ -240,11 +307,17 @@ func TestQualityConversion(t *testing.T) {
{
"two layers, low and medium",
&livekit.TrackInfo{
Layers: []*livekit.VideoLayer{
{Quality: livekit.VideoQuality_LOW},
{Quality: livekit.VideoQuality_MEDIUM},
Codecs: []*livekit.SimulcastCodecInfo{
{
MimeType: mime.MimeTypeVP8.String(),
Layers: []*livekit.VideoLayer{
{Quality: livekit.VideoQuality_LOW},
{Quality: livekit.VideoQuality_MEDIUM},
},
},
},
},
mime.MimeTypeVP8,
map[livekit.VideoQuality]QualityAndLayer{
livekit.VideoQuality_LOW: {quality: livekit.VideoQuality_LOW, layer: 0},
livekit.VideoQuality_MEDIUM: {quality: livekit.VideoQuality_MEDIUM, layer: 1},
@@ -254,11 +327,17 @@ func TestQualityConversion(t *testing.T) {
{
"two layers, low and high",
&livekit.TrackInfo{
Layers: []*livekit.VideoLayer{
{Quality: livekit.VideoQuality_LOW},
{Quality: livekit.VideoQuality_HIGH},
Codecs: []*livekit.SimulcastCodecInfo{
{
MimeType: mime.MimeTypeVP8.String(),
Layers: []*livekit.VideoLayer{
{Quality: livekit.VideoQuality_LOW},
{Quality: livekit.VideoQuality_HIGH},
},
},
},
},
mime.MimeTypeVP8,
map[livekit.VideoQuality]QualityAndLayer{
livekit.VideoQuality_LOW: {quality: livekit.VideoQuality_LOW, layer: 0},
livekit.VideoQuality_MEDIUM: {quality: livekit.VideoQuality_HIGH, layer: 1},
@@ -268,11 +347,17 @@ func TestQualityConversion(t *testing.T) {
{
"two layers, medium and high",
&livekit.TrackInfo{
Layers: []*livekit.VideoLayer{
{Quality: livekit.VideoQuality_MEDIUM},
{Quality: livekit.VideoQuality_HIGH},
Codecs: []*livekit.SimulcastCodecInfo{
{
MimeType: mime.MimeTypeVP8.String(),
Layers: []*livekit.VideoLayer{
{Quality: livekit.VideoQuality_MEDIUM},
{Quality: livekit.VideoQuality_HIGH},
},
},
},
},
mime.MimeTypeVP8,
map[livekit.VideoQuality]QualityAndLayer{
livekit.VideoQuality_LOW: {quality: livekit.VideoQuality_MEDIUM, layer: 0},
livekit.VideoQuality_MEDIUM: {quality: livekit.VideoQuality_MEDIUM, layer: 0},
@@ -282,12 +367,18 @@ func TestQualityConversion(t *testing.T) {
{
"three layers",
&livekit.TrackInfo{
Layers: []*livekit.VideoLayer{
{Quality: livekit.VideoQuality_LOW},
{Quality: livekit.VideoQuality_MEDIUM},
{Quality: livekit.VideoQuality_HIGH},
Codecs: []*livekit.SimulcastCodecInfo{
{
MimeType: mime.MimeTypeVP8.String(),
Layers: []*livekit.VideoLayer{
{Quality: livekit.VideoQuality_LOW},
{Quality: livekit.VideoQuality_MEDIUM},
{Quality: livekit.VideoQuality_HIGH},
},
},
},
},
mime.MimeTypeVP8,
map[livekit.VideoQuality]QualityAndLayer{
livekit.VideoQuality_LOW: {quality: livekit.VideoQuality_LOW, layer: 0},
livekit.VideoQuality_MEDIUM: {quality: livekit.VideoQuality_MEDIUM, layer: 1},
@@ -299,10 +390,10 @@ func TestQualityConversion(t *testing.T) {
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
for testQuality, expectedResult := range test.qualityToLayer {
actualLayer := VideoQualityToSpatialLayer(testQuality, test.trackInfo)
actualLayer := VideoQualityToSpatialLayer(test.mimeType, testQuality, test.trackInfo)
require.Equal(t, expectedResult.layer, actualLayer)
actualQuality := SpatialLayerToVideoQuality(actualLayer, test.trackInfo)
actualQuality := SpatialLayerToVideoQuality(test.mimeType, actualLayer, test.trackInfo)
require.Equal(t, expectedResult.quality, actualQuality)
}
})
@@ -313,120 +404,165 @@ func TestVideoQualityToRidConversion(t *testing.T) {
tests := []struct {
name string
trackInfo *livekit.TrackInfo
mimeTye mime.MimeType
qualityToRid map[livekit.VideoQuality]string
}{
{
"no track info",
nil,
mime.MimeTypeVP8,
map[livekit.VideoQuality]string{
livekit.VideoQuality_LOW: QuarterResolution,
livekit.VideoQuality_MEDIUM: HalfResolution,
livekit.VideoQuality_HIGH: FullResolution,
livekit.VideoQuality_LOW: quarterResolutionQ,
livekit.VideoQuality_MEDIUM: halfResolutionH,
livekit.VideoQuality_HIGH: fullResolutionF,
},
},
{
"no layers",
&livekit.TrackInfo{},
mime.MimeTypeVP8,
map[livekit.VideoQuality]string{
livekit.VideoQuality_LOW: QuarterResolution,
livekit.VideoQuality_MEDIUM: HalfResolution,
livekit.VideoQuality_HIGH: FullResolution,
livekit.VideoQuality_LOW: quarterResolutionQ,
livekit.VideoQuality_MEDIUM: halfResolutionH,
livekit.VideoQuality_HIGH: fullResolutionF,
},
},
{
"single layer, low",
&livekit.TrackInfo{
Layers: []*livekit.VideoLayer{
{Quality: livekit.VideoQuality_LOW},
Codecs: []*livekit.SimulcastCodecInfo{
{
MimeType: mime.MimeTypeVP8.String(),
Layers: []*livekit.VideoLayer{
{Quality: livekit.VideoQuality_LOW},
},
},
},
},
mime.MimeTypeVP8,
map[livekit.VideoQuality]string{
livekit.VideoQuality_LOW: QuarterResolution,
livekit.VideoQuality_MEDIUM: QuarterResolution,
livekit.VideoQuality_HIGH: QuarterResolution,
livekit.VideoQuality_LOW: quarterResolutionQ,
livekit.VideoQuality_MEDIUM: quarterResolutionQ,
livekit.VideoQuality_HIGH: quarterResolutionQ,
},
},
{
"single layer, medium",
&livekit.TrackInfo{
Layers: []*livekit.VideoLayer{
{Quality: livekit.VideoQuality_MEDIUM},
Codecs: []*livekit.SimulcastCodecInfo{
{
MimeType: mime.MimeTypeVP8.String(),
Layers: []*livekit.VideoLayer{
{Quality: livekit.VideoQuality_MEDIUM},
},
},
},
},
mime.MimeTypeVP8,
map[livekit.VideoQuality]string{
livekit.VideoQuality_LOW: QuarterResolution,
livekit.VideoQuality_MEDIUM: QuarterResolution,
livekit.VideoQuality_HIGH: QuarterResolution,
livekit.VideoQuality_LOW: quarterResolutionQ,
livekit.VideoQuality_MEDIUM: quarterResolutionQ,
livekit.VideoQuality_HIGH: quarterResolutionQ,
},
},
{
"single layer, high",
&livekit.TrackInfo{
Layers: []*livekit.VideoLayer{
{Quality: livekit.VideoQuality_HIGH},
Codecs: []*livekit.SimulcastCodecInfo{
{
MimeType: mime.MimeTypeVP8.String(),
Layers: []*livekit.VideoLayer{
{Quality: livekit.VideoQuality_HIGH},
},
},
},
},
mime.MimeTypeVP8,
map[livekit.VideoQuality]string{
livekit.VideoQuality_LOW: QuarterResolution,
livekit.VideoQuality_MEDIUM: QuarterResolution,
livekit.VideoQuality_HIGH: QuarterResolution,
livekit.VideoQuality_LOW: quarterResolutionQ,
livekit.VideoQuality_MEDIUM: quarterResolutionQ,
livekit.VideoQuality_HIGH: quarterResolutionQ,
},
},
{
"two layers, low and medium",
&livekit.TrackInfo{
Layers: []*livekit.VideoLayer{
{Quality: livekit.VideoQuality_LOW},
{Quality: livekit.VideoQuality_MEDIUM},
Codecs: []*livekit.SimulcastCodecInfo{
{
MimeType: mime.MimeTypeVP8.String(),
Layers: []*livekit.VideoLayer{
{Quality: livekit.VideoQuality_LOW},
{Quality: livekit.VideoQuality_MEDIUM},
},
},
},
},
mime.MimeTypeVP8,
map[livekit.VideoQuality]string{
livekit.VideoQuality_LOW: QuarterResolution,
livekit.VideoQuality_MEDIUM: HalfResolution,
livekit.VideoQuality_HIGH: HalfResolution,
livekit.VideoQuality_LOW: quarterResolutionQ,
livekit.VideoQuality_MEDIUM: halfResolutionH,
livekit.VideoQuality_HIGH: halfResolutionH,
},
},
{
"two layers, low and high",
&livekit.TrackInfo{
Layers: []*livekit.VideoLayer{
{Quality: livekit.VideoQuality_LOW},
{Quality: livekit.VideoQuality_HIGH},
Codecs: []*livekit.SimulcastCodecInfo{
{
MimeType: mime.MimeTypeVP8.String(),
Layers: []*livekit.VideoLayer{
{Quality: livekit.VideoQuality_LOW},
{Quality: livekit.VideoQuality_HIGH},
},
},
},
},
mime.MimeTypeVP8,
map[livekit.VideoQuality]string{
livekit.VideoQuality_LOW: QuarterResolution,
livekit.VideoQuality_MEDIUM: HalfResolution,
livekit.VideoQuality_HIGH: HalfResolution,
livekit.VideoQuality_LOW: quarterResolutionQ,
livekit.VideoQuality_MEDIUM: halfResolutionH,
livekit.VideoQuality_HIGH: halfResolutionH,
},
},
{
"two layers, medium and high",
&livekit.TrackInfo{
Layers: []*livekit.VideoLayer{
{Quality: livekit.VideoQuality_MEDIUM},
{Quality: livekit.VideoQuality_HIGH},
Codecs: []*livekit.SimulcastCodecInfo{
{
MimeType: mime.MimeTypeVP8.String(),
Layers: []*livekit.VideoLayer{
{Quality: livekit.VideoQuality_MEDIUM},
{Quality: livekit.VideoQuality_HIGH},
},
},
},
},
mime.MimeTypeVP8,
map[livekit.VideoQuality]string{
livekit.VideoQuality_LOW: QuarterResolution,
livekit.VideoQuality_MEDIUM: QuarterResolution,
livekit.VideoQuality_HIGH: HalfResolution,
livekit.VideoQuality_LOW: quarterResolutionQ,
livekit.VideoQuality_MEDIUM: quarterResolutionQ,
livekit.VideoQuality_HIGH: halfResolutionH,
},
},
{
"three layers",
&livekit.TrackInfo{
Layers: []*livekit.VideoLayer{
{Quality: livekit.VideoQuality_LOW},
{Quality: livekit.VideoQuality_MEDIUM},
{Quality: livekit.VideoQuality_HIGH},
Codecs: []*livekit.SimulcastCodecInfo{
{
MimeType: mime.MimeTypeVP8.String(),
Layers: []*livekit.VideoLayer{
{Quality: livekit.VideoQuality_LOW},
{Quality: livekit.VideoQuality_MEDIUM},
{Quality: livekit.VideoQuality_HIGH},
},
},
},
},
mime.MimeTypeVP8,
map[livekit.VideoQuality]string{
livekit.VideoQuality_LOW: QuarterResolution,
livekit.VideoQuality_MEDIUM: HalfResolution,
livekit.VideoQuality_HIGH: FullResolution,
livekit.VideoQuality_LOW: quarterResolutionQ,
livekit.VideoQuality_MEDIUM: halfResolutionH,
livekit.VideoQuality_HIGH: fullResolutionF,
},
},
}
@@ -434,9 +570,323 @@ func TestVideoQualityToRidConversion(t *testing.T) {
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
for testQuality, expectedRid := range test.qualityToRid {
actualRid := VideoQualityToRid(testQuality, test.trackInfo)
actualRid := VideoQualityToRid(test.mimeTye, testQuality, test.trackInfo, DefaultVideoLayersRid)
require.Equal(t, expectedRid, actualRid)
}
})
}
}
func TestGetSpatialLayerForRid(t *testing.T) {
tests := []struct {
name string
trackInfo *livekit.TrackInfo
mimeType mime.MimeType
ridToSpatialLayer map[string]int32
}{
{
"no track info",
nil,
mime.MimeTypeVP8,
map[string]int32{
quarterResolutionQ: InvalidLayerSpatial,
halfResolutionH: InvalidLayerSpatial,
fullResolutionF: InvalidLayerSpatial,
},
},
{
"no layers",
&livekit.TrackInfo{},
mime.MimeTypeVP8,
map[string]int32{
// SIMULCAST-CODEC-TODO
// quarterResolutionQ: InvalidLayerSpatial,
// halfResolutionH: InvalidLayerSpatial,
// fullResolutionF: InvalidLayerSpatial,
quarterResolutionQ: 0,
halfResolutionH: 0,
fullResolutionF: 0,
},
},
{
"no rid",
&livekit.TrackInfo{},
mime.MimeTypeVP8,
map[string]int32{
"": 0,
},
},
{
"single layer",
&livekit.TrackInfo{
Codecs: []*livekit.SimulcastCodecInfo{
{
MimeType: mime.MimeTypeVP8.String(),
Layers: []*livekit.VideoLayer{
{Quality: livekit.VideoQuality_LOW, SpatialLayer: 0},
},
},
},
},
mime.MimeTypeVP8,
map[string]int32{
quarterResolutionQ: 0,
halfResolutionH: 0,
fullResolutionF: 0,
},
},
{
"layers",
&livekit.TrackInfo{
Codecs: []*livekit.SimulcastCodecInfo{
{
MimeType: mime.MimeTypeVP8.String(),
Layers: []*livekit.VideoLayer{
{Quality: livekit.VideoQuality_LOW, SpatialLayer: 0, Rid: quarterResolutionQ},
{Quality: livekit.VideoQuality_MEDIUM, SpatialLayer: 1, Rid: halfResolutionH},
},
},
},
},
mime.MimeTypeVP8,
map[string]int32{
quarterResolutionQ: 0,
halfResolutionH: 1,
// SIMULCAST-CODEC-TODO
// fullResolutionF: InvalidLayerSpatial,
fullResolutionF: 0,
},
},
{
"layers - no rid",
&livekit.TrackInfo{
Codecs: []*livekit.SimulcastCodecInfo{
{
MimeType: mime.MimeTypeVP8.String(),
Layers: []*livekit.VideoLayer{
{Quality: livekit.VideoQuality_LOW, SpatialLayer: 0},
{Quality: livekit.VideoQuality_MEDIUM, SpatialLayer: 1},
},
},
},
},
mime.MimeTypeVP8,
map[string]int32{
quarterResolutionQ: 0,
halfResolutionH: 0,
fullResolutionF: 0,
},
},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
for testRid, expectedSpatialLayer := range test.ridToSpatialLayer {
actualSpatialLayer := GetSpatialLayerForRid(test.mimeType, testRid, test.trackInfo)
require.Equal(t, expectedSpatialLayer, actualSpatialLayer)
}
})
}
}
func TestGetSpatialLayerForVideoQuality(t *testing.T) {
tests := []struct {
name string
trackInfo *livekit.TrackInfo
mimeType mime.MimeType
videoQualityToSpatialLayer map[livekit.VideoQuality]int32
}{
{
"no track info",
nil,
mime.MimeTypeVP8,
map[livekit.VideoQuality]int32{
livekit.VideoQuality_LOW: InvalidLayerSpatial,
livekit.VideoQuality_MEDIUM: InvalidLayerSpatial,
livekit.VideoQuality_HIGH: InvalidLayerSpatial,
livekit.VideoQuality_OFF: InvalidLayerSpatial,
},
},
{
"no layers",
&livekit.TrackInfo{},
mime.MimeTypeVP8,
map[livekit.VideoQuality]int32{
livekit.VideoQuality_LOW: 0,
livekit.VideoQuality_MEDIUM: 0,
livekit.VideoQuality_HIGH: 0,
livekit.VideoQuality_OFF: InvalidLayerSpatial,
},
},
{
"not all layers",
&livekit.TrackInfo{
Codecs: []*livekit.SimulcastCodecInfo{
{
MimeType: mime.MimeTypeVP8.String(),
Layers: []*livekit.VideoLayer{
{Quality: livekit.VideoQuality_LOW, SpatialLayer: 0, Rid: quarterResolutionQ},
{Quality: livekit.VideoQuality_MEDIUM, SpatialLayer: 1, Rid: halfResolutionH},
},
},
},
},
mime.MimeTypeVP8,
map[livekit.VideoQuality]int32{
livekit.VideoQuality_LOW: 0,
livekit.VideoQuality_MEDIUM: 1,
livekit.VideoQuality_HIGH: 1,
livekit.VideoQuality_OFF: InvalidLayerSpatial,
},
},
{
"all layers",
&livekit.TrackInfo{
Codecs: []*livekit.SimulcastCodecInfo{
{
MimeType: mime.MimeTypeVP8.String(),
Layers: []*livekit.VideoLayer{
{Quality: livekit.VideoQuality_LOW, SpatialLayer: 0, Rid: quarterResolutionQ},
{Quality: livekit.VideoQuality_MEDIUM, SpatialLayer: 1, Rid: halfResolutionH},
{Quality: livekit.VideoQuality_HIGH, SpatialLayer: 2, Rid: fullResolutionF},
},
},
},
},
mime.MimeTypeVP8,
map[livekit.VideoQuality]int32{
livekit.VideoQuality_LOW: 0,
livekit.VideoQuality_MEDIUM: 1,
livekit.VideoQuality_HIGH: 2,
livekit.VideoQuality_OFF: InvalidLayerSpatial,
},
},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
for testVideoQuality, expectedSpatialLayer := range test.videoQualityToSpatialLayer {
actualSpatialLayer := GetSpatialLayerForVideoQuality(test.mimeType, testVideoQuality, test.trackInfo)
require.Equal(t, expectedSpatialLayer, actualSpatialLayer)
}
})
}
}
func TestGetVideoQualityorSpatialLayer(t *testing.T) {
tests := []struct {
name string
trackInfo *livekit.TrackInfo
mimeType mime.MimeType
spatialLayerToVideoQuality map[int32]livekit.VideoQuality
}{
{
"no track info",
nil,
mime.MimeTypeVP8,
map[int32]livekit.VideoQuality{
InvalidLayerSpatial: livekit.VideoQuality_OFF,
0: livekit.VideoQuality_OFF,
1: livekit.VideoQuality_OFF,
2: livekit.VideoQuality_OFF,
},
},
{
"no layers",
&livekit.TrackInfo{},
mime.MimeTypeVP8,
map[int32]livekit.VideoQuality{
InvalidLayerSpatial: livekit.VideoQuality_OFF,
0: livekit.VideoQuality_OFF,
1: livekit.VideoQuality_OFF,
2: livekit.VideoQuality_OFF,
},
},
{
"layers",
&livekit.TrackInfo{
Codecs: []*livekit.SimulcastCodecInfo{
{
MimeType: mime.MimeTypeVP8.String(),
Layers: []*livekit.VideoLayer{
{Quality: livekit.VideoQuality_LOW, SpatialLayer: 0, Rid: quarterResolutionQ},
{Quality: livekit.VideoQuality_MEDIUM, SpatialLayer: 1, Rid: halfResolutionH},
},
},
},
},
mime.MimeTypeVP8,
map[int32]livekit.VideoQuality{
InvalidLayerSpatial: livekit.VideoQuality_OFF,
0: livekit.VideoQuality_LOW,
1: livekit.VideoQuality_MEDIUM,
2: livekit.VideoQuality_OFF,
},
},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
for testSpatialLayer, expectedVideoQuality := range test.spatialLayerToVideoQuality {
actualVideoQuality := GetVideoQualityForSpatialLayer(test.mimeType, testSpatialLayer, test.trackInfo)
require.Equal(t, expectedVideoQuality, actualVideoQuality)
}
})
}
}
func TestNormalizeVideoLayersRid(t *testing.T) {
tests := []struct {
name string
rids VideoLayersRid
normalized VideoLayersRid
}{
{
"empty",
VideoLayersRid{},
VideoLayersRid{},
},
{
"unknown pattern",
VideoLayersRid{"3", "2", "1"},
VideoLayersRid{"3", "2", "1"},
},
{
"qhf",
videoLayersRidQHF,
videoLayersRidQHF,
},
{
"scrambled qhf",
VideoLayersRid{"f", "h", "q"},
videoLayersRidQHF,
},
{
"partial qhf",
VideoLayersRid{"h", "q"},
VideoLayersRid{"q", "h", ""},
},
{
"210",
videoLayersRid210,
videoLayersRid210,
},
{
"scrambled 210",
VideoLayersRid{"2", "0", "1"},
videoLayersRid210,
},
{
"partial 210",
VideoLayersRid{"1", "2"},
VideoLayersRid{"2", "1", ""},
},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
normalizedRids := NormalizeVideoLayersRid(test.rids)
require.Equal(t, test.normalized, normalizedRids)
})
}
}
pkg/sfu/bwe/bwe.go
+25
View File
@@ -31,6 +31,29 @@ const (
// ------------------------------------------------
type BWEType int
const (
BWETypeNone BWEType = iota
BWETypeRemote
BWETypeSendSide
)
func (b BWEType) String() string {
switch b {
case BWETypeNone:
return "NONE"
case BWETypeRemote:
return "REMOTE"
case BWETypeSendSide:
return "SEND_SIDE"
default:
return fmt.Sprintf("%d", int(b))
}
}
// ------------------------------------------------
type CongestionState int
const (
@@ -55,6 +78,8 @@ func (c CongestionState) String() string {
// ------------------------------------------------
type BWE interface {
Type() BWEType
SetBWEListener(bweListner BWEListener)
Reset()
pkg/sfu/bwe/remotebwe/remote_bwe.go
+6
View File
@@ -24,6 +24,8 @@ import (
"github.com/livekit/protocol/utils/mono"
)
var _ bwe.BWE = (*RemoteBWE)(nil)
// ---------------------------------------------------------------------------
type RemoteBWEConfig struct {
@@ -81,6 +83,10 @@ func NewRemoteBWE(params RemoteBWEParams) *RemoteBWE {
return r
}
func (r *RemoteBWE) Type() bwe.BWEType {
return bwe.BWETypeRemote
}
func (r *RemoteBWE) SetBWEListener(bweListener bwe.BWEListener) {
r.lock.Lock()
defer r.lock.Unlock()
pkg/sfu/bwe/sendsidebwe/congestion_detector.go
+68 -31
View File
@@ -121,8 +121,8 @@ var (
MinBytesRatio: 0.5,
MinDurationRatio: 0.5,
JQRMinDelay: 40 * time.Millisecond,
DQRMaxDelay: 15 * time.Millisecond,
JQRMinDelay: 50 * time.Millisecond,
DQRMaxDelay: 20 * time.Millisecond,
WeightedLoss: defaultWeightedLossConfig,
JQRMinWeightedLoss: 0.25,
@@ -177,17 +177,20 @@ func (c congestionReason) String() string {
}
// -------------------------------------------------------------------------------
type qdMeasurement struct {
jqrConfig CongestionSignalConfig
dqrConfig CongestionSignalConfig
jqrMin int64
dqrMax int64
numGroups int
numJQRGroups int
numDQRGroups int
minSendTime int64
maxSendTime int64
type qdMeasurement struct {
jqrConfig CongestionSignalConfig
dqrConfig CongestionSignalConfig
jqrMinDelay int64
jqrMinTrendCoefficient float64
dqrMaxDelay int64
numGroups int
numJQRGroups int
numDQRGroups int
minSendTime int64
maxSendTime int64
propagatedQueuingDelays []int64
isSealed bool
minGroupIdx int
@@ -196,13 +199,14 @@ type qdMeasurement struct {
queuingRegion queuingRegion
}
func newQDMeasurement(jqrConfig CongestionSignalConfig, dqrConfig CongestionSignalConfig, jqrMin int64, dqrMax int64) *qdMeasurement {
func newQDMeasurement(jqrConfig CongestionSignalConfig, dqrConfig CongestionSignalConfig, jqrMinDelay int64, jqrMinTrendCoefficient float64, dqrMaxDelay int64) *qdMeasurement {
return &qdMeasurement{
jqrConfig: jqrConfig,
dqrConfig: dqrConfig,
jqrMin: jqrMin,
dqrMax: dqrMax,
queuingRegion: queuingRegionIndeterminate,
jqrConfig: jqrConfig,
dqrConfig: dqrConfig,
jqrMinDelay: jqrMinDelay,
jqrMinTrendCoefficient: jqrMinTrendCoefficient,
dqrMaxDelay: dqrMaxDelay,
queuingRegion: queuingRegionIndeterminate,
}
}
@@ -228,8 +232,10 @@ func (q *qdMeasurement) ProcessPacketGroup(pg *packetGroup, groupIdx int) {
}
q.maxSendTime = max(q.maxSendTime, maxSendTime)
q.propagatedQueuingDelays = append(q.propagatedQueuingDelays, pqd)
switch {
case pqd < q.dqrMax:
case pqd < q.dqrMaxDelay:
q.numDQRGroups++
if q.numJQRGroups > 0 {
// broken continuity, seal
@@ -239,12 +245,12 @@ func (q *qdMeasurement) ProcessPacketGroup(pg *packetGroup, groupIdx int) {
q.queuingRegion = queuingRegionDQR
}
case pqd > q.jqrMin:
case pqd > q.jqrMinDelay:
q.numJQRGroups++
if q.numDQRGroups > 0 {
// broken continuity, seal
q.isSealed = true
} else if q.jqrConfig.IsTriggered(q.numJQRGroups, q.maxSendTime-q.minSendTime) {
} else if q.jqrConfig.IsTriggered(q.numJQRGroups, q.maxSendTime-q.minSendTime) && q.trendCoefficient() > q.jqrMinTrendCoefficient {
q.isSealed = true
q.queuingRegion = queuingRegionJQR
}
@@ -279,6 +285,8 @@ func (q *qdMeasurement) MarshalLogObject(e zapcore.ObjectEncoder) error {
e.AddInt("numDQRGroups", q.numDQRGroups)
e.AddInt64("minSendTime", q.minSendTime)
e.AddInt64("maxSendTime", q.maxSendTime)
e.AddArray("propagatedQueuingDelays", logger.Int64Slice(q.propagatedQueuingDelays))
e.AddFloat64("trendCoefficient", q.trendCoefficient())
e.AddDuration("duration", time.Duration((q.maxSendTime-q.minSendTime)*1000))
e.AddBool("isSealed", q.isSealed)
e.AddInt("minGroupIdx", q.minGroupIdx)
@@ -287,6 +295,35 @@ func (q *qdMeasurement) MarshalLogObject(e zapcore.ObjectEncoder) error {
return nil
}
func (q *qdMeasurement) trendCoefficient() float64 {
concordantPairs := 0
discordantPairs := 0
// the packet groups are processed from newest to oldest,
// so a concordant pair is when the value drops,
// i. e. the propagated queuing delay is increasing if newer (earlier entity in slice) is higher than older (later entity in slice)
for i := 0; i < len(q.propagatedQueuingDelays)-1; i++ {
for j := i + 1; j < len(q.propagatedQueuingDelays); j++ {
if q.propagatedQueuingDelays[i] > q.propagatedQueuingDelays[j] {
concordantPairs++
} else if q.propagatedQueuingDelays[i] < q.propagatedQueuingDelays[j] {
discordantPairs++
}
}
}
if (concordantPairs + discordantPairs) == 0 {
// if the min requirements is only one sample, trend calculation is not possible, declare highest trend value
if len(q.propagatedQueuingDelays) == 1 {
return 1.0
}
return 0.0
}
return (float64(concordantPairs) - float64(discordantPairs)) / (float64(concordantPairs) + float64(discordantPairs))
}
// -------------------------------------------------------------------------------
type lossMeasurement struct {
@@ -405,8 +442,9 @@ type CongestionDetectorConfig struct {
ProbeRegulator ccutils.ProbeRegulatorConfig `yaml:"probe_regulator,omitempty"`
ProbeSignal ProbeSignalConfig `yaml:"probe_signal,omitempty"`
JQRMinDelay time.Duration `yaml:"jqr_min_delay,omitempty"`
DQRMaxDelay time.Duration `yaml:"dqr_max_delay,omitempty"`
JQRMinDelay time.Duration `yaml:"jqr_min_delay,omitempty"`
JQRMinTrendCoefficient float64 `yaml:"jqr_min_trend_coefficient,omitempty"`
DQRMaxDelay time.Duration `yaml:"dqr_max_delay,omitempty"`
WeightedLoss WeightedLossConfig `yaml:"weighted_loss,omitempty"`
JQRMinWeightedLoss float64 `yaml:"jqr_min_weighted_loss,omitempty"`
@@ -452,8 +490,9 @@ var (
ProbeRegulator: ccutils.DefaultProbeRegulatorConfig,
ProbeSignal: defaultProbeSignalConfig,
JQRMinDelay: 40 * time.Millisecond,
DQRMaxDelay: 15 * time.Millisecond,
JQRMinDelay: 50 * time.Millisecond,
JQRMinTrendCoefficient: 0.8,
DQRMaxDelay: 20 * time.Millisecond,
WeightedLoss: defaultWeightedLossConfig,
JQRMinWeightedLoss: 0.25,
@@ -570,10 +609,7 @@ func (c *congestionDetector) getBWEListener() bwe.BWEListener {
func (c *congestionDetector) HandleTWCCFeedback(report *rtcp.TransportLayerCC) {
c.lock.Lock()
recvRefTime, isOutOfOrder := c.twccFeedback.ProcessReport(report, mono.Now())
if isOutOfOrder {
c.params.Logger.Infow("send side bwe: received out-of-order feedback report")
}
recvRefTime, _ := c.twccFeedback.ProcessReport(report, mono.Now())
if len(c.packetGroups) == 0 {
c.packetGroups = append(
@@ -635,7 +671,7 @@ func (c *congestionDetector) HandleTWCCFeedback(report *rtcp.TransportLayerCC) {
}
}
// 1. go through the TWCC feedback report and record recive time as reported by remote
// 1. go through the TWCC feedback report and record receive time as reported by remote
// 2. process acknowledged packet and group them
//
// losses are not recorded if a feedback report is completely lost.
@@ -852,6 +888,7 @@ func (c *congestionDetector) updateCongestionSignal(
qdJQRConfig,
qdDQRConfig,
c.params.Config.JQRMinDelay.Microseconds(),
c.params.Config.JQRMinTrendCoefficient,
c.params.Config.DQRMaxDelay.Microseconds(),
)
c.lossMeasurement = newLossMeasurement(
@@ -1109,7 +1146,7 @@ func (c *congestionDetector) updateCongestionState(state bwe.CongestionState) (b
// when in congested state, monitor changes in captured traffic ratio (CTR)
// to ensure allocations are in line with latest estimates, it is possible that
// the estimate is incorrect when congestion starts and the allocation may be
// sub-optimal and not enough to reduce/relieve congestion, by monitoing CTR
// sub-optimal and not enough to reduce/relieve congestion, by monitoring CTR
// on a continuous basis allocations can be adjusted in the direction of
// reducing/relieving congestion
if state == bwe.CongestionStateCongested && fromState != bwe.CongestionStateCongested {
pkg/sfu/bwe/sendsidebwe/send_side_bwe.go
+6
View File
@@ -23,6 +23,8 @@ import (
"github.com/pion/rtcp"
)
var _ bwe.BWE = (*SendSideBWE)(nil)
//
// Based on a simplified/modified version of JitterPath paper
// (https://homepage.iis.sinica.edu.tw/papers/lcs/2114-F.pdf)
@@ -88,6 +90,10 @@ func NewSendSideBWE(params SendSideBWEParams) *SendSideBWE {
}
}
func (s *SendSideBWE) Type() bwe.BWEType {
return bwe.BWETypeSendSide
}
func (s *SendSideBWE) SetBWEListener(bweListener bwe.BWEListener) {
s.congestionDetector.SetBWEListener(bweListener)
}
pkg/sfu/bwe/sendsidebwe/twcc_feedback.go
+6 -7
View File
@@ -15,7 +15,6 @@
package sendsidebwe
import (
"errors"
"time"
"github.com/livekit/protocol/logger"
@@ -35,12 +34,6 @@ const (
// ------------------------------------------------------
var (
errFeedbackReportOutOfOrder = errors.New("feedback report out-of-order")
)
// ------------------------------------------------------
type twccFeedbackParams struct {
Logger logger.Logger
}
@@ -78,6 +71,12 @@ func (t *twccFeedback) ProcessReport(report *rtcp.TransportLayerCC, at time.Time
if (report.FbPktCount - t.highestFeedbackCount) > (1 << 7) {
t.numReportsOutOfOrder++
isOutOfOrder = true
t.params.Logger.Infow(
"send side bwe: received out-of-order feedback report",
"highestFeedbackCount", t.highestFeedbackCount,
"fbPktCount", report.FbPktCount,
"numReportsOutOfOrder", t.numReportsOutOfOrder,
)
}
// reference time wrap around handling
pkg/sfu/codecmunger/codecmunger.go
+2 -2
View File
@@ -27,8 +27,8 @@ var (
)
type CodecMunger interface {
GetState() interface{}
SeedState(state interface{})
GetState() any
SeedState(state any)
SetLast(extPkt *buffer.ExtPacket)
UpdateOffsets(extPkt *buffer.ExtPacket)
pkg/sfu/codecmunger/null.go
+4 -4
View File
@@ -20,22 +20,22 @@ import (
)
type Null struct {
seededState interface{}
seededState any
}
func NewNull(_logger logger.Logger) *Null {
return &Null{}
}
func (n *Null) GetState() interface{} {
func (n *Null) GetState() any {
return nil
}
func (n *Null) SeedState(state interface{}) {
func (n *Null) SeedState(state any) {
n.seededState = state
}
func (n *Null) GetSeededState() interface{} {
func (n *Null) GetSeededState() any {
return n.seededState
}
pkg/sfu/codecmunger/vp8.go
+25 -14
View File
@@ -15,8 +15,9 @@
package codecmunger
import (
"github.com/elliotchance/orderedmap/v2"
"github.com/elliotchance/orderedmap/v3"
"github.com/livekit/mediatransportutil/pkg/codec"
"github.com/livekit/protocol/livekit"
"github.com/livekit/protocol/logger"
@@ -60,13 +61,18 @@ func NewVP8(logger logger.Logger) *VP8 {
}
}
func NewVP8FromNull(cm CodecMunger, logger logger.Logger) *VP8 {
func NewVP8FromOther(cm CodecMunger, logger logger.Logger) *VP8 {
v := NewVP8(logger)
v.SeedState(cm.(*Null).GetSeededState())
switch cm := cm.(type) {
case *Null:
v.SeedState(cm.GetSeededState())
case *VP8:
v.SeedState(cm.GetState())
}
return v
}
func (v *VP8) GetState() interface{} {
func (v *VP8) GetState() any {
return &livekit.VP8MungerState{
ExtLastPictureId: v.extLastPictureId,
PictureIdUsed: v.pictureIdUsed,
@@ -78,10 +84,15 @@ func (v *VP8) GetState() interface{} {
}
}
func (v *VP8) SeedState(seed interface{}) {
func (v *VP8) SeedState(seed any) {
var state *livekit.VP8MungerState
switch cm := seed.(type) {
case *livekit.RTPForwarderState_Vp8Munger:
state := cm.Vp8Munger
state = cm.Vp8Munger
case *livekit.VP8MungerState:
state = cm
}
if state != nil {
v.extLastPictureId = state.ExtLastPictureId
v.pictureIdUsed = state.PictureIdUsed
v.lastTl0PicIdx = uint8(state.LastTl0PicIdx)
@@ -93,7 +104,7 @@ func (v *VP8) SeedState(seed interface{}) {
}
func (v *VP8) SetLast(extPkt *buffer.ExtPacket) {
vp8, ok := extPkt.Payload.(buffer.VP8)
vp8, ok := extPkt.Payload.(codec.VP8)
if !ok {
return
}
@@ -118,7 +129,7 @@ func (v *VP8) SetLast(extPkt *buffer.ExtPacket) {
}
func (v *VP8) UpdateOffsets(extPkt *buffer.ExtPacket) {
vp8, ok := extPkt.Payload.(buffer.VP8)
vp8, ok := extPkt.Payload.(codec.VP8)
if !ok {
return
}
@@ -143,7 +154,7 @@ func (v *VP8) UpdateOffsets(extPkt *buffer.ExtPacket) {
}
func (v *VP8) UpdateAndGet(extPkt *buffer.ExtPacket, snOutOfOrder bool, snHasGap bool, maxTemporalLayer int32) (int, []byte, error) {
vp8, ok := extPkt.Payload.(buffer.VP8)
vp8, ok := extPkt.Payload.(codec.VP8)
if !ok {
return 0, nil, ErrNotVP8
}
@@ -164,7 +175,7 @@ func (v *VP8) UpdateAndGet(extPkt *buffer.ExtPacket, snOutOfOrder bool, snHasGap
// when it reaches a certain size.
mungedPictureId := uint16((extPictureId - pictureIdOffset) & 0x7fff)
vp8Packet := &buffer.VP8{
vp8Packet := codec.VP8{
FirstByte: vp8.FirstByte,
I: vp8.I,
M: mungedPictureId > 127,
@@ -177,7 +188,7 @@ func (v *VP8) UpdateAndGet(extPkt *buffer.ExtPacket, snOutOfOrder bool, snHasGap
K: vp8.K,
KEYIDX: vp8.KEYIDX - v.keyIdxOffset,
IsKeyFrame: vp8.IsKeyFrame,
HeaderSize: vp8.HeaderSize + buffer.VPxPictureIdSizeDiff(mungedPictureId > 127, vp8.M),
HeaderSize: vp8.HeaderSize + codec.VPxPictureIdSizeDiff(mungedPictureId > 127, vp8.M),
}
vp8HeaderBytes, err := vp8Packet.Marshal()
if err != nil {
@@ -271,7 +282,7 @@ func (v *VP8) UpdateAndGet(extPkt *buffer.ExtPacket, snOutOfOrder bool, snHasGap
v.lastTl0PicIdx = mungedTl0PicIdx
v.lastKeyIdx = mungedKeyIdx
vp8Packet := &buffer.VP8{
vp8Packet := codec.VP8{
FirstByte: vp8.FirstByte,
I: vp8.I,
M: mungedPictureId > 127,
@@ -284,7 +295,7 @@ func (v *VP8) UpdateAndGet(extPkt *buffer.ExtPacket, snOutOfOrder bool, snHasGap
K: vp8.K,
KEYIDX: mungedKeyIdx,
IsKeyFrame: vp8.IsKeyFrame,
HeaderSize: vp8.HeaderSize + buffer.VPxPictureIdSizeDiff(mungedPictureId > 127, vp8.M),
HeaderSize: vp8.HeaderSize + codec.VPxPictureIdSizeDiff(mungedPictureId > 127, vp8.M),
}
vp8HeaderBytes, err := vp8Packet.Marshal()
if err != nil {
@@ -336,7 +347,7 @@ func (v *VP8) UpdateAndGetPadding(newPicture bool) ([]byte, error) {
v.keyIdxOffset -= uint8(offset)
}
vp8Packet := &buffer.VP8{
vp8Packet := &codec.VP8{
FirstByte: 0x10, // partition 0, start of VP8 Partition, reference frame
I: v.pictureIdUsed,
M: pictureId > 127,
pkg/sfu/codecmunger/vp8_test.go
+14 -14
View File
@@ -20,9 +20,9 @@ import (
"github.com/stretchr/testify/require"
"github.com/livekit/mediatransportutil/pkg/codec"
"github.com/livekit/protocol/logger"
"github.com/livekit/livekit-server/pkg/sfu/buffer"
"github.com/livekit/livekit-server/pkg/sfu/testutils"
)
@@ -52,7 +52,7 @@ func TestSetLast(t *testing.T) {
Timestamp: 0xabcdef,
SSRC: 0x12345678,
}
vp8 := &buffer.VP8{
vp8 := &codec.VP8{
FirstByte: 25,
I: true,
M: true,
@@ -102,7 +102,7 @@ func TestUpdateOffsets(t *testing.T) {
Timestamp: 0xabcdef,
SSRC: 0x12345678,
}
vp8 := &buffer.VP8{
vp8 := &codec.VP8{
FirstByte: 25,
I: true,
M: true,
@@ -125,7 +125,7 @@ func TestUpdateOffsets(t *testing.T) {
Timestamp: 0xabcdef,
SSRC: 0x87654321,
}
vp8 = &buffer.VP8{
vp8 = &codec.VP8{
FirstByte: 25,
I: true,
M: true,
@@ -172,7 +172,7 @@ func TestOutOfOrderPictureId(t *testing.T) {
Timestamp: 0xabcdef,
SSRC: 0x12345678,
}
vp8 := &buffer.VP8{
vp8 := &codec.VP8{
FirstByte: 25,
I: true,
M: true,
@@ -205,7 +205,7 @@ func TestOutOfOrderPictureId(t *testing.T) {
vp8.PictureID = 13469
extPkt, _ = testutils.GetTestExtPacketVP8(params, vp8)
expectedVP8 := &buffer.VP8{
expectedVP8 := &codec.VP8{
FirstByte: 25,
I: true,
M: true,
@@ -244,7 +244,7 @@ func TestOutOfOrderPictureId(t *testing.T) {
vp8.PictureID = 13468
extPkt, _ = testutils.GetTestExtPacketVP8(params, vp8)
expectedVP8 = &buffer.VP8{
expectedVP8 = &codec.VP8{
FirstByte: 25,
I: true,
M: true,
@@ -275,7 +275,7 @@ func TestTemporalLayerFiltering(t *testing.T) {
Timestamp: 0xabcdef,
SSRC: 0x12345678,
}
vp8 := &buffer.VP8{
vp8 := &codec.VP8{
FirstByte: 25,
I: true,
M: true,
@@ -341,7 +341,7 @@ func TestGapInSequenceNumberSamePicture(t *testing.T) {
SSRC: 0x12345678,
PayloadSize: 33,
}
vp8 := &buffer.VP8{
vp8 := &codec.VP8{
FirstByte: 25,
I: true,
M: true,
@@ -359,7 +359,7 @@ func TestGapInSequenceNumberSamePicture(t *testing.T) {
extPkt, _ := testutils.GetTestExtPacketVP8(params, vp8)
v.SetLast(extPkt)
expectedVP8 := &buffer.VP8{
expectedVP8 := &codec.VP8{
FirstByte: 25,
I: true,
M: true,
@@ -382,7 +382,7 @@ func TestGapInSequenceNumberSamePicture(t *testing.T) {
require.Equal(t, marshalledVP8, buf)
// telling there is a gap in sequence number will add pictures to missing picture cache
expectedVP8 = &buffer.VP8{
expectedVP8 = &codec.VP8{
FirstByte: 25,
I: true,
M: true,
@@ -418,7 +418,7 @@ func TestUpdateAndGetPadding(t *testing.T) {
SSRC: 0x12345678,
PayloadSize: 20,
}
vp8 := &buffer.VP8{
vp8 := &codec.VP8{
FirstByte: 25,
I: true,
M: true,
@@ -440,7 +440,7 @@ func TestUpdateAndGetPadding(t *testing.T) {
// getting padding with repeat of last picture
buf, err := v.UpdateAndGetPadding(false)
require.NoError(t, err)
expectedVP8 := buffer.VP8{
expectedVP8 := codec.VP8{
FirstByte: 16,
I: true,
M: true,
@@ -462,7 +462,7 @@ func TestUpdateAndGetPadding(t *testing.T) {
// getting padding with new picture
buf, err = v.UpdateAndGetPadding(true)
require.NoError(t, err)
expectedVP8 = buffer.VP8{
expectedVP8 = codec.VP8{
FirstByte: 16,
I: true,
M: true,
pkg/sfu/connectionquality/connectionstats.go
+31 -11
View File
@@ -22,11 +22,11 @@ import (
"go.uber.org/atomic"
"google.golang.org/protobuf/types/known/timestamppb"
"github.com/livekit/protocol/codecs/mime"
"github.com/livekit/protocol/livekit"
"github.com/livekit/protocol/logger"
"github.com/livekit/livekit-server/pkg/sfu/buffer"
"github.com/livekit/livekit-server/pkg/sfu/mime"
"github.com/livekit/livekit-server/pkg/sfu/rtpstats"
)
@@ -276,29 +276,30 @@ func (cs *ConnectionStats) updateScoreFromReceiverReport(at time.Time) (float32,
return cs.updateScoreWithAggregate(agg, time.Time{}, at), streams
}
func (cs *ConnectionStats) updateScoreAt(at time.Time) (float32, map[uint32]*buffer.StreamStatsWithLayers) {
func (cs *ConnectionStats) updateScoreAt(at time.Time) (float32, map[uint32]*buffer.StreamStatsWithLayers, bool) {
if cs.params.SenderProvider != nil {
// receiver report based quality scoring, use stats from receiver report for scoring
return cs.updateScoreFromReceiverReport(at)
score, streams := cs.updateScoreFromReceiverReport(at)
return score, streams, true
}
if cs.params.ReceiverProvider == nil {
return MinMOS, nil
return MinMOS, nil, false
}
streams := cs.params.ReceiverProvider.GetDeltaStats()
if len(streams) == 0 {
mos, _ := cs.scorer.GetMOSAndQuality()
return mos, nil
return mos, nil, false
}
agg := toAggregateDeltaInfo(streams, false)
if agg == nil {
// no receiver report in the window
mos, _ := cs.scorer.GetMOSAndQuality()
return mos, streams
return mos, streams, false
}
return cs.updateScoreWithAggregate(agg, cs.params.ReceiverProvider.GetLastSenderReportTime(), at), streams
return cs.updateScoreWithAggregate(agg, cs.params.ReceiverProvider.GetLastSenderReportTime(), at), streams, false
}
func (cs *ConnectionStats) updateStreamingStart(at time.Time) time.Time {
@@ -324,12 +325,12 @@ func (cs *ConnectionStats) updateStreamingStart(at time.Time) time.Time {
}
func (cs *ConnectionStats) getStat() {
score, streams := cs.updateScoreAt(time.Time{})
score, streams, isSender := cs.updateScoreAt(time.Time{})
if cs.onStatsUpdate != nil && len(streams) != 0 {
analyticsStreams := make([]*livekit.AnalyticsStream, 0, len(streams))
for ssrc, stream := range streams {
as := toAnalyticsStream(ssrc, stream.RTPStats, stream.RTPStatsRemoteView)
as := toAnalyticsStream(ssrc, stream.RTPStats, stream.RTPStatsRemoteView, isSender)
if as == nil {
continue
}
@@ -373,6 +374,7 @@ func (cs *ConnectionStats) updateStatsWorker() {
for {
select {
case <-cs.done.Watch():
cs.getStat()
return
case <-tk.C:
@@ -429,6 +431,13 @@ func toAggregateDeltaInfo(streams map[uint32]*buffer.StreamStatsWithLayers, useR
for _, s := range streams {
if useRemoteView {
if s.RTPStatsRemoteView != nil {
// discount jitter from publisher side + internal processing while reporting downstream jitter
if s.RTPStats != nil {
s.RTPStatsRemoteView.JitterMax -= s.RTPStats.JitterMax
if s.RTPStatsRemoteView.JitterMax < 0.0 {
s.RTPStatsRemoteView.JitterMax = 0.0
}
}
deltaInfoList = append(deltaInfoList, s.RTPStatsRemoteView)
}
} else {
@@ -444,13 +453,17 @@ func toAnalyticsStream(
ssrc uint32,
deltaStats *rtpstats.RTPDeltaInfo,
deltaStatsRemoteView *rtpstats.RTPDeltaInfo,
isSender bool,
) *livekit.AnalyticsStream {
if deltaStats == nil {
return nil
}
// discount the feed side loss when reporting forwarded track stats,
// discount jitter from publisher side + internal processing while reporting downstream jitter
packetsLost := deltaStats.PacketsLost
rtt := deltaStats.RttMax
maxJitter := deltaStats.JitterMax
if deltaStatsRemoteView != nil {
packetsLost = deltaStatsRemoteView.PacketsLost
if deltaStatsRemoteView.PacketsMissing > packetsLost {
@@ -458,7 +471,14 @@ func toAnalyticsStream(
} else {
packetsLost -= deltaStatsRemoteView.PacketsMissing
}
rtt = deltaStatsRemoteView.RttMax
maxJitter = deltaStatsRemoteView.JitterMax
} else if isSender {
rtt = 0
maxJitter = 0
}
return &livekit.AnalyticsStream{
StartTime: timestamppb.New(deltaStats.StartTime),
EndTime: timestamppb.New(deltaStats.EndTime),
@@ -472,8 +492,8 @@ func toAnalyticsStream(
PacketsLost: packetsLost,
PacketsOutOfOrder: deltaStats.PacketsOutOfOrder,
Frames: deltaStats.Frames,
Rtt: deltaStats.RttMax,
Jitter: uint32(deltaStats.JitterMax),
Rtt: rtt,
Jitter: uint32(maxJitter),
Nacks: deltaStats.Nacks,
Plis: deltaStats.Plis,
Firs: deltaStats.Firs,
pkg/sfu/connectionquality/connectionstats_test.go
+2 -2
View File
@@ -22,8 +22,8 @@ import (
"github.com/stretchr/testify/require"
"github.com/livekit/livekit-server/pkg/sfu/buffer"
"github.com/livekit/livekit-server/pkg/sfu/mime"
"github.com/livekit/livekit-server/pkg/sfu/rtpstats"
"github.com/livekit/protocol/codecs/mime"
"github.com/livekit/protocol/livekit"
"github.com/livekit/protocol/logger"
)
@@ -319,7 +319,7 @@ func TestConnectionQuality(t *testing.T) {
// unmute and send packets to bring quality back up
now = now.Add(duration)
cs.UpdateMuteAt(false, now.Add(2*time.Second))
for i := 0; i < 3; i++ {
for range 3 {
trp.setStreams(map[uint32]*buffer.StreamStatsWithLayers{
1: {
RTPStats: &rtpstats.RTPDeltaInfo{
pkg/sfu/connectionquality/scorer.go
+3 -6
View File
@@ -482,7 +482,7 @@ func (q *qualityScorer) updateAtLocked(stat *windowStat, at time.Time) {
ulgr.Debugw("quality rise")
default:
packets := stat.packets + stat.packetsPadding
if packets != 0 && (stat.packetsLost*100/packets) > 10 {
if packets != 0 && ((stat.packetsLost-stat.packetsMissing-stat.packetsOutOfOrder)*100/packets) > 10 {
ulgr.Debugw("quality hold - high loss")
}
}
@@ -525,10 +525,7 @@ func (q *qualityScorer) isUnmutedEnough(at time.Time) bool {
sinceLayerUnmute = at.Sub(q.layerUnmutedAt)
}
validDuration := sinceUnmute
if sinceLayerUnmute < validDuration {
validDuration = sinceLayerUnmute
}
validDuration := min(sinceLayerUnmute, sinceUnmute)
sinceLastUpdate := at.Sub(q.lastUpdateAt)
@@ -567,7 +564,7 @@ func (q *qualityScorer) getAdjustedPacketLossWeight(stat *windowStat) float64 {
// 2. enough time has elapsed since last calculation
if q.numPPSReadings > cPPSMinReadings && time.Since(q.modeCalculatedAt) > cModeCalculationInterval {
q.ppsMode = 0
for i := 0; i < len(q.ppsHistogram); i++ {
for i := range len(q.ppsHistogram) {
if q.ppsHistogram[i] > q.ppsMode {
q.ppsMode = i
}
pkg/sfu/datachannel/datachannel_writer.go
+72 -10
View File
@@ -15,24 +15,30 @@ const (
singleWriteTimeout = 50 * time.Millisecond
)
var ErrDataDroppedBySlowReader = errors.New("data dropped by slow reader")
var (
ErrDataDroppedBySlowReader = errors.New("data dropped by slow reader")
ErrDataDroppedByHighBufferedAmount = errors.New("data dropped due to high buffered amount")
)
type BufferedAmountGetter interface {
BufferedAmount() uint64
}
type DataChannelWriter[T BufferedAmountGetter] struct {
bufferGetter T
rawDC datachannel.ReadWriteCloserDeadliner
slowThreshold int
rate *BitrateCalculator
bufferGetter T
rawDC datachannel.ReadWriteCloserDeadliner
slowThreshold int
rate *BitrateCalculator
reliable bool
targetLatency time.Duration
minBufferedAmount uint64
}
// NewDataChannelWriter creates a new DataChannelWriter by detaching the data channel, when
// writing to the datachanel times out, it will block and retry if the receiver's bitrate is
// above the slowThreshold or drop the data if it's below the threshold. If the slowThreshold
// is 0, it will never retry on write timeout.
func NewDataChannelWriter[T BufferedAmountGetter](bufferGetter T, rawDC datachannel.ReadWriteCloserDeadliner, slowThreshold int) *DataChannelWriter[T] {
// NewDataChannelWriterReliable creates a new DataChannelWriter for reliable data channel by
// detaching it, when writing to the datachanel times out, it will block and retry if the
// receiver's bitrate is above the slowThreshold or drop the data if it's below the threshold.
// If the slowThreshold is 0, it will never retry on write timeout.
func NewDataChannelWriterReliable[T BufferedAmountGetter](bufferGetter T, rawDC datachannel.ReadWriteCloserDeadliner, slowThreshold int) *DataChannelWriter[T] {
var rate *BitrateCalculator
if slowThreshold > 0 {
rate = NewBitrateCalculator(BitrateDuration, BitrateWindow)
@@ -42,6 +48,26 @@ func NewDataChannelWriter[T BufferedAmountGetter](bufferGetter T, rawDC datachan
rawDC: rawDC,
slowThreshold: slowThreshold,
rate: rate,
reliable: true,
}
}
// NewDataChannelWriterUnreliable creates a new DataChannelWriter for unreliable data channel.
// It will drop data when the buffered amount is too high to maintain the target latency.
// The latency is estimated based on the bitrate in past 1 second. If targetLatency is 0, no
// buffering control is applied.
func NewDataChannelWriterUnreliable[T BufferedAmountGetter](bufferGetter T, rawDC datachannel.ReadWriteCloserDeadliner, targetLatency time.Duration, minBufferedAmount uint64) *DataChannelWriter[T] {
var rate *BitrateCalculator
if targetLatency > 0 {
rate = NewBitrateCalculator(BitrateDuration, BitrateWindow)
}
return &DataChannelWriter[T]{
bufferGetter: bufferGetter,
rawDC: rawDC,
rate: rate,
targetLatency: targetLatency,
minBufferedAmount: minBufferedAmount,
reliable: false,
}
}
@@ -50,6 +76,14 @@ func (w *DataChannelWriter[T]) BufferedAmountGetter() T {
}
func (w *DataChannelWriter[T]) Write(p []byte) (n int, err error) {
if w.reliable {
return w.writeReliable(p)
} else {
return w.writeUnreliable(p)
}
}
func (w *DataChannelWriter[T]) writeReliable(p []byte) (n int, err error) {
for {
err = w.rawDC.SetWriteDeadline(time.Now().Add(singleWriteTimeout))
if err != nil {
@@ -75,6 +109,34 @@ func (w *DataChannelWriter[T]) Write(p []byte) (n int, err error) {
}
}
func (w *DataChannelWriter[T]) writeUnreliable(p []byte) (n int, err error) {
if w.targetLatency == 0 {
err = w.rawDC.SetWriteDeadline(time.Now().Add(singleWriteTimeout))
if err != nil {
return 0, err
}
return w.rawDC.Write(p)
}
if bitrate, ok := w.rate.Bitrate(time.Now()); ok {
// control buffer latency to ~100ms
if w.bufferGetter.BufferedAmount() > uint64(time.Duration(bitrate)*w.targetLatency/8/time.Second) && w.bufferGetter.BufferedAmount() > w.minBufferedAmount {
return 0, ErrDataDroppedByHighBufferedAmount
}
}
err = w.rawDC.SetWriteDeadline(time.Now().Add(singleWriteTimeout))
if err != nil {
return 0, err
}
n, err = w.rawDC.Write(p)
if err != nil {
w.rate.AddBytes(n, int(w.bufferGetter.BufferedAmount()), mono.Now())
}
return
}
func (w *DataChannelWriter[T]) Close() error {
return w.rawDC.Close()
}
pkg/sfu/datachannel/datachannel_writer_test.go
+64 -4
View File
@@ -2,18 +2,19 @@ package datachannel
import (
"context"
"sync/atomic"
"testing"
"time"
"github.com/pion/datachannel"
"github.com/pion/transport/v3/deadline"
"github.com/pion/transport/v4/deadline"
"github.com/stretchr/testify/require"
)
func TestDataChannelWriter(t *testing.T) {
mockDC := newMockDataChannelWriter()
// slow threshold is 1000B/s
w := NewDataChannelWriter(mockDC, mockDC, 8000)
w := NewDataChannelWriterReliable(mockDC, mockDC, 8000)
require.Equal(t, mockDC, w.BufferedAmountGetter())
buf := make([]byte, 2000)
// write 2000 bytes so it should not drop in 2 seconds
@@ -23,7 +24,7 @@ func TestDataChannelWriter(t *testing.T) {
require.Equal(t, 2000, n)
t1 := time.Now()
mockDC.SetNextWriteCompleteAt(t0.Add(time.Second))
mockDC.SetNextWriteCompleteAt(t0.Add(time.Second * 3 / 2))
n, err = w.Write(buf[:10])
require.NoError(t, err)
require.Equal(t, 10, n)
@@ -38,7 +39,7 @@ func TestDataChannelWriter(t *testing.T) {
func TestDataChannelWriter_NoSlowThreshold(t *testing.T) {
mockDC := newMockDataChannelWriter()
w := NewDataChannelWriter(mockDC, mockDC, 0)
w := NewDataChannelWriterReliable(mockDC, mockDC, 0)
buf := make([]byte, 2000)
n, err := w.Write(buf)
require.NoError(t, err)
@@ -55,6 +56,24 @@ func TestDataChannelWriter_NoSlowThreshold(t *testing.T) {
require.Equal(t, 0, n)
}
func TestDataChannelWriter_Unreliable(t *testing.T) {
mockDC := newMockLossyDataChannelWriter(8192)
w := NewDataChannelWriterUnreliable(mockDC, mockDC, 100*time.Millisecond, 2000)
for range 10 {
buf := make([]byte, 128)
_, err := w.Write(buf)
require.NoError(t, err)
time.Sleep(100 * time.Millisecond)
}
buf := make([]byte, 4096)
_, err := w.Write(buf)
require.NoError(t, err)
// should drop due to high buffered amount
_, err = w.Write(buf)
require.ErrorIs(t, err, ErrDataDroppedByHighBufferedAmount)
}
// mockDataChannelWriter
type mockDataChannelWriter struct {
datachannel.ReadWriteCloserDeadliner
nextWriteCompleteAt time.Time
@@ -92,3 +111,44 @@ func (m *mockDataChannelWriter) SetWriteDeadline(t time.Time) error {
func (m *mockDataChannelWriter) SetNextWriteCompleteAt(t time.Time) {
m.nextWriteCompleteAt = t
}
// mockLossyDataChannelWriter
type mockLossyDataChannelWriter struct {
datachannel.ReadWriteCloserDeadliner
bufferedAmount atomic.Int64
targetBitrate int
lastWriteAt time.Time
}
func newMockLossyDataChannelWriter(targetBitrate int) *mockLossyDataChannelWriter {
return &mockLossyDataChannelWriter{
targetBitrate: targetBitrate,
lastWriteAt: time.Now(),
}
}
func (m *mockLossyDataChannelWriter) BufferedAmount() uint64 {
return uint64(m.bufferedAmount.Load())
}
func (m *mockLossyDataChannelWriter) Write(b []byte) (int, error) {
m.bufferedAmount.Add(int64(len(b)))
if time.Now().Before(m.lastWriteAt) {
return len(b), nil
}
// drain buffer based on target bitrate
canWriteBytes := time.Since(m.lastWriteAt) * time.Duration(m.targetBitrate) / time.Second / 8
if m.bufferedAmount.Load() <= int64(canWriteBytes) {
m.lastWriteAt = m.lastWriteAt.Add(time.Duration(int64(time.Second) * int64(m.BufferedAmount()) / (int64(m.targetBitrate) / 8)))
m.bufferedAmount.Store(0)
} else {
m.lastWriteAt = time.Now()
m.bufferedAmount.Add(-int64(canWriteBytes))
}
return len(b), nil
}
func (m *mockLossyDataChannelWriter) SetWriteDeadline(t time.Time) error {
return nil
}
pkg/sfu/downtrack.go
+463 -317
View File
File diff suppressed because it is too large Load Diff
pkg/sfu/forwarder.go
+246 -168
View File
@@ -27,6 +27,7 @@ import (
"go.uber.org/zap/zapcore"
"github.com/livekit/mediatransportutil"
"github.com/livekit/protocol/codecs/mime"
"github.com/livekit/protocol/livekit"
"github.com/livekit/protocol/logger"
"github.com/livekit/protocol/utils"
@@ -34,14 +35,13 @@ import (
"github.com/livekit/livekit-server/pkg/sfu/buffer"
"github.com/livekit/livekit-server/pkg/sfu/codecmunger"
"github.com/livekit/livekit-server/pkg/sfu/mime"
dd "github.com/livekit/livekit-server/pkg/sfu/rtpextension/dependencydescriptor"
"github.com/livekit/livekit-server/pkg/sfu/rtpstats"
sfuutils "github.com/livekit/livekit-server/pkg/sfu/utils"
"github.com/livekit/livekit-server/pkg/sfu/videolayerselector"
"github.com/livekit/livekit-server/pkg/sfu/videolayerselector/temporallayerselector"
)
// Forwarder
const (
FlagPauseOnDowngrade = true
FlagFilterRTX = false
@@ -54,6 +54,11 @@ const (
SwitchAheadThresholdSeconds = float64(0.025) // 25ms
)
var (
errSkipStartOnOutOfOrderPacket = errors.New("skip start on out-of-order packet")
errSwitchPointTooFarBehind = errors.New("switch point too far behind")
)
// -------------------------------------------------------------------
type VideoPauseReason int
@@ -180,6 +185,7 @@ func (v *VideoTransition) MarshalLogObject(e zapcore.ObjectEncoder) error {
type TranslationParams struct {
shouldDrop bool
isStarting bool
isResuming bool
isSwitching bool
rtp TranslationParamsRTP
@@ -209,13 +215,14 @@ func (r refInfo) MarshalLogObject(e zapcore.ObjectEncoder) error {
// -------------------------------------------------------------------
type Forwarder struct {
lock sync.RWMutex
mime mime.MimeType
clockRate uint32
kind webrtc.RTPCodecType
logger logger.Logger
skipReferenceTS bool
rtpStats *rtpstats.RTPStatsSender
lock sync.RWMutex
mime mime.MimeType
clockRate uint32
kind webrtc.RTPCodecType
logger logger.Logger
skipReferenceTS bool
disableOpportunisticAllocation bool
rtpStats *rtpstats.RTPStatsSender
muted bool
pubMuted bool
@@ -230,7 +237,8 @@ type Forwarder struct {
referenceLayerSpatial int32
dummyStartTSOffset uint64
refInfos [buffer.DefaultMaxLayerSpatial + 1]refInfo
refIsSVC bool
refVideoLayerMode livekit.VideoLayer_Mode
isDDAvailable bool
provisional *VideoAllocationProvisional
@@ -247,20 +255,22 @@ func NewForwarder(
kind webrtc.RTPCodecType,
logger logger.Logger,
skipReferenceTS bool,
disableOpportunisticAllocation bool,
rtpStats *rtpstats.RTPStatsSender,
) *Forwarder {
f := &Forwarder{
mime: mime.MimeTypeUnknown,
kind: kind,
logger: logger,
skipReferenceTS: skipReferenceTS,
rtpStats: rtpStats,
referenceLayerSpatial: buffer.InvalidLayerSpatial,
lastAllocation: VideoAllocationDefault,
lastReferencePayloadType: -1,
rtpMunger: NewRTPMunger(logger),
vls: videolayerselector.NewNull(logger),
codecMunger: codecmunger.NewNull(logger),
mime: mime.MimeTypeUnknown,
kind: kind,
logger: logger,
skipReferenceTS: skipReferenceTS,
disableOpportunisticAllocation: disableOpportunisticAllocation,
rtpStats: rtpStats,
referenceLayerSpatial: buffer.InvalidLayerSpatial,
lastAllocation: VideoAllocationDefault,
lastReferencePayloadType: -1,
rtpMunger: NewRTPMunger(logger),
vls: videolayerselector.NewNull(logger),
codecMunger: codecmunger.NewNull(logger),
}
if f.kind == webrtc.RTPCodecTypeVideo {
@@ -297,10 +307,14 @@ func (f *Forwarder) SetMaxTemporalLayerSeen(maxTemporalLayerSeen int32) bool {
return true
}
func (f *Forwarder) DetermineCodec(codec webrtc.RTPCodecCapability, extensions []webrtc.RTPHeaderExtensionParameter) {
func (f *Forwarder) DetermineCodec(codec webrtc.RTPCodecCapability, extensions []webrtc.RTPHeaderExtensionParameter, videoLayerMode livekit.VideoLayer_Mode) {
f.lock.Lock()
defer f.lock.Unlock()
if videoLayerMode == livekit.VideoLayer_ONE_SPATIAL_LAYER_PER_STREAM_INCOMPLETE_RTCP_SR {
f.skipReferenceTS = true
}
toMimeType := mime.NormalizeMimeType(codec.MimeType)
codecChanged := f.mime != mime.MimeTypeUnknown && f.mime != toMimeType
if codecChanged {
@@ -308,6 +322,7 @@ func (f *Forwarder) DetermineCodec(codec webrtc.RTPCodecCapability, extensions [
}
f.mime = toMimeType
f.clockRate = codec.ClockRate
f.refVideoLayerMode = videoLayerMode
ddAvailable := func(exts []webrtc.RTPHeaderExtensionParameter) bool {
for _, ext := range exts {
@@ -320,7 +335,7 @@ func (f *Forwarder) DetermineCodec(codec webrtc.RTPCodecCapability, extensions [
switch f.mime {
case mime.MimeTypeVP8:
f.codecMunger = codecmunger.NewVP8FromNull(f.codecMunger, f.logger)
f.codecMunger = codecmunger.NewVP8FromOther(f.codecMunger, f.logger)
if f.vls != nil {
if vls := videolayerselector.NewSimulcastFromOther(f.vls); vls != nil {
f.vls = vls
@@ -333,47 +348,66 @@ func (f *Forwarder) DetermineCodec(codec webrtc.RTPCodecCapability, extensions [
f.vls.SetTemporalLayerSelector(temporallayerselector.NewVP8(f.logger))
case mime.MimeTypeH264, mime.MimeTypeH265:
f.codecMunger = codecmunger.NewNull(f.logger)
if f.vls != nil {
f.vls = videolayerselector.NewSimulcastFromOther(f.vls)
if vls := videolayerselector.NewSimulcastFromOther(f.vls); vls != nil {
f.vls = vls
} else {
f.logger.Errorw("failed to create simulcast on codec change", nil)
}
} else {
f.vls = videolayerselector.NewSimulcast(f.logger)
}
case mime.MimeTypeVP9:
// DD-TODO : we only enable dd layer selector for av1/vp9 now, in the future we can enable it for vp8 too
isDDAvailable := ddAvailable(extensions)
if isDDAvailable {
f.codecMunger = codecmunger.NewNull(f.logger)
if sfuutils.IsSimulcastMode(videoLayerMode) {
if f.vls != nil {
f.vls = videolayerselector.NewDependencyDescriptorFromOther(f.vls)
f.vls = videolayerselector.NewSimulcastFromOther(f.vls)
} else {
f.vls = videolayerselector.NewDependencyDescriptor(f.logger)
}
} else {
if f.vls != nil {
f.vls = videolayerselector.NewVP9FromOther(f.vls)
f.isDDAvailable = ddAvailable(extensions)
if f.isDDAvailable {
if f.vls != nil {
f.vls = videolayerselector.NewDependencyDescriptorFromOther(f.vls)
} else {
f.vls = videolayerselector.NewDependencyDescriptor(f.logger)
}
} else {
f.vls = videolayerselector.NewVP9(f.logger)
if f.vls != nil {
f.vls = videolayerselector.NewVP9FromOther(f.vls)
} else {
f.vls = videolayerselector.NewVP9(f.logger)
}
}
}
// SVC-TODO: Support for VP9 simulcast. When DD is not available, have to pick selector based on VP9 SVC or Simulcast
case mime.MimeTypeAV1:
// DD-TODO : we only enable dd layer selector for av1/vp9 now, in the future we can enable it for vp8 too
isDDAvailable := ddAvailable(extensions)
if isDDAvailable {
if f.vls != nil {
f.vls = videolayerselector.NewDependencyDescriptorFromOther(f.vls)
} else {
f.vls = videolayerselector.NewDependencyDescriptor(f.logger)
}
} else {
f.codecMunger = codecmunger.NewNull(f.logger)
if sfuutils.IsSimulcastMode(videoLayerMode) {
if f.vls != nil {
f.vls = videolayerselector.NewSimulcastFromOther(f.vls)
} else {
f.vls = videolayerselector.NewSimulcast(f.logger)
}
} else {
f.isDDAvailable = ddAvailable(extensions)
if f.isDDAvailable {
if f.vls != nil {
f.vls = videolayerselector.NewDependencyDescriptorFromOther(f.vls)
} else {
f.vls = videolayerselector.NewDependencyDescriptor(f.logger)
}
} else {
if f.vls != nil {
f.vls = videolayerselector.NewSimulcastFromOther(f.vls)
} else {
f.vls = videolayerselector.NewSimulcast(f.logger)
}
}
}
// SVC-TODO: Support for AV1 Simulcast
}
}
@@ -460,7 +494,12 @@ func (f *Forwarder) Mute(muted bool, isSubscribeMutable bool) bool {
// It could result in some bandwidth consumed for stream without visibility in
// the case of intentional mute.
if muted && !isSubscribeMutable {
f.logger.Debugw("ignoring forwarder mute, paused due to congestion")
f.logger.Infow(
"ignoring forwarder mute, paused due to congestion",
"targetLayers", f.vls.GetTarget(),
"currentLayers", f.vls.GetCurrent(),
"lastAllocation", f.lastAllocation,
)
return false
}
@@ -529,6 +568,41 @@ func (f *Forwarder) SetMaxSpatialLayer(spatialLayer int32) (bool, buffer.VideoLa
f.logger.Debugw("setting max spatial layer", "layer", spatialLayer)
f.vls.SetMaxSpatial(spatialLayer)
if f.disableOpportunisticAllocation {
return true, f.vls.GetMax()
}
if f.vls.GetTarget().Spatial != buffer.InvalidLayerSpatial ||
f.isDeficientLocked() ||
f.lastAllocation.PauseReason == VideoPauseReasonMuted ||
f.lastAllocation.PauseReason == VideoPauseReasonPubMuted {
return true, f.vls.GetMax()
}
f.logger.Debugw("opportunistically setting target spatial layer", "layer", spatialLayer)
alloc := f.lastAllocation
// bitrates are not known
alloc.BandwidthRequested = 0
alloc.BandwidthDelta = 0
alloc.Bitrates = Bitrates{}
alloc.TargetLayer = f.vls.GetMax()
alloc.RequestLayerSpatial = f.vls.GetMax().Spatial
alloc.MaxLayer = f.vls.GetMax()
alloc.DistanceToDesired = getDistanceToDesired(
f.muted,
f.pubMuted,
f.vls.GetMaxSeen(),
nil,
alloc.Bitrates,
alloc.TargetLayer,
f.vls.GetMax(),
)
f.updateAllocation(alloc, "opportunistic")
return true, f.vls.GetMax()
}
@@ -577,14 +651,10 @@ func (f *Forwarder) GetMaxSubscribedSpatial() int32 {
layer := buffer.InvalidLayerSpatial // covers muted case
if !f.muted {
layer = f.vls.GetMax().Spatial
// If current is higher, mark the current layer as max subscribed layer
// to prevent the current layer from stopping before forwarder switches
// to the new and lower max layer,
if layer < f.vls.GetCurrent().Spatial {
layer = f.vls.GetCurrent().Spatial
}
layer = max(f.vls.GetMax().Spatial, f.vls.GetCurrent().Spatial)
// if reference layer is higher, hold there until an RTCP Sender Report from
// publisher is available as that is used for reference time stamp between layers.
@@ -612,21 +682,24 @@ func (f *Forwarder) getRefLayer() (int32, int32) {
return buffer.InvalidLayerSpatial, buffer.InvalidLayerSpatial
}
if f.refIsSVC {
if f.refVideoLayerMode == livekit.VideoLayer_MULTIPLE_SPATIAL_LAYERS_PER_STREAM {
return 0, currentLayerSpatial
}
return currentLayerSpatial, currentLayerSpatial
}
func (f *Forwarder) SetRefSenderReport(isSVC bool, layer int32, srData *livekit.RTCPSenderReportState) {
func (f *Forwarder) SetRefSenderReport(layer int32, srData *livekit.RTCPSenderReportState) {
f.lock.Lock()
defer f.lock.Unlock()
f.refIsSVC = isSVC
if layer >= 0 && int(layer) < len(f.refInfos) {
if layer == f.referenceLayerSpatial && f.refInfos[layer].senderReport == nil {
f.logger.Debugw("received RTCP sender report for reference layer spatial", "layer", layer)
f.logger.Debugw(
"received RTCP sender report for reference layer spatial",
"layer", layer,
"srData", rtpstats.WrappedRTCPSenderReportStateLogger{RTCPSenderReportState: srData},
)
}
f.refInfos[layer] = refInfo{srData, 0, false}
@@ -641,7 +714,7 @@ func (f *Forwarder) SetRefSenderReport(isSVC bool, layer int32, srData *livekit.
//
// It could happen like this
// 1. Normal operation: publisher sending sender reports and
// suscribers use reports from publisher to calculate and send
// subscribers use reports from publisher to calculate and send
// RTCP sender report.
// 2. Publisher pauses: there are no more reports.
// 3. When paused, subscriber can still use the publisher side sender
@@ -665,7 +738,7 @@ func (f *Forwarder) SetRefSenderReport(isSVC bool, layer int32, srData *livekit.
}
}
func (f *Forwarder) GetSenderReportParams() (int32, uint64, *livekit.RTCPSenderReportState) {
func (f *Forwarder) GetSenderReportParams() (int32, bool, uint64, *livekit.RTCPSenderReportState) {
f.lock.RLock()
defer f.lock.RUnlock()
@@ -673,10 +746,10 @@ func (f *Forwarder) GetSenderReportParams() (int32, uint64, *livekit.RTCPSenderR
if refLayer == buffer.InvalidLayerSpatial ||
f.refInfos[refLayer].senderReport == nil ||
!f.refInfos[refLayer].isTSOffsetValid {
return buffer.InvalidLayerSpatial, 0, nil
return buffer.InvalidLayerSpatial, false, 0, nil
}
return currentLayerSpatial, f.refInfos[refLayer].tsOffset, f.refInfos[refLayer].senderReport
return currentLayerSpatial, f.refVideoLayerMode == livekit.VideoLayer_MULTIPLE_SPATIAL_LAYERS_PER_STREAM, f.refInfos[refLayer].tsOffset, f.refInfos[refLayer].senderReport
}
func (f *Forwarder) isDeficientLocked() bool {
@@ -896,18 +969,24 @@ func (f *Forwarder) ProvisionalAllocatePrepare(availableLayers []int32, bitrates
f.lock.Lock()
defer f.lock.Unlock()
f.provisional = &VideoAllocationProvisional{
allocatedLayer: buffer.InvalidLayer,
muted: f.muted,
pubMuted: f.pubMuted,
maxSeenLayer: f.vls.GetMaxSeen(),
bitrates: bitrates,
maxLayer: f.vls.GetMax(),
currentLayer: f.vls.GetCurrent(),
if f.provisional == nil {
f.provisional = &VideoAllocationProvisional{}
}
f.provisional.availableLayers = make([]int32, len(availableLayers))
copy(f.provisional.availableLayers, availableLayers)
p := f.provisional
p.allocatedLayer = buffer.InvalidLayer
p.muted = f.muted
p.pubMuted = f.pubMuted
p.maxSeenLayer = f.vls.GetMaxSeen()
p.bitrates = bitrates
p.maxLayer = f.vls.GetMax()
p.currentLayer = f.vls.GetCurrent()
if cap(p.availableLayers) >= len(availableLayers) {
p.availableLayers = p.availableLayers[:len(availableLayers)]
} else {
p.availableLayers = make([]int32, len(availableLayers))
}
copy(p.availableLayers, availableLayers)
}
func (f *Forwarder) ProvisionalAllocateReset() {
@@ -1523,7 +1602,11 @@ func (f *Forwarder) updateAllocation(alloc VideoAllocation, reason string) Video
alloc.PauseReason != f.lastAllocation.PauseReason ||
alloc.TargetLayer != f.lastAllocation.TargetLayer ||
alloc.RequestLayerSpatial != f.lastAllocation.RequestLayerSpatial {
f.logger.Debugw(fmt.Sprintf("stream allocation: %s", reason), "allocation", &alloc)
f.logger.Debugw(
fmt.Sprintf("stream allocation: %s", reason),
"allocation", &alloc,
"lastAllocation", &f.lastAllocation,
)
}
f.lastAllocation = alloc
@@ -1563,18 +1646,7 @@ func (f *Forwarder) CheckSync() (bool, int32) {
f.lock.RLock()
defer f.lock.RUnlock()
locked, layer := f.vls.CheckSync()
if !locked {
return locked, layer
}
// max published layer (as seen by this forwarder) could be
// lower than max subscribed, mark de-synced if not deficient
if !f.isDeficientLocked() && f.vls.GetMax().Spatial > f.vls.GetTarget().Spatial {
return false, layer
}
return true, layer
return f.vls.CheckSync()
}
func (f *Forwarder) Restart() {
@@ -1586,11 +1658,11 @@ func (f *Forwarder) Restart() {
f.referenceLayerSpatial = buffer.InvalidLayerSpatial
f.lastReferencePayloadType = -1
for layer := 0; layer < len(f.refInfos); layer++ {
for layer := range len(f.refInfos) {
f.refInfos[layer] = refInfo{}
}
f.lastSwitchExtIncomingTS = 0
f.refIsSVC = false
f.refVideoLayerMode = livekit.VideoLayer_MODE_UNUSED
}
func (f *Forwarder) FilterRTX(nacks []uint16) (filtered []uint16, disallowedLayers [buffer.DefaultMaxLayerSpatial + 1]bool) {
@@ -1614,7 +1686,7 @@ func (f *Forwarder) FilterRTX(nacks []uint16) (filtered []uint16, disallowedLaye
if FlagFilterRTXLayers {
currentLayer := f.vls.GetCurrent()
targetLayer := f.vls.GetTarget()
for layer := int32(0); layer < buffer.DefaultMaxLayerSpatial+1; layer++ {
for layer := range buffer.DefaultMaxLayerSpatial + 1 {
if f.isDeficientLocked() && (targetLayer.Spatial < currentLayer.Spatial || layer > currentLayer.Spatial) {
disallowedLayers[layer] = true
}
@@ -1643,7 +1715,7 @@ func (f *Forwarder) GetTranslationParams(extPkt *buffer.ExtPacket, layer int32)
return TranslationParams{
shouldDrop: true,
}, ErrUnknownKind
}, errUnknownKind
}
func (f *Forwarder) getRefLayerRTPTimestamp(ts uint32, refLayer, targetLayer int32) (uint32, error) {
@@ -1651,7 +1723,7 @@ func (f *Forwarder) getRefLayerRTPTimestamp(ts uint32, refLayer, targetLayer int
return 0, fmt.Errorf("invalid layer(s), refLayer: %d, targetLayer: %d", refLayer, targetLayer)
}
if refLayer == targetLayer || f.refIsSVC {
if refLayer == targetLayer || f.refVideoLayerMode == livekit.VideoLayer_MULTIPLE_SPATIAL_LAYERS_PER_STREAM {
return ts, nil
}
@@ -1676,8 +1748,13 @@ func (f *Forwarder) getRefLayerRTPTimestamp(ts uint32, refLayer, targetLayer int
return ts + offset, nil
}
func (f *Forwarder) processSourceSwitch(extPkt *buffer.ExtPacket, layer int32) error {
func (f *Forwarder) processSourceSwitch(extPkt *buffer.ExtPacket, layer int32) (bool, error) {
starting := false
if !f.started {
if extPkt.IsOutOfOrder {
return starting, errSkipStartOnOutOfOrderPacket
}
f.started = true
f.referenceLayerSpatial = layer
f.rtpMunger.SetLastSnTs(extPkt)
@@ -1691,8 +1768,13 @@ func (f *Forwarder) processSourceSwitch(extPkt *buffer.ExtPacket, layer int32) e
"layer", layer,
"referenceLayerSpatial", f.referenceLayerSpatial,
)
return nil
starting = true
return starting, nil
} else if f.referenceLayerSpatial == buffer.InvalidLayerSpatial {
if extPkt.IsOutOfOrder {
return starting, errSkipStartOnOutOfOrderPacket
}
f.referenceLayerSpatial = layer
f.codecMunger.SetLast(extPkt)
f.logger.Debugw(
@@ -1704,6 +1786,8 @@ func (f *Forwarder) processSourceSwitch(extPkt *buffer.ExtPacket, layer int32) e
"layer", layer,
"referenceLayerSpatial", f.referenceLayerSpatial,
)
starting = true
}
logTransition := func(message string, extExpectedTS, extRefTS, extLastTS uint64, diffSeconds float64) {
@@ -1722,23 +1806,6 @@ func (f *Forwarder) processSourceSwitch(extPkt *buffer.ExtPacket, layer int32) e
"rtpStats", f.rtpStats,
)
}
// TODO-REMOVE-AFTER-DATA-COLLECTION
logTransitionInfo := func(message string, extExpectedTS, extRefTS, extLastTS uint64, diffSeconds float64) {
f.logger.Infow(
message,
"layer", layer,
"referenceLayerSpatial", f.referenceLayerSpatial,
"extExpectedTS", extExpectedTS,
"incomingTS", extPkt.Packet.Timestamp,
"extIncomingTS", extPkt.ExtTimestamp,
"extRefTS", extRefTS,
"extLastTS", extLastTS,
"diffSeconds", math.Abs(diffSeconds),
"refInfos", logger.ObjectSlice(f.refInfos[:]),
"lastSwitchExtIncomingTS", f.lastSwitchExtIncomingTS,
"rtpStats", f.rtpStats,
)
}
// Compute how much time passed between the previous forwarded packet
// and the current incoming (to be forwarded) packet and calculate
@@ -1750,8 +1817,7 @@ func (f *Forwarder) processSourceSwitch(extPkt *buffer.ExtPacket, layer int32) e
// 3. extExpectedTS -> expected timestamp of this packet calculated based on elapsed time since first packet
// Ideally, extRefTS and extExpectedTS should be very close and extLastTS should be before both of those.
// But, cases like muting/unmuting, clock vagaries, pacing, etc. make them not satisfy those conditions always.
rtpMungerState := f.rtpMunger.GetState()
extLastTS := rtpMungerState.ExtLastTimestamp
extLastTS := f.rtpMunger.GetExtLastTimestamp()
extExpectedTS := extLastTS
extRefTS := extLastTS
refTS := uint32(extRefTS)
@@ -1771,7 +1837,7 @@ func (f *Forwarder) processSourceSwitch(extPkt *buffer.ExtPacket, layer int32) e
"layer", layer,
"error", err,
)
return err
return starting, err
}
}
@@ -1818,7 +1884,6 @@ func (f *Forwarder) processSourceSwitch(extPkt *buffer.ExtPacket, layer int32) e
}
}
bigJump := false
var extNextTS uint64
if f.lastSSRC == 0 {
// If resuming (e. g. on unmute), keep next timestamp close to expected timestamp.
@@ -1844,14 +1909,12 @@ func (f *Forwarder) processSourceSwitch(extPkt *buffer.ExtPacket, layer int32) e
diffSeconds := float64(int64(extExpectedTS-extRefTS)) / float64(f.clockRate)
if diffSeconds >= 0.0 {
if f.resumeBehindThreshold > 0 && diffSeconds > f.resumeBehindThreshold {
logTransitionInfo("resume, reference too far behind", extExpectedTS, extRefTS, extLastTS, diffSeconds)
logTransition("resume, reference too far behind", extExpectedTS, extRefTS, extLastTS, diffSeconds)
extNextTS = extExpectedTS
bigJump = true
} else if diffSeconds > ResumeBehindHighThresholdSeconds {
// could be due to incoming time stamp lagging a lot, like an unpause of the track
logTransitionInfo("resume, reference very far behind", extExpectedTS, extRefTS, extLastTS, diffSeconds)
logTransition("resume, reference very far behind", extExpectedTS, extRefTS, extLastTS, diffSeconds)
extNextTS = extExpectedTS
bigJump = true
} else {
extNextTS = extRefTS
}
@@ -1872,7 +1935,7 @@ func (f *Forwarder) processSourceSwitch(extPkt *buffer.ExtPacket, layer int32) e
// (like "have waited for too long for layer switch, nothing available, switch to whatever is available" kind of condition).
logTransition("layer switch, reference too far behind", extExpectedTS, extRefTS, extLastTS, diffSeconds)
return errors.New("switch point too far behind")
return starting, errSwitchPointTooFarBehind
}
// use a nominal increase to ensure that timestamp is always moving forward
@@ -1893,54 +1956,37 @@ func (f *Forwarder) processSourceSwitch(extPkt *buffer.ExtPacket, layer int32) e
// nominal increase
extNextTS = extLastTS + 1
}
if bigJump { // TODO-REMOVE-AFTER-DATA-COLLECTION
f.logger.Infow(
"next timestamp on switch",
"switchingAt", switchingAt,
"layer", layer,
"extLastTS", extLastTS,
"lastMarker", rtpMungerState.LastMarker,
"extRefTS", extRefTS,
"dummyStartTSOffset", f.dummyStartTSOffset,
"referenceLayerSpatial", f.referenceLayerSpatial,
"extExpectedTS", extExpectedTS,
"extNextTS", extNextTS,
"tsJump", extNextTS-extLastTS,
"nextSN", rtpMungerState.ExtLastSequenceNumber+1,
"extIncomingSN", extPkt.ExtSequenceNumber,
"incomingTS", extPkt.Packet.Timestamp,
"extIncomingTS", extPkt.ExtTimestamp,
"rtpStats", f.rtpStats,
)
} else {
f.logger.Debugw(
"next timestamp on switch",
"switchingAt", switchingAt,
"layer", layer,
"extLastTS", extLastTS,
"lastMarker", rtpMungerState.LastMarker,
"extRefTS", extRefTS,
"dummyStartTSOffset", f.dummyStartTSOffset,
"referenceLayerSpatial", f.referenceLayerSpatial,
"extExpectedTS", extExpectedTS,
"extNextTS", extNextTS,
"tsJump", extNextTS-extLastTS,
"nextSN", rtpMungerState.ExtLastSequenceNumber+1,
"extIncomingSN", extPkt.ExtSequenceNumber,
"extIncomingTS", extPkt.ExtTimestamp,
"rtpStats", f.rtpStats,
)
}
f.logger.Debugw(
"next timestamp on switch",
"switchingAt", switchingAt,
"layer", layer,
"extLastTS", extLastTS,
"lastMarker", f.rtpMunger.GetState().LastMarker,
"extRefTS", extRefTS,
"dummyStartTSOffset", f.dummyStartTSOffset,
"referenceLayerSpatial", f.referenceLayerSpatial,
"extExpectedTS", extExpectedTS,
"extNextTS", extNextTS,
"tsJump", extNextTS-extLastTS,
"nextSN", f.rtpMunger.GetState().ExtLastSequenceNumber+1,
"extIncomingSN", extPkt.ExtSequenceNumber,
"extIncomingTS", extPkt.ExtTimestamp,
"rtpStats", f.rtpStats,
)
f.rtpMunger.UpdateSnTsOffsets(extPkt, 1, extNextTS-extLastTS)
f.codecMunger.UpdateOffsets(extPkt)
return nil
return starting, nil
}
// should be called with lock held
func (f *Forwarder) getTranslationParamsCommon(extPkt *buffer.ExtPacket, layer int32, tp *TranslationParams) error {
func (f *Forwarder) getTranslationParamsCommon(extPkt *buffer.ExtPacket, layer int32, tp *TranslationParams) (bool, error) {
var (
starting bool
err error
)
if f.lastSSRC != extPkt.Packet.SSRC {
if err := f.processSourceSwitch(extPkt, layer); err != nil {
if starting, err = f.processSourceSwitch(extPkt, layer); err != nil {
f.logger.Debugw(
"could not switch feed",
"error", err,
@@ -1954,7 +2000,7 @@ func (f *Forwarder) getTranslationParamsCommon(extPkt *buffer.ExtPacket, layer i
)
tp.shouldDrop = true
f.vls.Rollback()
return nil
return starting, nil
}
f.logger.Debugw(
"switching feed",
@@ -1977,29 +2023,30 @@ func (f *Forwarder) getTranslationParamsCommon(extPkt *buffer.ExtPacket, layer i
tpRTP, err := f.rtpMunger.UpdateAndGetSnTs(extPkt, tp.marker)
if err != nil {
tp.shouldDrop = true
if err == ErrPaddingOnlyPacket || err == ErrDuplicatePacket || err == ErrOutOfOrderSequenceNumberCacheMiss {
return nil
if err == errPaddingOnlyPacket || err == errDuplicatePacket || err == errOutOfOrderSequenceNumberCacheMiss {
return starting, nil
}
return err
return starting, err
}
tp.rtp = tpRTP
if len(extPkt.Packet.Payload) > 0 {
return f.translateCodecHeader(extPkt, tp)
return starting, f.translateCodecHeader(extPkt, tp)
}
return nil
return starting, nil
}
// should be called with lock held
func (f *Forwarder) getTranslationParamsAudio(extPkt *buffer.ExtPacket, layer int32) (TranslationParams, error) {
tp := TranslationParams{}
if err := f.getTranslationParamsCommon(extPkt, layer, &tp); err != nil {
starting, err := f.getTranslationParamsCommon(extPkt, layer, &tp)
tp.isStarting = starting
if err != nil {
tp.shouldDrop = true
return tp, err
}
return tp, nil
return tp, err
}
// should be called with lock held
@@ -2013,6 +2060,24 @@ func (f *Forwarder) getTranslationParamsVideo(extPkt *buffer.ExtPacket, layer in
result := f.vls.Select(extPkt, layer)
if !result.IsSelected {
if f.isDDAvailable && extPkt.DependencyDescriptor == nil {
f.logger.Infow(
"turning off dependency descriptor",
"layer", layer,
"refInfos", logger.ObjectSlice(f.refInfos[:]),
"lastSwitchExtIncomingTS", f.lastSwitchExtIncomingTS,
"currentLayer", f.vls.GetCurrent(),
"targetLayer", f.vls.GetCurrent(),
"maxLayer", f.vls.GetMax(),
)
f.isDDAvailable = false
switch f.mime {
case mime.MimeTypeVP9:
f.vls = videolayerselector.NewVP9FromOther(f.vls)
case mime.MimeTypeAV1:
f.vls = videolayerselector.NewSimulcastFromOther(f.vls)
}
}
tp.shouldDrop = true
if f.started && result.IsRelevant {
// call to update highest incoming sequence number and other internal structures
@@ -2029,7 +2094,8 @@ func (f *Forwarder) getTranslationParamsVideo(extPkt *buffer.ExtPacket, layer in
tp.ddBytes = result.DependencyDescriptorExtension
tp.marker = result.RTPMarker
err := f.getTranslationParamsCommon(extPkt, layer, &tp)
starting, err := f.getTranslationParamsCommon(extPkt, layer, &tp)
tp.isStarting = starting
if tp.shouldDrop {
return tp, err
}
@@ -2117,7 +2183,7 @@ func (f *Forwarder) maybeStart() {
)
}
func (f *Forwarder) GetSnTsForPadding(num int, forceMarker bool) ([]SnTs, error) {
func (f *Forwarder) GetSnTsForPadding(num int, frameRate uint32, forceMarker bool) ([]SnTs, error) {
f.lock.Lock()
defer f.lock.Unlock()
@@ -2131,7 +2197,13 @@ func (f *Forwarder) GetSnTsForPadding(num int, forceMarker bool) ([]SnTs, error)
if !f.vls.GetTarget().IsValid() {
forceMarker = true
}
return f.rtpMunger.UpdateAndGetPaddingSnTs(num, 0, 0, forceMarker, 0)
return f.rtpMunger.UpdateAndGetPaddingSnTs(
num,
f.clockRate,
frameRate,
forceMarker,
f.rtpMunger.GetExtLastTimestamp(),
)
}
func (f *Forwarder) GetSnTsForBlankFrames(frameRate uint32, numPackets int) ([]SnTs, bool, error) {
@@ -2145,7 +2217,7 @@ func (f *Forwarder) GetSnTsForBlankFrames(frameRate uint32, numPackets int) ([]S
numPackets++
}
extLastTS := f.rtpMunger.GetState().ExtLastTimestamp
extLastTS := f.rtpMunger.GetExtLastTimestamp()
extExpectedTS := extLastTS
if f.rtpStats != nil {
tsExt, err := f.rtpStats.GetExpectedRTPTimestamp(mono.Now())
@@ -2156,7 +2228,13 @@ func (f *Forwarder) GetSnTsForBlankFrames(frameRate uint32, numPackets int) ([]S
if int64(extExpectedTS-extLastTS) <= 0 {
extExpectedTS = extLastTS + 1
}
snts, err := f.rtpMunger.UpdateAndGetPaddingSnTs(numPackets, f.clockRate, frameRate, frameEndNeeded, extExpectedTS)
snts, err := f.rtpMunger.UpdateAndGetPaddingSnTs(
numPackets,
f.clockRate,
frameRate,
frameEndNeeded,
extExpectedTS,
)
return snts, frameEndNeeded, err
}
@@ -2167,7 +2245,7 @@ func (f *Forwarder) GetPadding(frameEndNeeded bool) ([]byte, error) {
return f.codecMunger.UpdateAndGetPadding(!frameEndNeeded)
}
func (f *Forwarder) RTPMungerDebugInfo() map[string]interface{} {
func (f *Forwarder) RTPMungerDebugInfo() map[string]any {
f.lock.RLock()
defer f.lock.RUnlock()
pkg/sfu/forwarder_test.go
+103 -30
View File
@@ -20,6 +20,8 @@ import (
"github.com/pion/webrtc/v4"
"github.com/stretchr/testify/require"
"github.com/livekit/mediatransportutil/pkg/codec"
"github.com/livekit/protocol/livekit"
"github.com/livekit/protocol/logger"
"github.com/livekit/livekit-server/pkg/sfu/buffer"
@@ -32,8 +34,14 @@ func disable(f *Forwarder) {
}
func newForwarder(codec webrtc.RTPCodecCapability, kind webrtc.RTPCodecType) *Forwarder {
f := NewForwarder(kind, logger.GetLogger(), true, nil)
f.DetermineCodec(codec, nil)
f := NewForwarder(
kind,
logger.GetLogger(),
true, // skipReferenceTS
true, // disableOpportunisticAllocation
nil,
)
f.DetermineCodec(codec, nil, livekit.VideoLayer_MODE_UNUSED)
return f
}
@@ -463,6 +471,9 @@ func TestForwarderProvisionalAllocate(t *testing.T) {
f.SetMaxPublishedLayer(buffer.DefaultMaxLayerSpatial)
f.SetMaxTemporalLayerSeen(buffer.DefaultMaxLayerTemporal)
// Reset to invalid layers for testing allocation from scratch
disable(f)
bitrates := Bitrates{
{1, 2, 3, 4},
{5, 6, 7, 8},
@@ -681,6 +692,9 @@ func TestForwarderProvisionalAllocateMute(t *testing.T) {
f.SetMaxSpatialLayer(buffer.DefaultMaxLayerSpatial)
f.SetMaxTemporalLayer(buffer.DefaultMaxLayerTemporal)
// Reset to invalid layers for testing muted state
disable(f)
bitrates := Bitrates{
{1, 2, 3, 4},
{5, 6, 7, 8},
@@ -722,6 +736,9 @@ func TestForwarderProvisionalAllocateGetCooperativeTransition(t *testing.T) {
f.SetMaxPublishedLayer(buffer.DefaultMaxLayerSpatial)
f.SetMaxTemporalLayerSeen(buffer.DefaultMaxLayerTemporal)
// Reset to invalid layers for testing cooperative transition from scratch
disable(f)
availableLayers := []int32{0, 1, 2}
bitrates := Bitrates{
{1, 2, 3, 4},
@@ -1272,22 +1289,42 @@ func TestForwarderGetTranslationParamsAudio(t *testing.T) {
f := newForwarder(testutils.TestOpusCodec, webrtc.RTPCodecTypeAudio)
params := &testutils.TestExtPacketParams{
SequenceNumber: 23332,
Timestamp: 0xabcdef,
SSRC: 0x12345678,
PayloadSize: 20,
IsOutOfOrder: true,
}
extPkt, _ := testutils.GetTestExtPacket(params)
// should not start on an out-of-order packet
expectedTP := TranslationParams{
shouldDrop: true,
}
actualTP, err := f.GetTranslationParams(extPkt, 0)
require.NoError(t, err)
require.Equal(t, expectedTP, actualTP)
require.False(t, f.started)
require.Zero(t, f.lastSSRC)
params = &testutils.TestExtPacketParams{
SequenceNumber: 23333,
Timestamp: 0xabcdef,
SSRC: 0x12345678,
PayloadSize: 20,
}
extPkt, _ := testutils.GetTestExtPacket(params)
extPkt, _ = testutils.GetTestExtPacket(params)
// should lock onto the first packet
expectedTP := TranslationParams{
// should lock onto the first in-order packet
expectedTP = TranslationParams{
isStarting: true,
rtp: TranslationParamsRTP{
snOrdering: SequenceNumberOrderingContiguous,
extSequenceNumber: 23333,
extTimestamp: 0xabcdef,
},
}
actualTP, err := f.GetTranslationParams(extPkt, 0)
actualTP, err = f.GetTranslationParams(extPkt, 0)
require.NoError(t, err)
require.Equal(t, expectedTP, actualTP)
require.True(t, f.started)
@@ -1436,13 +1473,14 @@ func TestForwarderGetTranslationParamsVideo(t *testing.T) {
f := newForwarder(testutils.TestVP8Codec, webrtc.RTPCodecTypeVideo)
params := &testutils.TestExtPacketParams{
SequenceNumber: 23333,
SequenceNumber: 23332,
Timestamp: 0xabcdef,
SSRC: 0x12345678,
PayloadSize: 20,
SetMarker: true,
Marker: true,
IsOutOfOrder: true,
}
vp8 := &buffer.VP8{
vp8 := &codec.VP8{
FirstByte: 25,
I: true,
M: true,
@@ -1459,13 +1497,47 @@ func TestForwarderGetTranslationParamsVideo(t *testing.T) {
}
extPkt, _ := testutils.GetTestExtPacketVP8(params, vp8)
// no target layers, should drop
// should not start on an out-of-order packet
expectedTP := TranslationParams{
shouldDrop: true,
}
actualTP, err := f.GetTranslationParams(extPkt, 0)
require.NoError(t, err)
require.Equal(t, expectedTP, actualTP)
require.False(t, f.started)
require.Zero(t, f.lastSSRC)
params = &testutils.TestExtPacketParams{
SequenceNumber: 23333,
Timestamp: 0xabcdef,
SSRC: 0x12345678,
PayloadSize: 20,
Marker: true,
}
vp8 = &codec.VP8{
FirstByte: 25,
I: true,
M: true,
PictureID: 13467,
L: true,
TL0PICIDX: 233,
T: true,
TID: 0,
Y: true,
K: true,
KEYIDX: 23,
HeaderSize: 6,
IsKeyFrame: false,
}
extPkt, _ = testutils.GetTestExtPacketVP8(params, vp8)
// no target layers, should drop
expectedTP = TranslationParams{
shouldDrop: true,
}
actualTP, err = f.GetTranslationParams(extPkt, 0)
require.NoError(t, err)
require.Equal(t, expectedTP, actualTP)
// although target layer matches, not a key frame, so should drop
f.vls.SetTarget(buffer.VideoLayer{
@@ -1480,7 +1552,7 @@ func TestForwarderGetTranslationParamsVideo(t *testing.T) {
require.Equal(t, expectedTP, actualTP)
// should lock onto packet (key frame)
vp8 = &buffer.VP8{
vp8 = &codec.VP8{
FirstByte: 25,
I: true,
M: true,
@@ -1496,7 +1568,7 @@ func TestForwarderGetTranslationParamsVideo(t *testing.T) {
IsKeyFrame: true,
}
extPkt, _ = testutils.GetTestExtPacketVP8(params, vp8)
expectedVP8 := &buffer.VP8{
expectedVP8 := &codec.VP8{
FirstByte: 25,
I: true,
M: true,
@@ -1511,8 +1583,9 @@ func TestForwarderGetTranslationParamsVideo(t *testing.T) {
HeaderSize: 6,
IsKeyFrame: true,
}
marshalledVP8, err := expectedVP8.Marshal()
marshalledVP8, _ := expectedVP8.Marshal()
expectedTP = TranslationParams{
isStarting: true,
isSwitching: true,
isResuming: true,
rtp: TranslationParamsRTP{
@@ -1576,7 +1649,7 @@ func TestForwarderGetTranslationParamsVideo(t *testing.T) {
PayloadSize: 20,
}
extPkt, _ = testutils.GetTestExtPacketVP8(params, vp8)
expectedVP8 = &buffer.VP8{
expectedVP8 = &codec.VP8{
FirstByte: 25,
I: true,
M: true,
@@ -1613,7 +1686,7 @@ func TestForwarderGetTranslationParamsVideo(t *testing.T) {
SSRC: 0x12345678,
PayloadSize: 20,
}
vp8 = &buffer.VP8{
vp8 = &codec.VP8{
FirstByte: 25,
S: true,
I: true,
@@ -1630,7 +1703,7 @@ func TestForwarderGetTranslationParamsVideo(t *testing.T) {
IsKeyFrame: true,
}
extPkt, _ = testutils.GetTestExtPacketVP8(params, vp8)
expectedVP8 = &buffer.VP8{
expectedVP8 = &codec.VP8{
FirstByte: 25,
I: true,
M: true,
@@ -1667,7 +1740,7 @@ func TestForwarderGetTranslationParamsVideo(t *testing.T) {
SSRC: 0x12345678,
PayloadSize: 20,
}
vp8 = &buffer.VP8{
vp8 = &codec.VP8{
FirstByte: 25,
I: true,
M: true,
@@ -1702,7 +1775,7 @@ func TestForwarderGetTranslationParamsVideo(t *testing.T) {
SSRC: 0x12345678,
PayloadSize: 20,
}
vp8 = &buffer.VP8{
vp8 = &codec.VP8{
FirstByte: 25,
I: true,
M: true,
@@ -1718,7 +1791,7 @@ func TestForwarderGetTranslationParamsVideo(t *testing.T) {
IsKeyFrame: false,
}
extPkt, _ = testutils.GetTestExtPacketVP8(params, vp8)
expectedVP8 = &buffer.VP8{
expectedVP8 = &codec.VP8{
FirstByte: 25,
I: true,
M: true,
@@ -1799,7 +1872,7 @@ func TestForwarderGetTranslationParamsVideo(t *testing.T) {
SSRC: 0x87654321,
PayloadSize: 20,
}
vp8 = &buffer.VP8{
vp8 = &codec.VP8{
FirstByte: 25,
I: true,
M: false,
@@ -1816,7 +1889,7 @@ func TestForwarderGetTranslationParamsVideo(t *testing.T) {
}
extPkt, _ = testutils.GetTestExtPacketVP8(params, vp8)
expectedVP8 = &buffer.VP8{
expectedVP8 = &codec.VP8{
FirstByte: 25,
I: true,
M: true,
@@ -1858,7 +1931,7 @@ func TestForwarderGetSnTsForPadding(t *testing.T) {
SSRC: 0x12345678,
PayloadSize: 20,
}
vp8 := &buffer.VP8{
vp8 := &codec.VP8{
FirstByte: 25,
I: true,
M: true,
@@ -1888,14 +1961,14 @@ func TestForwarderGetSnTsForPadding(t *testing.T) {
disable(f)
// should get back frame end needed as the last packet did not have RTP marker set
snts, err := f.GetSnTsForPadding(5, false)
snts, err := f.GetSnTsForPadding(5, 0, false)
require.NoError(t, err)
numPadding := 5
clockRate := uint32(0)
frameRate := uint32(5)
var sntsExpected = make([]SnTs, numPadding)
for i := 0; i < numPadding; i++ {
for i := range numPadding {
sntsExpected[i] = SnTs{
extSequenceNumber: 23333 + uint64(i) + 1,
extTimestamp: 0xabcdef + (uint64(i)*uint64(clockRate))/uint64(frameRate),
@@ -1904,10 +1977,10 @@ func TestForwarderGetSnTsForPadding(t *testing.T) {
require.Equal(t, sntsExpected, snts)
// now that there is a marker, timestamp should jump on first padding when asked again
snts, err = f.GetSnTsForPadding(numPadding, false)
snts, err = f.GetSnTsForPadding(numPadding, 0, false)
require.NoError(t, err)
for i := 0; i < numPadding; i++ {
for i := range numPadding {
sntsExpected[i] = SnTs{
extSequenceNumber: 23338 + uint64(i) + 1,
extTimestamp: 0xabcdef + (uint64(i+1)*uint64(clockRate))/uint64(frameRate),
@@ -1925,7 +1998,7 @@ func TestForwarderGetSnTsForBlankFrames(t *testing.T) {
SSRC: 0x12345678,
PayloadSize: 20,
}
vp8 := &buffer.VP8{
vp8 := &codec.VP8{
FirstByte: 25,
I: true,
M: true,
@@ -2002,7 +2075,7 @@ func TestForwarderGetPaddingVP8(t *testing.T) {
SSRC: 0x12345678,
PayloadSize: 20,
}
vp8 := &buffer.VP8{
vp8 := &codec.VP8{
FirstByte: 25,
I: true,
M: true,
@@ -2029,7 +2102,7 @@ func TestForwarderGetPaddingVP8(t *testing.T) {
_, _ = f.GetTranslationParams(extPkt, 0)
// getting padding with frame end needed, should repeat the last picture id
expectedVP8 := buffer.VP8{
expectedVP8 := codec.VP8{
FirstByte: 16,
I: true,
M: true,
@@ -2051,7 +2124,7 @@ func TestForwarderGetPaddingVP8(t *testing.T) {
require.Equal(t, marshalledVP8, buf)
// getting padding with no frame end needed, should get next picture id
expectedVP8 = buffer.VP8{
expectedVP8 = codec.VP8{
FirstByte: 16,
I: true,
M: true,
pkg/sfu/forwardstats.go
+54 -32
View File
@@ -1,27 +1,33 @@
package sfu
import (
"fmt"
"sync"
"time"
"go.uber.org/atomic"
"github.com/livekit/livekit-server/pkg/telemetry/prometheus"
"github.com/livekit/protocol/logger"
"github.com/livekit/protocol/utils"
"github.com/livekit/protocol/utils/mono"
)
const (
cHighForwardingLatency = 20 * time.Millisecond
cSkewFactor = 10
)
type ForwardStats struct {
lock sync.Mutex
lastLeftNano atomic.Int64
latency *utils.LatencyAggregate
lowest int64
highest int64
lastUpdateAt int64
closeCh chan struct{}
}
func NewForwardStats(latencyUpdateInterval, reportInterval, latencyWindowLength time.Duration) *ForwardStats {
s := &ForwardStats{
latency: utils.NewLatencyAggregate(latencyUpdateInterval, latencyWindowLength),
lowest: time.Second.Nanoseconds(),
closeCh: make(chan struct{}),
}
@@ -29,43 +35,58 @@ func NewForwardStats(latencyUpdateInterval, reportInterval, latencyWindowLength
return s
}
func (s *ForwardStats) Update(arrival, left int64) {
func (s *ForwardStats) Update(arrival, left int64) (int64, bool) {
transit := left - arrival
// ignore if transit is too large or negative, this could happen if system time is adjusted
if transit < 0 || time.Duration(transit) > 5*time.Second {
return
}
lastLeftNano := s.lastLeftNano.Load()
if left < lastLeftNano || !s.lastLeftNano.CompareAndSwap(lastLeftNano, left) {
return
}
isHighForwardingLatency := time.Duration(transit) > cHighForwardingLatency
s.lock.Lock()
defer s.lock.Unlock()
s.latency.Update(time.Duration(arrival), float64(transit))
s.lowest = min(transit, s.lowest)
s.highest = max(transit, s.highest)
s.lastUpdateAt = arrival
s.lock.Unlock()
prometheus.RecordForwardLatencySample(transit)
return transit, isHighForwardingLatency
}
func (s *ForwardStats) GetStats() (latency, jitter time.Duration) {
func (s *ForwardStats) GetStats(shortDuration time.Duration) (time.Duration, time.Duration) {
s.lock.Lock()
w := s.latency.Summarize()
// a dummy sample to flush the pipe to current time
now := mono.UnixNano()
if (now - s.lastUpdateAt) > shortDuration.Nanoseconds() {
s.latency.Update(time.Duration(now), 0)
}
wLong := s.latency.Summarize()
lowest := s.lowest
s.lowest = time.Second.Nanoseconds()
highest := s.highest
s.highest = 0
s.lock.Unlock()
latency, jitter = time.Duration(w.Mean()), time.Duration(w.StdDev())
// TODO: remove this check after debugging unexpected jitter issue
if jitter > 10*time.Second {
logger.Infow("unexpected forward jitter",
"jitter", jitter,
"stats", fmt.Sprintf("count %.2f, mean %.2f, stdDev %.2f", w.Count(), w.Mean(), w.StdDev()),
latencyLong, jitterLong := time.Duration(wLong.Mean()), time.Duration(wLong.StdDev())
if jitterLong > latencyLong*cSkewFactor {
logger.Infow(
"high jitter in forwarding path",
"lowest", time.Duration(lowest),
"highest", time.Duration(highest),
"countLong", wLong.Count(),
"latencyLong", latencyLong,
"jitterLong", jitterLong,
)
}
return
return latencyLong, jitterLong
}
func (s *ForwardStats) GetLastStats(duration time.Duration) (latency, jitter time.Duration) {
func (s *ForwardStats) GetShortStats(shortDuration time.Duration) (time.Duration, time.Duration) {
s.lock.Lock()
defer s.lock.Unlock()
w := s.latency.SummarizeLast(duration)
return time.Duration(w.Mean()), time.Duration(w.StdDev())
wShort := s.latency.SummarizeLast(shortDuration)
s.lock.Unlock()
return time.Duration(wShort.Mean()), time.Duration(wShort.StdDev())
}
func (s *ForwardStats) Stop() {
@@ -75,15 +96,16 @@ func (s *ForwardStats) Stop() {
func (s *ForwardStats) report(reportInterval time.Duration) {
ticker := time.NewTicker(reportInterval)
defer ticker.Stop()
for {
select {
case <-s.closeCh:
return
case <-ticker.C:
latency, jitter := s.GetLastStats(reportInterval)
latencySlow, jitterSlow := s.GetStats()
prometheus.RecordForwardJitter(uint32(jitter/time.Millisecond), uint32(jitterSlow/time.Millisecond))
prometheus.RecordForwardLatency(uint32(latency/time.Millisecond), uint32(latencySlow/time.Millisecond))
latencyLong, jitterLong := s.GetStats(reportInterval)
prometheus.RecordForwardJitter(uint32(jitterLong.Nanoseconds()))
prometheus.RecordForwardLatency(uint32(latencyLong.Nanoseconds()))
}
}
}
pkg/sfu/interceptor/rtx.go
+22 -6
View File
@@ -39,13 +39,17 @@ type streamInfo struct {
type RTXInfoExtractorFactory struct {
onStreamFound func(*interceptor.StreamInfo)
onRTXPairFound func(repair, base uint32)
onRTXPairFound func(repair, base uint32, rsid string)
lock sync.Mutex
streams map[uint32]streamInfo
logger logger.Logger
}
func NewRTXInfoExtractorFactory(onStreamFound func(*interceptor.StreamInfo), onRTXPairFound func(repair, base uint32), logger logger.Logger) *RTXInfoExtractorFactory {
func NewRTXInfoExtractorFactory(
onStreamFound func(*interceptor.StreamInfo),
onRTXPairFound func(repair, base uint32, rsid string),
logger logger.Logger,
) *RTXInfoExtractorFactory {
return &RTXInfoExtractorFactory{
onStreamFound: onStreamFound,
onRTXPairFound: onRTXPairFound,
@@ -61,16 +65,23 @@ func (f *RTXInfoExtractorFactory) NewInterceptor(id string) (interceptor.Interce
}, nil
}
func (f *RTXInfoExtractorFactory) setStreamInfo(ssrc uint32, mid, rid, rsid string) {
func (f *RTXInfoExtractorFactory) SetStreamInfo(ssrc uint32, mid, rid, rsid string) {
var repairSsrc, baseSsrc uint32
var repairSid string
f.lock.Lock()
if mid == "" || (rid == "" && rsid == "") {
f.lock.Unlock()
return
}
if rsid != "" {
// repair stream found, find base stream
for base, info := range f.streams {
if info.mid == mid && info.rid == rsid {
repairSsrc = ssrc
baseSsrc = base
repairSid = rsid
delete(f.streams, base)
break
}
@@ -81,6 +92,7 @@ func (f *RTXInfoExtractorFactory) setStreamInfo(ssrc uint32, mid, rid, rsid stri
if info.mid == mid && info.rsid == rid {
repairSsrc = repair
baseSsrc = ssrc
repairSid = rid
delete(f.streams, repair)
break
}
@@ -99,10 +111,12 @@ func (f *RTXInfoExtractorFactory) setStreamInfo(ssrc uint32, mid, rid, rsid stri
f.lock.Unlock()
if repairSsrc != 0 && baseSsrc != 0 {
f.onRTXPairFound(repairSsrc, baseSsrc)
f.onRTXPairFound(repairSsrc, baseSsrc, repairSid)
}
}
// ------------------------------------------
type RTXInfoExtractor struct {
interceptor.NoOp
@@ -131,6 +145,8 @@ func (u *RTXInfoExtractor) BindRemoteStream(info *interceptor.StreamInfo, reader
}
}
// ------------------------------------------
type rtxInfoReader struct {
tryTimes int
reader interceptor.RTPReader
@@ -171,10 +187,10 @@ func (r *rtxInfoReader) Read(b []byte, a interceptor.Attributes) (int, intercept
if mid != "" && (rid != "" || rsid != "") {
r.logger.Debugw("stream found", "mid", mid, "rid", rid, "rsid", rsid, "ssrc", header.SSRC)
r.tryTimes = -1
go r.factory.setStreamInfo(header.SSRC, mid, rid, rsid)
go r.factory.SetStreamInfo(header.SSRC, mid, rid, rsid)
} else {
// ignore padding only packet for probe count
if !(header.Padding && n-header.MarshalSize()-int(b[n-1]) == 0) {
if !header.Padding || n-header.MarshalSize()-int(b[n-1]) != 0 {
r.tryTimes--
}
}
pkg/sfu/interceptor/unhandlesimulcast.go
+183
View File
@@ -0,0 +1,183 @@
// Copyright 2023 LiveKit, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package interceptor
import (
"github.com/pion/interceptor"
"github.com/pion/rtp"
"github.com/pion/sdp/v3"
"github.com/pion/webrtc/v4"
"go.uber.org/zap/zapcore"
"github.com/livekit/livekit-server/pkg/sfu/utils"
"github.com/livekit/protocol/logger"
)
const (
simulcastProbeCount = 10
)
type SimulcastTrackInfo struct {
Mid string
StreamID string
RepairSSRC uint32 // set only when `IsRepairStream: false`, i. e. RTX SSRC for the primary stream
IsRepairStream bool
}
func (s *SimulcastTrackInfo) MarshalLogObject(e zapcore.ObjectEncoder) error {
e.AddString("Mid", s.Mid)
e.AddString("StreamID", s.StreamID)
e.AddUint32("RepairSSRC", s.RepairSSRC)
e.AddBool("IsRepairStream", s.IsRepairStream)
return nil
}
// -------------------------------------------------------------------
type UnhandleSimulcastOption func(u *UnhandleSimulcastInterceptor) error
func UnhandleSimulcastTracks(logger logger.Logger, tracks map[uint32]SimulcastTrackInfo) UnhandleSimulcastOption {
return func(u *UnhandleSimulcastInterceptor) error {
u.logger = logger
u.simTracks = tracks
return nil
}
}
type UnhandleSimulcastInterceptorFactory struct {
opts []UnhandleSimulcastOption
}
func (f *UnhandleSimulcastInterceptorFactory) NewInterceptor(id string) (interceptor.Interceptor, error) {
i := &UnhandleSimulcastInterceptor{simTracks: map[uint32]SimulcastTrackInfo{}}
for _, o := range f.opts {
if err := o(i); err != nil {
return nil, err
}
}
return i, nil
}
func NewUnhandleSimulcastInterceptorFactory(opts ...UnhandleSimulcastOption) (*UnhandleSimulcastInterceptorFactory, error) {
return &UnhandleSimulcastInterceptorFactory{opts: opts}, nil
}
type unhandleSimulcastRTPReader struct {
SimulcastTrackInfo
logger logger.Logger
tryTimes int
reader interceptor.RTPReader
midExtID uint8
ridExtID uint8
rsidExtID uint8
}
func (u *unhandleSimulcastRTPReader) Read(b []byte, a interceptor.Attributes) (int, interceptor.Attributes, error) {
n, a, err := u.reader.Read(b, a)
if u.tryTimes < 0 || err != nil {
return n, a, err
}
header := rtp.Header{}
hsize, err := header.Unmarshal(b[:n])
if err != nil {
return n, a, nil
}
var mid, rid, rsid string
if payload := header.GetExtension(u.midExtID); payload != nil {
mid = string(payload)
}
if payload := header.GetExtension(u.ridExtID); payload != nil {
rid = string(payload)
}
if payload := header.GetExtension(u.rsidExtID); payload != nil {
rid = string(payload)
}
if mid != "" && (rid != "" || rsid != "") {
u.logger.Debugw(
"unhandle stream found",
"mid", mid,
"rid", rid,
"rsid", rsid,
"ssrc", header.SSRC,
"simulcastTrackInfo", u.SimulcastTrackInfo,
)
u.tryTimes = -1
return n, a, nil
} else {
// ignore padding only packet for probe count
if !header.Padding || n-header.MarshalSize()-int(b[n-1]) != 0 {
u.tryTimes--
}
}
if mid == "" {
header.SetExtension(u.midExtID, []byte(u.Mid))
}
if rid == "" && !u.IsRepairStream {
header.SetExtension(u.ridExtID, []byte(u.StreamID))
}
if rsid == "" && u.IsRepairStream {
header.SetExtension(u.rsidExtID, []byte(u.StreamID))
}
hsize2 := header.MarshalSize()
if hsize2-hsize+n > len(b) { // no enough buf to set extension
return n, a, nil
}
copy(b[hsize2:], b[hsize:n])
header.MarshalTo(b)
u.logger.Debugw(
"unhandle stream injecting",
"mid", mid,
"rid", rid,
"rsid", rsid,
"ssrc", header.SSRC,
"simulcastTrackInfo", u.SimulcastTrackInfo,
)
return hsize2 - hsize + n, a, nil
}
type UnhandleSimulcastInterceptor struct {
interceptor.NoOp
logger logger.Logger
simTracks map[uint32]SimulcastTrackInfo
}
func (u *UnhandleSimulcastInterceptor) BindRemoteStream(info *interceptor.StreamInfo, reader interceptor.RTPReader) interceptor.RTPReader {
if t, ok := u.simTracks[info.SSRC]; ok {
midExtensionID := utils.GetHeaderExtensionID(info.RTPHeaderExtensions, webrtc.RTPHeaderExtensionCapability{URI: sdp.SDESMidURI})
streamIDExtensionID := utils.GetHeaderExtensionID(info.RTPHeaderExtensions, webrtc.RTPHeaderExtensionCapability{URI: sdp.SDESRTPStreamIDURI})
repairStreamIDExtensionID := utils.GetHeaderExtensionID(info.RTPHeaderExtensions, webrtc.RTPHeaderExtensionCapability{URI: sdp.SDESRepairRTPStreamIDURI})
if midExtensionID == 0 || streamIDExtensionID == 0 || repairStreamIDExtensionID == 0 {
return reader
}
return &unhandleSimulcastRTPReader{
SimulcastTrackInfo: t,
logger: u.logger,
reader: reader,
tryTimes: simulcastProbeCount,
midExtID: uint8(midExtensionID),
ridExtID: uint8(streamIDExtensionID),
rsidExtID: uint8(repairStreamIDExtensionID),
}
}
return reader
}
pkg/sfu/mime/mimetype.go
-281
View File
@@ -1,281 +0,0 @@
// Copyright 2023 LiveKit, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package mime
import (
"strings"
"github.com/pion/webrtc/v4"
)
const (
MimeTypePrefixAudio = "audio/"
MimeTypePrefixVideo = "video/"
)
type MimeTypeCodec int
const (
MimeTypeCodecUnknown MimeTypeCodec = iota
MimeTypeCodecH264
MimeTypeCodecH265
MimeTypeCodecOpus
MimeTypeCodecRED
MimeTypeCodecVP8
MimeTypeCodecVP9
MimeTypeCodecAV1
MimeTypeCodecG722
MimeTypeCodecPCMU
MimeTypeCodecPCMA
MimeTypeCodecRTX
MimeTypeCodecFlexFEC
MimeTypeCodecULPFEC
)
func (m MimeTypeCodec) String() string {
switch m {
case MimeTypeCodecUnknown:
return "MimeTypeCodecUnknown"
case MimeTypeCodecH264:
return "H264"
case MimeTypeCodecH265:
return "H265"
case MimeTypeCodecOpus:
return "opus"
case MimeTypeCodecRED:
return "red"
case MimeTypeCodecVP8:
return "VP8"
case MimeTypeCodecVP9:
return "VP9"
case MimeTypeCodecAV1:
return "AV1"
case MimeTypeCodecG722:
return "G722"
case MimeTypeCodecPCMU:
return "PCMU"
case MimeTypeCodecPCMA:
return "PCMA"
case MimeTypeCodecRTX:
return "rtx"
case MimeTypeCodecFlexFEC:
return "flexfec"
case MimeTypeCodecULPFEC:
return "ulpfec"
}
return "MimeTypeCodecUnknown"
}
func NormalizeMimeTypeCodec(codec string) MimeTypeCodec {
switch {
case strings.EqualFold(codec, "h264"):
return MimeTypeCodecH264
case strings.EqualFold(codec, "h265"):
return MimeTypeCodecH265
case strings.EqualFold(codec, "opus"):
return MimeTypeCodecOpus
case strings.EqualFold(codec, "red"):
return MimeTypeCodecRED
case strings.EqualFold(codec, "vp8"):
return MimeTypeCodecVP8
case strings.EqualFold(codec, "vp9"):
return MimeTypeCodecVP9
case strings.EqualFold(codec, "av1"):
return MimeTypeCodecAV1
case strings.EqualFold(codec, "g722"):
return MimeTypeCodecG722
case strings.EqualFold(codec, "pcmu"):
return MimeTypeCodecPCMU
case strings.EqualFold(codec, "pcma"):
return MimeTypeCodecPCMA
case strings.EqualFold(codec, "rtx"):
return MimeTypeCodecRTX
case strings.EqualFold(codec, "flexfec"):
return MimeTypeCodecFlexFEC
case strings.EqualFold(codec, "ulpfec"):
return MimeTypeCodecULPFEC
}
return MimeTypeCodecUnknown
}
func GetMimeTypeCodec(mime string) MimeTypeCodec {
i := strings.IndexByte(mime, '/')
if i == -1 {
return MimeTypeCodecUnknown
}
return NormalizeMimeTypeCodec(mime[i+1:])
}
func IsMimeTypeCodecStringOpus(codec string) bool {
return NormalizeMimeTypeCodec(codec) == MimeTypeCodecOpus
}
func IsMimeTypeCodecStringRED(codec string) bool {
return NormalizeMimeTypeCodec(codec) == MimeTypeCodecRED
}
func IsMimeTypeCodecStringH264(codec string) bool {
return NormalizeMimeTypeCodec(codec) == MimeTypeCodecH264
}
type MimeType int
const (
MimeTypeUnknown MimeType = iota
MimeTypeH264
MimeTypeH265
MimeTypeOpus
MimeTypeRED
MimeTypeVP8
MimeTypeVP9
MimeTypeAV1
MimeTypeG722
MimeTypePCMU
MimeTypePCMA
MimeTypeRTX
MimeTypeFlexFEC
MimeTypeULPFEC
)
func (m MimeType) String() string {
switch m {
case MimeTypeUnknown:
return "MimeTypeUnknown"
case MimeTypeH264:
return webrtc.MimeTypeH264
case MimeTypeH265:
return webrtc.MimeTypeH265
case MimeTypeOpus:
return webrtc.MimeTypeOpus
case MimeTypeRED:
return "audio/red"
case MimeTypeVP8:
return webrtc.MimeTypeVP8
case MimeTypeVP9:
return webrtc.MimeTypeVP9
case MimeTypeAV1:
return webrtc.MimeTypeAV1
case MimeTypeG722:
return webrtc.MimeTypeG722
case MimeTypePCMU:
return webrtc.MimeTypePCMU
case MimeTypePCMA:
return webrtc.MimeTypePCMA
case MimeTypeRTX:
return webrtc.MimeTypeRTX
case MimeTypeFlexFEC:
return webrtc.MimeTypeFlexFEC
case MimeTypeULPFEC:
return "video/ulpfec"
}
return "MimeTypeUnknown"
}
func NormalizeMimeType(mime string) MimeType {
switch {
case strings.EqualFold(mime, webrtc.MimeTypeH264):
return MimeTypeH264
case strings.EqualFold(mime, webrtc.MimeTypeH265):
return MimeTypeH265
case strings.EqualFold(mime, webrtc.MimeTypeOpus):
return MimeTypeOpus
case strings.EqualFold(mime, "audio/red"):
return MimeTypeRED
case strings.EqualFold(mime, webrtc.MimeTypeVP8):
return MimeTypeVP8
case strings.EqualFold(mime, webrtc.MimeTypeVP9):
return MimeTypeVP9
case strings.EqualFold(mime, webrtc.MimeTypeAV1):
return MimeTypeAV1
case strings.EqualFold(mime, webrtc.MimeTypeG722):
return MimeTypeG722
case strings.EqualFold(mime, webrtc.MimeTypePCMU):
return MimeTypePCMU
case strings.EqualFold(mime, webrtc.MimeTypePCMA):
return MimeTypePCMA
case strings.EqualFold(mime, webrtc.MimeTypeRTX):
return MimeTypeRTX
case strings.EqualFold(mime, webrtc.MimeTypeFlexFEC):
return MimeTypeFlexFEC
case strings.EqualFold(mime, "video/ulpfec"):
return MimeTypeULPFEC
}
return MimeTypeUnknown
}
func IsMimeTypeStringEqual(mime1 string, mime2 string) bool {
return NormalizeMimeType(mime1) == NormalizeMimeType(mime2)
}
func IsMimeTypeStringAudio(mime string) bool {
return strings.HasPrefix(mime, MimeTypePrefixAudio)
}
func IsMimeTypeAudio(mimeType MimeType) bool {
return strings.HasPrefix(mimeType.String(), MimeTypePrefixAudio)
}
func IsMimeTypeStringVideo(mime string) bool {
return strings.HasPrefix(mime, MimeTypePrefixVideo)
}
func IsMimeTypeVideo(mimeType MimeType) bool {
return strings.HasPrefix(mimeType.String(), MimeTypePrefixVideo)
}
// SVC-TODO: Have to use more conditions to differentiate between
// SVC-TODO: SVC and non-SVC (could be single layer or simulcast).
// SVC-TODO: May only need to differentiate between simulcast and non-simulcast
// SVC-TODO: i. e. may be possible to treat single layer as SVC to get proper/intended functionality.
func IsMimeTypeSVC(mimeType MimeType) bool {
switch mimeType {
case MimeTypeAV1, MimeTypeVP9:
return true
}
return false
}
func IsMimeTypeStringSVC(mime string) bool {
return IsMimeTypeSVC(NormalizeMimeType(mime))
}
func IsMimeTypeStringRED(mime string) bool {
return NormalizeMimeType(mime) == MimeTypeRED
}
func IsMimeTypeStringOpus(mime string) bool {
return NormalizeMimeType(mime) == MimeTypeOpus
}
func IsMimeTypeStringRTX(mime string) bool {
return NormalizeMimeType(mime) == MimeTypeRTX
}
func IsMimeTypeStringVP8(mime string) bool {
return NormalizeMimeType(mime) == MimeTypeVP8
}
func IsMimeTypeStringVP9(mime string) bool {
return NormalizeMimeType(mime) == MimeTypeVP9
}
func IsMimeTypeStringH264(mime string) bool {
return NormalizeMimeType(mime) == MimeTypeH264
}
pkg/sfu/pacer/base.go
+13 -2
View File
@@ -20,6 +20,7 @@ import (
"time"
"github.com/livekit/livekit-server/pkg/sfu/bwe"
"github.com/livekit/mediatransportutil"
"github.com/livekit/protocol/logger"
"github.com/livekit/protocol/utils/mono"
"github.com/pion/rtp"
@@ -56,9 +57,17 @@ func (b *Base) TimeSinceLastSentPacket() time.Duration {
func (b *Base) SendPacket(p *Packet) (int, error) {
defer func() {
if p.HeaderPool != nil && p.Header != nil {
*p.Header = rtp.Header{}
p.HeaderPool.Put(p.Header)
}
if p.Pool != nil && p.PoolEntity != nil {
p.Pool.Put(p.PoolEntity)
}
*p = Packet{}
PacketFactory.Put(p)
}()
err := b.patchRTPHeaderExtensions(p)
@@ -83,8 +92,10 @@ func (b *Base) SendPacket(p *Packet) (int, error) {
func (b *Base) patchRTPHeaderExtensions(p *Packet) error {
sendingAt := mono.Now()
if p.AbsSendTimeExtID != 0 {
absSendTime := rtp.NewAbsSendTimeExtension(sendingAt)
absSendTimeBytes, err := absSendTime.Marshal()
absSendTimeExt := rtp.AbsSendTimeExtension{
Timestamp: uint64(mediatransportutil.ToNtpTime(sendingAt) >> 14),
}
absSendTimeBytes, err := absSendTimeExt.Marshal()
if err != nil {
return err
}
pkg/sfu/pacer/leaky_bucket.go
+1 -1
View File
@@ -80,8 +80,8 @@ func (l *LeakyBucket) Enqueue(p *Packet) {
func (l *LeakyBucket) sendWorker() {
l.lock.RLock()
interval := l.interval
bitrate := l.bitrate
l.lock.RUnlock()
var bitrate int
timer := time.NewTimer(interval)
overage := 0
pkg/sfu/pacer/pacer.go
+19
View File
@@ -23,8 +23,27 @@ import (
"github.com/pion/webrtc/v4"
)
var (
PacketFactory = &sync.Pool{
New: func() any {
return &Packet{}
},
}
)
// --------------------------------------
type PacerBehavior string
const (
PacerBehaviorPassThrough PacerBehavior = "pass-through"
PacerBehaviorNoQueue PacerBehavior = "no-queue"
PacerBehaviorLeakybucket PacerBehavior = "leaky-bucket"
)
type Packet struct {
Header *rtp.Header
HeaderPool *sync.Pool
HeaderSize int
Payload []byte
IsRTX bool
pkg/sfu/packettrailer/packet_trailer.go
+46
View File
@@ -0,0 +1,46 @@
// Copyright 2026 LiveKit, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package packettrailer
var Magic = [4]byte{'L', 'K', 'T', 'S'}
const (
xorByte = 0xFF
envelopeSize = 5 // 1B trailer_len + 4B magic
)
// StripTrailer returns the number of bytes to strip from the end of an RTP
// payload if it contains an LKTS trailer. The trailer is located by checking
// for the "LKTS" magic suffix and then reading the XORed trailer_len byte
// immediately before it. Returns 0 if absent or ineligible.
func StripTrailer(payload []byte, marker bool) int {
if !marker || len(payload) < envelopeSize {
return 0
}
tail := payload[len(payload)-4:]
if tail[0] != Magic[0] || tail[1] != Magic[1] ||
tail[2] != Magic[2] || tail[3] != Magic[3] {
return 0
}
trailerLen := int(payload[len(payload)-5] ^ xorByte)
if trailerLen < envelopeSize || trailerLen > len(payload) {
return 0
}
return trailerLen
}
pkg/sfu/packettrailer/packet_trailer_test.go
+182
View File
@@ -0,0 +1,182 @@
// Copyright 2026 LiveKit, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package packettrailer
import (
"encoding/binary"
"testing"
)
const (
tagTimestampUs = 0x01
tagFrameID = 0x02
)
// appendTLV appends a single XORed TLV element to dst.
func appendTLV(dst []byte, tag byte, value []byte) []byte {
dst = append(dst, tag^xorByte, byte(len(value))^xorByte)
for _, b := range value {
dst = append(dst, b^xorByte)
}
return dst
}
// appendEnvelope appends the 5-byte envelope (XORed trailer_len + magic).
func appendEnvelope(dst []byte, trailerLen byte) []byte {
dst = append(dst, trailerLen^xorByte)
dst = append(dst, Magic[:]...)
return dst
}
// makeTrailer builds a complete LKTS trailer with both timestamp and frame_id TLVs.
func makeTrailer(timestampUs int64, frameID uint32) []byte {
var trailer []byte
var tsBuf [8]byte
binary.BigEndian.PutUint64(tsBuf[:], uint64(timestampUs))
trailer = appendTLV(trailer, tagTimestampUs, tsBuf[:])
var fidBuf [4]byte
binary.BigEndian.PutUint32(fidBuf[:], frameID)
trailer = appendTLV(trailer, tagFrameID, fidBuf[:])
trailerLen := byte(len(trailer) + envelopeSize)
trailer = appendEnvelope(trailer, trailerLen)
return trailer
}
// makePayloadWithTrailer builds a fake video payload followed by a full LKTS trailer.
func makePayloadWithTrailer(videoLen int, timestampUs int64, frameID uint32) []byte {
video := make([]byte, videoLen)
for i := range video {
video[i] = byte(i)
}
return append(video, makeTrailer(timestampUs, frameID)...)
}
// makeTimestampOnlyTrailer builds a trailer with only the timestamp TLV.
func makeTimestampOnlyTrailer(timestampUs int64) []byte {
var trailer []byte
var tsBuf [8]byte
binary.BigEndian.PutUint64(tsBuf[:], uint64(timestampUs))
trailer = appendTLV(trailer, tagTimestampUs, tsBuf[:])
trailerLen := byte(len(trailer) + envelopeSize)
trailer = appendEnvelope(trailer, trailerLen)
return trailer
}
func TestStripTrailer(t *testing.T) {
fullTrailerSize := 21 // (1+1+8) + (1+1+4) + 5
tsOnlyTrailerSize := 15 // (1+1+8) + 5
tests := []struct {
name string
payload []byte
marker bool
wantStrip int
}{
{
name: "marker set with full trailer (timestamp + frame_id)",
payload: makePayloadWithTrailer(20, 1700000000000000, 42),
marker: true,
wantStrip: fullTrailerSize,
},
{
name: "marker set with timestamp-only trailer",
payload: func() []byte {
video := make([]byte, 20)
return append(video, makeTimestampOnlyTrailer(1700000000000000)...)
}(),
marker: true,
wantStrip: tsOnlyTrailerSize,
},
{
name: "marker not set with valid trailer",
payload: makePayloadWithTrailer(20, 1700000000000000, 42),
marker: false,
wantStrip: 0,
},
{
name: "marker set without magic",
payload: make([]byte, 32),
marker: true,
wantStrip: 0,
},
{
name: "marker set but payload too short for envelope",
payload: []byte{0x4C, 0x4B, 0x54, 0x53},
marker: true,
wantStrip: 0,
},
{
name: "marker set with partial magic mismatch",
payload: func() []byte {
p := makePayloadWithTrailer(20, 1700000000000000, 42)
p[len(p)-1] = 'x'
return p
}(),
marker: true,
wantStrip: 0,
},
{
name: "trailer_len exceeds payload length",
payload: func() []byte {
var trailer []byte
var tsBuf [8]byte
binary.BigEndian.PutUint64(tsBuf[:], 42)
trailer = appendTLV(trailer, tagTimestampUs, tsBuf[:])
trailer = appendEnvelope(trailer, 200)
return trailer
}(),
marker: true,
wantStrip: 0,
},
{
name: "trailer_len smaller than envelope (invalid)",
payload: func() []byte {
video := make([]byte, 20)
var trailer []byte
var tsBuf [8]byte
binary.BigEndian.PutUint64(tsBuf[:], 42)
trailer = appendTLV(trailer, tagTimestampUs, tsBuf[:])
trailer = appendEnvelope(trailer, 3)
return append(video, trailer...)
}(),
marker: true,
wantStrip: 0,
},
{
name: "exactly envelope-only trailer",
payload: appendEnvelope(nil, byte(envelopeSize)),
marker: true,
wantStrip: envelopeSize,
},
{
name: "empty payload",
payload: []byte{},
marker: true,
wantStrip: 0,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got := StripTrailer(tt.payload, tt.marker)
if got != tt.wantStrip {
t.Errorf("StripTrailer() = %d, want %d", got, tt.wantStrip)
}
})
}
}
pkg/sfu/playoutdelay_test.go
+3 -1
View File
@@ -26,7 +26,9 @@ import (
)
func TestPlayoutDelay(t *testing.T) {
stats := rtpstats.NewRTPStatsSender(rtpstats.RTPStatsParams{ClockRate: 900000, Logger: logger.GetLogger()}, 128)
stats := rtpstats.NewRTPStatsSender(rtpstats.RTPStatsParams{}, 128)
stats.SetClockRate(90000)
c, err := NewPlayoutDelayController(100, 120, logger.GetLogger(), stats)
require.NoError(t, err)
pkg/sfu/receiver.go
+91 -678
View File
@@ -15,181 +15,38 @@
package sfu
import (
"errors"
"io"
"strings"
"sync"
"time"
"github.com/pion/rtcp"
"github.com/pion/webrtc/v4"
"go.uber.org/atomic"
"github.com/livekit/mediatransportutil/pkg/bucket"
"github.com/livekit/protocol/codecs/mime"
"github.com/livekit/protocol/livekit"
"github.com/livekit/protocol/logger"
"github.com/livekit/protocol/utils"
"github.com/livekit/livekit-server/pkg/sfu/audio"
"github.com/livekit/livekit-server/pkg/sfu/buffer"
"github.com/livekit/livekit-server/pkg/sfu/connectionquality"
"github.com/livekit/livekit-server/pkg/sfu/mime"
dd "github.com/livekit/livekit-server/pkg/sfu/rtpextension/dependencydescriptor"
"github.com/livekit/livekit-server/pkg/sfu/rtpstats"
"github.com/livekit/livekit-server/pkg/sfu/streamtracker"
)
var (
ErrReceiverClosed = errors.New("receiver closed")
ErrDownTrackAlreadyExist = errors.New("DownTrack already exist")
ErrBufferNotFound = errors.New("buffer not found")
ErrDuplicateLayer = errors.New("duplicate layer")
)
// --------------------------------------
type PLIThrottleConfig struct {
LowQuality time.Duration `yaml:"low_quality,omitempty"`
MidQuality time.Duration `yaml:"mid_quality,omitempty"`
HighQuality time.Duration `yaml:"high_quality,omitempty"`
}
var (
DefaultPLIThrottleConfig = PLIThrottleConfig{
LowQuality: 500 * time.Millisecond,
MidQuality: time.Second,
HighQuality: time.Second,
}
)
// --------------------------------------
type AudioConfig struct {
audio.AudioLevelConfig `yaml:",inline"`
// enable red encoding downtrack for opus only audio up track
ActiveREDEncoding bool `yaml:"active_red_encoding,omitempty"`
// enable proxying weakest subscriber loss to publisher in RTCP Receiver Report
EnableLossProxying bool `yaml:"enable_loss_proxying,omitempty"`
}
var (
DefaultAudioConfig = AudioConfig{
AudioLevelConfig: audio.DefaultAudioLevelConfig,
}
)
// --------------------------------------
type AudioLevelHandle func(level uint8, duration uint32)
type Bitrates [buffer.DefaultMaxLayerSpatial + 1][buffer.DefaultMaxLayerTemporal + 1]int64
type ReceiverCodecState int
const (
ReceiverCodecStateNormal ReceiverCodecState = iota
ReceiverCodecStateSuspended
ReceiverCodecStateInvalid
)
// TrackReceiver defines an interface receive media from remote peer
type TrackReceiver interface {
TrackID() livekit.TrackID
StreamID() string
// returns the initial codec of the receiver, it is determined by the track's codec
// and will not change if the codec changes during the session (publisher changes codec)
Codec() webrtc.RTPCodecParameters
Mime() mime.MimeType
HeaderExtensions() []webrtc.RTPHeaderExtensionParameter
IsClosed() bool
ReadRTP(buf []byte, layer uint8, esn uint64) (int, error)
GetLayeredBitrate() ([]int32, Bitrates)
GetAudioLevel() (float64, bool)
SendPLI(layer int32, force bool)
SetUpTrackPaused(paused bool)
SetMaxExpectedSpatialLayer(layer int32)
AddDownTrack(track TrackSender) error
DeleteDownTrack(participantID livekit.ParticipantID)
GetDownTracks() []TrackSender
DebugInfo() map[string]interface{}
TrackInfo() *livekit.TrackInfo
UpdateTrackInfo(ti *livekit.TrackInfo)
// Get primary receiver if this receiver represents a RED codec; otherwise it will return itself
GetPrimaryReceiverForRed() TrackReceiver
// Get red receiver for primary codec, used by forward red encodings for opus only codec
GetRedReceiver() TrackReceiver
GetTemporalLayerFpsForSpatial(layer int32) []float32
GetTrackStats() *livekit.RTPStats
// AddOnReady adds a function to be called when the receiver is ready, the callback
// could be called immediately if the receiver is ready when the callback is added
AddOnReady(func())
AddOnCodecStateChange(func(webrtc.RTPCodecParameters, ReceiverCodecState))
CodecState() ReceiverCodecState
}
type redPktWriteFunc func(pkt *buffer.ExtPacket, spatialLayer int32) int
var _ TrackReceiver = (*WebRTCReceiver)(nil)
// WebRTCReceiver receives a media track
type WebRTCReceiver struct {
logger logger.Logger
*ReceiverBase
pliThrottleConfig PLIThrottleConfig
audioConfig AudioConfig
trackID livekit.TrackID
streamID string
kind webrtc.RTPCodecType
receiver *webrtc.RTPReceiver
codec webrtc.RTPCodecParameters
codecState ReceiverCodecState
codecStateLock sync.Mutex
onCodecStateChange []func(webrtc.RTPCodecParameters, ReceiverCodecState)
isSVC bool
isRED bool
onCloseHandler func()
closeOnce sync.Once
closed atomic.Bool
useTrackers bool
trackInfo atomic.Pointer[livekit.TrackInfo]
receiver *webrtc.RTPReceiver
onCloseHandler func()
onRTCP func([]rtcp.Packet)
bufferMu sync.RWMutex
buffers [buffer.DefaultMaxLayerSpatial + 1]*buffer.Buffer
upTracks [buffer.DefaultMaxLayerSpatial + 1]TrackRemote
rtt uint32
lbThreshold int
streamTrackerManager *StreamTrackerManager
downTrackSpreader *DownTrackSpreader
upTracksMu sync.Mutex
upTracks [buffer.DefaultMaxLayerSpatial + 1]TrackRemote
connectionStats *connectionquality.ConnectionStats
onStatsUpdate func(w *WebRTCReceiver, stat *livekit.AnalyticsStat)
onMaxLayerChange func(maxLayer int32)
primaryReceiver atomic.Pointer[RedPrimaryReceiver]
redReceiver atomic.Pointer[RedReceiver]
redPktWriter atomic.Value // redPktWriteFunc
forwardStats *ForwardStats
onStatsUpdate func(w *WebRTCReceiver, stat *livekit.AnalyticsStat)
}
type ReceiverOpts func(w *WebRTCReceiver) *WebRTCReceiver
@@ -197,7 +54,7 @@ type ReceiverOpts func(w *WebRTCReceiver) *WebRTCReceiver
// WithPliThrottleConfig indicates minimum time(ms) between sending PLIs
func WithPliThrottleConfig(pliThrottleConfig PLIThrottleConfig) ReceiverOpts {
return func(w *WebRTCReceiver) *WebRTCReceiver {
w.pliThrottleConfig = pliThrottleConfig
w.ReceiverBase.SetPLIThrottleConfig(pliThrottleConfig)
return w
}
}
@@ -205,15 +62,14 @@ func WithPliThrottleConfig(pliThrottleConfig PLIThrottleConfig) ReceiverOpts {
// WithAudioConfig sets up parameters for active speaker detection
func WithAudioConfig(audioConfig AudioConfig) ReceiverOpts {
return func(w *WebRTCReceiver) *WebRTCReceiver {
w.audioConfig = audioConfig
w.ReceiverBase.SetAudioConfig(audioConfig)
return w
}
}
// WithStreamTrackers enables StreamTracker use for simulcast
func WithStreamTrackers() ReceiverOpts {
func WithEnableRTPStreamRestartDetection(enable bool) ReceiverOpts {
return func(w *WebRTCReceiver) *WebRTCReceiver {
w.useTrackers = true
w.ReceiverBase.SetEnableRTPStreamRestartDetection(enable)
return w
}
}
@@ -225,14 +81,14 @@ func WithStreamTrackers() ReceiverOpts {
// Set to 0 (disabled) by default.
func WithLoadBalanceThreshold(downTracks int) ReceiverOpts {
return func(w *WebRTCReceiver) *WebRTCReceiver {
w.lbThreshold = downTracks
w.ReceiverBase.SetLBThreshold(downTracks)
return w
}
}
func WithForwardStats(forwardStats *ForwardStats) ReceiverOpts {
return func(w *WebRTCReceiver) *WebRTCReceiver {
w.forwardStats = forwardStats
w.ReceiverBase.SetForwardStats(forwardStats)
return w
}
}
@@ -248,243 +104,119 @@ func NewWebRTCReceiver(
opts ...ReceiverOpts,
) *WebRTCReceiver {
w := &WebRTCReceiver{
logger: logger,
receiver: receiver,
trackID: livekit.TrackID(track.ID()),
streamID: track.StreamID(),
codec: track.Codec(),
codecState: ReceiverCodecStateNormal,
kind: track.Kind(),
onRTCP: onRTCP,
isSVC: mime.IsMimeTypeStringSVC(track.Codec().MimeType),
isRED: mime.IsMimeTypeStringRED(track.Codec().MimeType),
receiver: receiver,
onRTCP: onRTCP,
}
w.ReceiverBase = NewReceiverBase(
ReceiverBaseParams{
TrackID: livekit.TrackID(track.ID()),
StreamID: track.StreamID(),
Kind: track.Kind(),
Codec: track.Codec(),
HeaderExtensions: receiver.GetParameters().HeaderExtensions,
Logger: logger,
StreamTrackerManagerConfig: streamTrackerManagerConfig,
StreamTrackerManagerListener: w,
IsSelfClosing: true,
OnClosed: w.onClosed,
},
trackInfo,
ReceiverCodecStateNormal,
)
for _, opt := range opts {
w = opt(w)
}
w.trackInfo.Store(utils.CloneProto(trackInfo))
w.downTrackSpreader = NewDownTrackSpreader(DownTrackSpreaderParams{
Threshold: w.lbThreshold,
Logger: logger,
})
w.connectionStats = connectionquality.NewConnectionStats(connectionquality.ConnectionStatsParams{
ReceiverProvider: w,
Logger: w.logger.WithValues("direction", "up"),
Logger: logger.WithValues("direction", "up"),
})
w.connectionStats.OnStatsUpdate(func(_cs *connectionquality.ConnectionStats, stat *livekit.AnalyticsStat) {
if w.onStatsUpdate != nil {
w.onStatsUpdate(w, stat)
}
})
codec := track.Codec()
w.connectionStats.Start(
mime.NormalizeMimeType(w.codec.MimeType),
mime.NormalizeMimeType(codec.MimeType),
// TODO: technically not correct to declare FEC on when RED. Need the primary codec's fmtp line to check.
mime.IsMimeTypeStringRED(w.codec.MimeType) || strings.Contains(strings.ToLower(w.codec.SDPFmtpLine), "useinbandfec=1"),
mime.IsMimeTypeStringRED(codec.MimeType) || strings.Contains(strings.ToLower(codec.SDPFmtpLine), "useinbandfec=1"),
)
w.streamTrackerManager = NewStreamTrackerManager(logger, trackInfo, w.isSVC, w.codec.ClockRate, streamTrackerManagerConfig)
w.streamTrackerManager.SetListener(w)
// SVC-TODO: Handle DD for non-SVC cases???
if w.isSVC {
for _, ext := range receiver.GetParameters().HeaderExtensions {
if ext.URI == dd.ExtensionURI {
w.streamTrackerManager.AddDependencyDescriptorTrackers()
break
}
}
}
w.UpdateTrackInfo(trackInfo)
return w
}
func (w *WebRTCReceiver) TrackInfo() *livekit.TrackInfo {
return w.trackInfo.Load()
}
func (w *WebRTCReceiver) UpdateTrackInfo(ti *livekit.TrackInfo) {
w.trackInfo.Store(utils.CloneProto(ti))
w.streamTrackerManager.UpdateTrackInfo(ti)
}
func (w *WebRTCReceiver) OnStatsUpdate(fn func(w *WebRTCReceiver, stat *livekit.AnalyticsStat)) {
w.onStatsUpdate = fn
}
func (w *WebRTCReceiver) OnMaxLayerChange(fn func(maxLayer int32)) {
w.bufferMu.Lock()
w.onMaxLayerChange = fn
w.bufferMu.Unlock()
}
func (w *WebRTCReceiver) getOnMaxLayerChange() func(maxLayer int32) {
w.bufferMu.RLock()
defer w.bufferMu.RUnlock()
return w.onMaxLayerChange
}
func (w *WebRTCReceiver) GetConnectionScoreAndQuality() (float32, livekit.ConnectionQuality) {
return w.connectionStats.GetScoreAndQuality()
}
func (w *WebRTCReceiver) IsClosed() bool {
return w.closed.Load()
}
func (w *WebRTCReceiver) SetRTT(rtt uint32) {
w.bufferMu.Lock()
if w.rtt == rtt {
w.bufferMu.Unlock()
return
}
w.rtt = rtt
buffers := w.buffers
w.bufferMu.Unlock()
for _, buff := range buffers {
if buff == nil {
continue
}
buff.SetRTT(rtt)
}
}
func (w *WebRTCReceiver) StreamID() string {
return w.streamID
}
func (w *WebRTCReceiver) TrackID() livekit.TrackID {
return w.trackID
}
func (w *WebRTCReceiver) ssrc(layer int) uint32 {
w.upTracksMu.Lock()
defer w.upTracksMu.Unlock()
if track := w.upTracks[layer]; track != nil {
return uint32(track.SSRC())
}
return 0
}
func (w *WebRTCReceiver) Codec() webrtc.RTPCodecParameters {
return w.codec
}
func (w *WebRTCReceiver) Mime() mime.MimeType {
return mime.NormalizeMimeType(w.codec.MimeType)
}
func (w *WebRTCReceiver) HeaderExtensions() []webrtc.RTPHeaderExtensionParameter {
return w.receiver.GetParameters().HeaderExtensions
}
func (w *WebRTCReceiver) Kind() webrtc.RTPCodecType {
return w.kind
}
func (w *WebRTCReceiver) AddUpTrack(track TrackRemote, buff *buffer.Buffer) error {
if w.closed.Load() {
if w.isClosed.Load() {
return ErrReceiverClosed
}
layer := int32(0)
if w.Kind() == webrtc.RTPCodecTypeVideo && !w.isSVC {
layer = buffer.RidToSpatialLayer(track.RID(), w.trackInfo.Load())
if w.Kind() == webrtc.RTPCodecTypeVideo && w.videoLayerMode != livekit.VideoLayer_MULTIPLE_SPATIAL_LAYERS_PER_STREAM {
layer = buffer.GetSpatialLayerForRid(w.Mime(), track.RID(), w.ReceiverBase.TrackInfo())
}
buff.SetLogger(w.logger.WithValues("layer", layer))
buff.SetAudioLevelParams(audio.AudioLevelParams{
Config: w.audioConfig.AudioLevelConfig,
})
buff.SetAudioLossProxying(w.audioConfig.EnableLossProxying)
buff.OnRtcpFeedback(w.sendRTCP)
buff.OnRtcpSenderReport(func() {
srData := buff.GetSenderReportData()
w.downTrackSpreader.Broadcast(func(dt TrackSender) {
_ = dt.HandleRTCPSenderReportData(w.codec.PayloadType, w.isSVC, layer, srData)
})
})
if w.Kind() == webrtc.RTPCodecTypeVideo && layer == 0 {
buff.OnCodecChange(w.handleCodecChange)
if layer < 0 {
w.ReceiverBase.Logger().Warnw(
"invalid layer", nil,
"rid", track.RID(),
"trackInfo", logger.Proto(w.ReceiverBase.TrackInfo()),
)
return ErrInvalidLayer
}
var duration time.Duration
switch layer {
case 2:
duration = w.pliThrottleConfig.HighQuality
case 1:
duration = w.pliThrottleConfig.MidQuality
case 0:
duration = w.pliThrottleConfig.LowQuality
default:
duration = w.pliThrottleConfig.MidQuality
}
if duration != 0 {
buff.SetPLIThrottle(duration.Nanoseconds())
}
w.bufferMu.Lock()
w.upTracksMu.Lock()
if w.upTracks[layer] != nil {
w.bufferMu.Unlock()
w.upTracksMu.Unlock()
return ErrDuplicateLayer
}
w.upTracks[layer] = track
w.buffers[layer] = buff
rtt := w.rtt
w.bufferMu.Unlock()
w.upTracksMu.Unlock()
buff.SetRTT(rtt)
buff.SetPaused(w.streamTrackerManager.IsPaused())
if w.Kind() == webrtc.RTPCodecTypeVideo && w.useTrackers {
w.streamTrackerManager.AddTracker(layer)
}
go w.forwardRTP(layer, buff)
w.ReceiverBase.AddBuffer(buff, layer)
buff.OnRtcpFeedback(w.sendRTCP)
w.ReceiverBase.StartBuffer(buff, layer)
return nil
}
// SetUpTrackPaused indicates upstream will not be sending any data.
// this will reflect the "muted" status and will pause streamtracker to ensure we don't turn off
// the layer
func (w *WebRTCReceiver) SetUpTrackPaused(paused bool) {
w.streamTrackerManager.SetPaused(paused)
func (w *WebRTCReceiver) NumUpTracks() int {
numUpTracks := 0
w.bufferMu.RLock()
for _, buff := range w.buffers {
if buff == nil {
continue
w.upTracksMu.Lock()
for _, track := range w.upTracks {
if track != nil {
numUpTracks++
}
buff.SetPaused(paused)
}
w.bufferMu.RUnlock()
w.upTracksMu.Unlock()
w.connectionStats.UpdateMute(paused)
return numUpTracks
}
func (w *WebRTCReceiver) AddDownTrack(track TrackSender) error {
if w.closed.Load() {
return ErrReceiverClosed
}
if w.downTrackSpreader.HasDownTrack(track.SubscriberID()) {
w.logger.Infow("subscriberID already exists, replacing downtrack", "subscriberID", track.SubscriberID())
}
track.UpTrackMaxPublishedLayerChange(w.streamTrackerManager.GetMaxPublishedLayer())
track.UpTrackMaxTemporalLayerSeenChange(w.streamTrackerManager.GetMaxTemporalLayerSeen())
w.downTrackSpreader.Store(track)
w.logger.Debugw("downtrack added", "subscriberID", track.SubscriberID())
return nil
}
func (w *WebRTCReceiver) GetDownTracks() []TrackSender {
return w.downTrackSpreader.GetDownTracks()
func (w *WebRTCReceiver) UpdateTrackInfo(ti *livekit.TrackInfo) {
w.ReceiverBase.UpdateTrackInfo(ti)
w.connectionStats.UpdateMute(ti.GetMuted())
}
func (w *WebRTCReceiver) notifyMaxExpectedLayer(layer int32) {
@@ -499,9 +231,8 @@ func (w *WebRTCReceiver) notifyMaxExpectedLayer(layer int32) {
}
expectedBitrate := int64(0)
for _, vl := range ti.Layers {
l := buffer.VideoQualityToSpatialLayer(vl.Quality, ti)
if l <= layer {
for _, vl := range buffer.GetVideoLayersForMimeType(w.Mime(), ti) {
if vl.SpatialLayer <= layer {
expectedBitrate += int64(vl.Bitrate)
}
}
@@ -510,89 +241,54 @@ func (w *WebRTCReceiver) notifyMaxExpectedLayer(layer int32) {
}
func (w *WebRTCReceiver) SetMaxExpectedSpatialLayer(layer int32) {
w.streamTrackerManager.SetMaxExpectedSpatialLayer(layer)
w.ReceiverBase.SetMaxExpectedSpatialLayer(layer)
w.notifyMaxExpectedLayer(layer)
if layer == buffer.InvalidLayerSpatial {
w.connectionStats.UpdateLayerMute(true)
} else {
w.connectionStats.UpdateLayerMute(false)
w.connectionStats.AddLayerTransition(w.streamTrackerManager.DistanceToDesired())
w.connectionStats.AddLayerTransition(w.ReceiverBase.StreamTrackerManager().DistanceToDesired())
}
}
// StreamTrackerManagerListener.OnAvailableLayersChanged
func (w *WebRTCReceiver) OnAvailableLayersChanged() {
w.downTrackSpreader.Broadcast(func(dt TrackSender) {
dt.UpTrackLayersChange()
})
w.connectionStats.AddLayerTransition(w.streamTrackerManager.DistanceToDesired())
w.connectionStats.AddLayerTransition(w.ReceiverBase.StreamTrackerManager().DistanceToDesired())
}
// StreamTrackerManagerListener.OnBitrateAvailabilityChanged
func (w *WebRTCReceiver) OnBitrateAvailabilityChanged() {
w.downTrackSpreader.Broadcast(func(dt TrackSender) {
dt.UpTrackBitrateAvailabilityChange()
})
}
// StreamTrackerManagerListener.OnMaxPublishedLayerChanged
func (w *WebRTCReceiver) OnMaxPublishedLayerChanged(maxPublishedLayer int32) {
w.downTrackSpreader.Broadcast(func(dt TrackSender) {
dt.UpTrackMaxPublishedLayerChange(maxPublishedLayer)
})
w.notifyMaxExpectedLayer(maxPublishedLayer)
w.connectionStats.AddLayerTransition(w.streamTrackerManager.DistanceToDesired())
w.connectionStats.AddLayerTransition(w.ReceiverBase.StreamTrackerManager().DistanceToDesired())
}
// StreamTrackerManagerListener.OnMaxTemporalLayerSeenChanged
func (w *WebRTCReceiver) OnMaxTemporalLayerSeenChanged(maxTemporalLayerSeen int32) {
w.downTrackSpreader.Broadcast(func(dt TrackSender) {
dt.UpTrackMaxTemporalLayerSeenChange(maxTemporalLayerSeen)
})
w.connectionStats.AddLayerTransition(w.streamTrackerManager.DistanceToDesired())
w.connectionStats.AddLayerTransition(w.ReceiverBase.StreamTrackerManager().DistanceToDesired())
}
// StreamTrackerManagerListener.OnMaxAvailableLayerChanged
func (w *WebRTCReceiver) OnMaxAvailableLayerChanged(maxAvailableLayer int32) {
if onMaxLayerChange := w.getOnMaxLayerChange(); onMaxLayerChange != nil {
onMaxLayerChange(maxAvailableLayer)
}
}
// StreamTrackerManagerListener.OnBitrateReport
func (w *WebRTCReceiver) OnBitrateReport(availableLayers []int32, bitrates Bitrates) {
w.downTrackSpreader.Broadcast(func(dt TrackSender) {
dt.UpTrackBitrateReport(availableLayers, bitrates)
})
w.connectionStats.AddLayerTransition(w.streamTrackerManager.DistanceToDesired())
w.connectionStats.AddLayerTransition(w.ReceiverBase.StreamTrackerManager().DistanceToDesired())
}
func (w *WebRTCReceiver) GetLayeredBitrate() ([]int32, Bitrates) {
return w.streamTrackerManager.GetLayeredBitrate()
}
// OnCloseHandler method to be called on remote tracked removed
// OnCloseHandler method to be called on remote track removed
func (w *WebRTCReceiver) OnCloseHandler(fn func()) {
w.onCloseHandler = fn
}
// DeleteDownTrack removes a DownTrack from a Receiver
func (w *WebRTCReceiver) DeleteDownTrack(subscriberID livekit.ParticipantID) {
if w.closed.Load() {
return
}
w.downTrackSpreader.Free(subscriberID)
w.logger.Debugw("downtrack deleted", "subscriberID", subscriberID)
}
func (w *WebRTCReceiver) sendRTCP(packets []rtcp.Packet) {
if packets == nil || w.closed.Load() {
if packets == nil || w.isClosed.Load() {
return
}
@@ -601,93 +297,10 @@ func (w *WebRTCReceiver) sendRTCP(packets []rtcp.Packet) {
}
}
func (w *WebRTCReceiver) SendPLI(layer int32, force bool) {
// SVC-TODO : should send LRR (Layer Refresh Request) instead of PLI
buff := w.getBuffer(layer)
if buff == nil {
return
}
buff.SendPLI(force)
}
func (w *WebRTCReceiver) getBuffer(layer int32) *buffer.Buffer {
w.bufferMu.RLock()
defer w.bufferMu.RUnlock()
return w.getBufferLocked(layer)
}
func (w *WebRTCReceiver) getBufferLocked(layer int32) *buffer.Buffer {
// for svc codecs, use layer = 0 always.
// spatial layers are in-built and handled by single buffer
if w.isSVC {
layer = 0
}
if layer < 0 || int(layer) >= len(w.buffers) {
return nil
}
return w.buffers[layer]
}
func (w *WebRTCReceiver) ReadRTP(buf []byte, layer uint8, esn uint64) (int, error) {
b := w.getBuffer(int32(layer))
if b == nil {
return 0, ErrBufferNotFound
}
return b.GetPacket(buf, esn)
}
func (w *WebRTCReceiver) GetTrackStats() *livekit.RTPStats {
w.bufferMu.RLock()
defer w.bufferMu.RUnlock()
stats := make([]*livekit.RTPStats, 0, len(w.buffers))
for _, buff := range w.buffers {
if buff == nil {
continue
}
sswl := buff.GetStats()
if sswl == nil {
continue
}
stats = append(stats, sswl)
}
return rtpstats.AggregateRTPStats(stats)
}
func (w *WebRTCReceiver) GetAudioLevel() (float64, bool) {
if w.Kind() == webrtc.RTPCodecTypeVideo {
return 0, false
}
w.bufferMu.RLock()
defer w.bufferMu.RUnlock()
for _, buff := range w.buffers {
if buff == nil {
continue
}
return buff.GetAudioLevel()
}
return 0, false
}
func (w *WebRTCReceiver) GetDeltaStats() map[uint32]*buffer.StreamStatsWithLayers {
w.bufferMu.RLock()
defer w.bufferMu.RUnlock()
deltaStats := make(map[uint32]*buffer.StreamStatsWithLayers, len(w.buffers))
for layer, buff := range w.buffers {
buffers := w.ReceiverBase.GetAllBuffers()
deltaStats := make(map[uint32]*buffer.StreamStatsWithLayers, len(buffers))
for layer, buff := range buffers {
if buff == nil {
continue
}
@@ -709,11 +322,9 @@ func (w *WebRTCReceiver) GetDeltaStats() map[uint32]*buffer.StreamStatsWithLayer
}
func (w *WebRTCReceiver) GetLastSenderReportTime() time.Time {
w.bufferMu.RLock()
defer w.bufferMu.RUnlock()
buffers := w.ReceiverBase.GetAllBuffers()
latestSRTime := time.Time{}
for _, buff := range w.buffers {
for _, buff := range buffers {
if buff == nil {
continue
}
@@ -727,120 +338,22 @@ func (w *WebRTCReceiver) GetLastSenderReportTime() time.Time {
return latestSRTime
}
func (w *WebRTCReceiver) forwardRTP(layer int32, buff *buffer.Buffer) {
defer func() {
w.closeOnce.Do(func() {
w.closed.Store(true)
w.closeTracks()
if pr := w.primaryReceiver.Load(); pr != nil {
pr.Close()
}
if pr := w.redReceiver.Load(); pr != nil {
pr.Close()
}
})
w.streamTrackerManager.RemoveTracker(layer)
if w.isSVC {
w.streamTrackerManager.RemoveAllTrackers()
}
}()
var spatialTrackers [buffer.DefaultMaxLayerSpatial + 1]streamtracker.StreamTrackerWorker
if layer < 0 || int(layer) >= len(spatialTrackers) {
w.logger.Errorw("invalid layer", nil, "layer", layer)
return
}
spatialTrackers[layer] = w.streamTrackerManager.GetTracker(layer)
pktBuf := make([]byte, bucket.MaxPktSize)
for {
pkt, err := buff.ReadExtended(pktBuf)
if err == io.EOF {
return
}
if pkt.Packet.PayloadType != uint8(w.codec.PayloadType) {
// drop packets as we don't support codec fallback directly
continue
}
spatialLayer := layer
if pkt.Spatial >= 0 {
// svc packet, take spatial layer info from packet
spatialLayer = pkt.Spatial
}
if int(spatialLayer) >= len(spatialTrackers) {
w.logger.Errorw(
"unexpected spatial layer", nil,
"spatialLayer", spatialLayer,
"pktSpatialLayer", pkt.Spatial,
)
continue
}
writeCount := w.downTrackSpreader.Broadcast(func(dt TrackSender) {
_ = dt.WriteRTP(pkt, spatialLayer)
})
if f := w.redPktWriter.Load(); f != nil {
writeCount += f.(redPktWriteFunc)(pkt, spatialLayer)
}
// track delay/jitter
if writeCount > 0 && w.forwardStats != nil {
w.forwardStats.Update(pkt.Arrival, time.Now().UnixNano())
}
// track video layers
if w.Kind() == webrtc.RTPCodecTypeVideo {
if spatialTrackers[spatialLayer] == nil {
spatialTrackers[spatialLayer] = w.streamTrackerManager.GetTracker(spatialLayer)
if spatialTrackers[spatialLayer] == nil {
spatialTrackers[spatialLayer] = w.streamTrackerManager.AddTracker(spatialLayer)
}
}
if spatialTrackers[spatialLayer] != nil {
spatialTrackers[spatialLayer].Observe(
pkt.Temporal,
len(pkt.RawPacket),
len(pkt.Packet.Payload),
pkt.Packet.Marker,
pkt.Packet.Timestamp,
pkt.DependencyDescriptor,
)
}
}
}
}
// closeTracks close all tracks from Receiver
func (w *WebRTCReceiver) closeTracks() {
func (w *WebRTCReceiver) onClosed() {
w.connectionStats.Close()
w.streamTrackerManager.Close()
closeTrackSenders(w.downTrackSpreader.ResetAndGetDownTracks())
if w.onCloseHandler != nil {
w.onCloseHandler()
}
}
func (w *WebRTCReceiver) DebugInfo() map[string]interface{} {
isSimulcast := !w.isSVC
if ti := w.trackInfo.Load(); ti != nil {
isSimulcast = isSimulcast && len(ti.Layers) > 1
}
info := map[string]interface{}{
"SVC": w.isSVC,
"Simulcast": isSimulcast,
}
func (w *WebRTCReceiver) DebugInfo() map[string]any {
info := w.ReceiverBase.DebugInfo()
w.bufferMu.RLock()
upTrackInfo := make([]map[string]interface{}, 0, len(w.upTracks))
w.upTracksMu.Lock()
upTrackInfo := make([]map[string]any, 0, len(w.upTracks))
for layer, ut := range w.upTracks {
if ut != nil {
upTrackInfo = append(upTrackInfo, map[string]interface{}{
upTrackInfo = append(upTrackInfo, map[string]any{
"Layer": layer,
"SSRC": ut.SSRC(),
"Msid": ut.Msid(),
@@ -848,110 +361,10 @@ func (w *WebRTCReceiver) DebugInfo() map[string]interface{} {
})
}
}
w.bufferMu.RUnlock()
w.upTracksMu.Unlock()
info["UpTracks"] = upTrackInfo
return info
}
func (w *WebRTCReceiver) GetPrimaryReceiverForRed() TrackReceiver {
if !w.isRED || w.closed.Load() {
return w
}
if w.primaryReceiver.Load() == nil {
pr := NewRedPrimaryReceiver(w, DownTrackSpreaderParams{
Threshold: w.lbThreshold,
Logger: w.logger,
})
if w.primaryReceiver.CompareAndSwap(nil, pr) {
w.redPktWriter.Store(redPktWriteFunc(pr.ForwardRTP))
}
}
return w.primaryReceiver.Load()
}
func (w *WebRTCReceiver) GetRedReceiver() TrackReceiver {
if w.isRED || w.closed.Load() {
return w
}
if w.redReceiver.Load() == nil {
pr := NewRedReceiver(w, DownTrackSpreaderParams{
Threshold: w.lbThreshold,
Logger: w.logger,
})
if w.redReceiver.CompareAndSwap(nil, pr) {
w.redPktWriter.Store(redPktWriteFunc(pr.ForwardRTP))
}
}
return w.redReceiver.Load()
}
func (w *WebRTCReceiver) GetTemporalLayerFpsForSpatial(layer int32) []float32 {
b := w.getBuffer(layer)
if b == nil {
return nil
}
if !w.isSVC {
return b.GetTemporalLayerFpsForSpatial(0)
}
return b.GetTemporalLayerFpsForSpatial(layer)
}
func (w *WebRTCReceiver) AddOnReady(fn func()) {
// webRTCReceiver is always ready after created
fn()
}
func (w *WebRTCReceiver) handleCodecChange(newCodec webrtc.RTPCodecParameters) {
// we don't support the codec fallback directly, set the codec state to invalid once it happens
w.SetCodecState(ReceiverCodecStateInvalid)
}
func (w *WebRTCReceiver) AddOnCodecStateChange(f func(webrtc.RTPCodecParameters, ReceiverCodecState)) {
w.codecStateLock.Lock()
w.onCodecStateChange = append(w.onCodecStateChange, f)
w.codecStateLock.Unlock()
}
func (w *WebRTCReceiver) CodecState() ReceiverCodecState {
w.codecStateLock.Lock()
defer w.codecStateLock.Unlock()
return w.codecState
}
func (w *WebRTCReceiver) SetCodecState(state ReceiverCodecState) {
w.codecStateLock.Lock()
if w.codecState == state || w.codecState == ReceiverCodecStateInvalid {
w.codecStateLock.Unlock()
return
}
w.codecState = state
fns := w.onCodecStateChange
w.codecStateLock.Unlock()
for _, f := range fns {
f(w.codec, state)
}
}
// -----------------------------------------------------------
// closes all track senders in parallel, returns when all are closed
func closeTrackSenders(senders []TrackSender) {
wg := sync.WaitGroup{}
for _, dt := range senders {
dt := dt
wg.Add(1)
go func() {
defer wg.Done()
dt.Close()
}()
}
wg.Wait()
}
pkg/sfu/receiver_base.go
+1294
View File
File diff suppressed because it is too large Load Diff
pkg/sfu/receiver_test.go
+6 -6
View File
@@ -102,15 +102,15 @@ func BenchmarkWriteRTP(b *testing.B) {
}
func benchmarkNoPool(b *testing.B, downTracks int) {
for i := 0; i < b.N; i++ {
for dt := 0; dt < downTracks; dt++ {
for b.Loop() {
for range downTracks {
writeRTP()
}
}
}
func benchmarkPool(b *testing.B, wp *workerpool.WorkerPool, buckets []int) {
for i := 0; i < b.N; i++ {
for b.Loop() {
var wg sync.WaitGroup
for j := range buckets {
downTracks := buckets[j]
@@ -130,7 +130,7 @@ func benchmarkPool(b *testing.B, wp *workerpool.WorkerPool, buckets []int) {
}
func benchmarkGoroutine(b *testing.B, buckets []int) {
for i := 0; i < b.N; i++ {
for b.Loop() {
var wg sync.WaitGroup
for j := range buckets {
downTracks := buckets[j]
@@ -150,7 +150,7 @@ func benchmarkGoroutine(b *testing.B, buckets []int) {
}
func benchmarkLoadBalanced(b *testing.B, numProcs, step, downTracks int) {
for i := 0; i < b.N; i++ {
for b.Loop() {
start := atomic.NewUint64(0)
step := uint64(step)
end := uint64(downTracks)
@@ -177,7 +177,7 @@ func benchmarkLoadBalanced(b *testing.B, numProcs, step, downTracks int) {
}
func benchmarkLoadBalancedPool(b *testing.B, wp *workerpool.WorkerPool, numProcs, step, downTracks int) {
for i := 0; i < b.N; i++ {
for b.Loop() {
start := atomic.NewUint64(0)
step := uint64(step)
end := uint64(downTracks)
pkg/sfu/redprimaryreceiver.go
+69 -13
View File
@@ -21,12 +21,16 @@ import (
"go.uber.org/atomic"
"github.com/pion/rtp"
"github.com/pion/webrtc/v4"
"github.com/livekit/livekit-server/pkg/sfu/buffer"
"github.com/livekit/livekit-server/pkg/sfu/utils"
"github.com/livekit/protocol/livekit"
"github.com/livekit/protocol/logger"
)
var _ REDTransformer = (*RedPrimaryReceiver)(nil)
var (
ErrIncompleteRedHeader = errors.New("incomplete red block header")
ErrIncompleteRedBlock = errors.New("incomplete red block payload")
@@ -34,28 +38,30 @@ var (
type RedPrimaryReceiver struct {
TrackReceiver
downTrackSpreader *DownTrackSpreader
downTrackSpreader *utils.DownTrackSpreader[TrackSender]
logger logger.Logger
closed atomic.Bool
redPT uint8
firstPktReceived bool
lastSeq uint16
firstPktReceived bool
lastSeq uint16
highestForwardedExtSequenceNumber uint64
highestForwardedExtTimestamp uint64
// bitset for upstream packet receive history [lastSeq-8, lastSeq-1], bit 1 represents packet received
pktHistory byte
}
func NewRedPrimaryReceiver(receiver TrackReceiver, dsp DownTrackSpreaderParams) *RedPrimaryReceiver {
func NewRedPrimaryReceiver(receiver TrackReceiver, dsp utils.DownTrackSpreaderParams) REDTransformer {
return &RedPrimaryReceiver{
TrackReceiver: receiver,
downTrackSpreader: NewDownTrackSpreader(dsp),
downTrackSpreader: utils.NewDownTrackSpreader[TrackSender](dsp),
logger: dsp.Logger,
redPT: uint8(receiver.Codec().PayloadType),
}
}
func (r *RedPrimaryReceiver) ForwardRTP(pkt *buffer.ExtPacket, spatialLayer int32) int {
func (r *RedPrimaryReceiver) ForwardRTP(pkt *buffer.ExtPacket, spatialLayer int32) int32 {
// extract primary payload from RED and forward to downtracks
if r.downTrackSpreader.DownTrackCount() == 0 {
return 0
@@ -63,9 +69,11 @@ func (r *RedPrimaryReceiver) ForwardRTP(pkt *buffer.ExtPacket, spatialLayer int3
if pkt.Packet.PayloadType != r.redPT {
// forward non-red packet directly
return r.downTrackSpreader.Broadcast(func(dt TrackSender) {
_ = dt.WriteRTP(pkt, spatialLayer)
var writeCount atomic.Int32
r.downTrackSpreader.Broadcast(func(dt TrackSender) {
writeCount.Add(dt.WriteRTP(pkt, spatialLayer))
})
return writeCount.Load()
}
pkts, err := r.getSendPktsFromRed(pkt.Packet)
@@ -74,7 +82,7 @@ func (r *RedPrimaryReceiver) ForwardRTP(pkt *buffer.ExtPacket, spatialLayer int3
return 0
}
var count int
var writeCount atomic.Int32
for i, sendPkt := range pkts {
pPkt := *pkt
if i != len(pkts)-1 {
@@ -85,13 +93,53 @@ func (r *RedPrimaryReceiver) ForwardRTP(pkt *buffer.ExtPacket, spatialLayer int3
}
pPkt.Packet = sendPkt
if r.highestForwardedExtTimestamp != 0 {
if (pPkt.ExtSequenceNumber > r.highestForwardedExtSequenceNumber && pPkt.ExtTimestamp < r.highestForwardedExtTimestamp) ||
(pPkt.ExtSequenceNumber < r.highestForwardedExtSequenceNumber && pPkt.ExtTimestamp > r.highestForwardedExtTimestamp) {
r.logger.Warnw(
"sequence number OR timestamp inversion, dropping", nil,
"numPackets", len(pkts),
"primaryIncomingSN", pkt.Packet.Header.SequenceNumber,
"primaryIncomingTS", pkt.Packet.Header.Timestamp,
"primaryExtractedSN", pkts[len(pkts)-1].SequenceNumber,
"primaryExtractedTS", pkts[len(pkts)-1].Timestamp,
"primaryESN", pkt.ExtSequenceNumber,
"primaryETS", pkt.ExtTimestamp,
"packetIndex", i,
"packetExtractedSN", pkts[i].SequenceNumber,
"packetESN", pPkt.ExtSequenceNumber,
"packetExtractedTS", pkts[i].Timestamp,
"packetETS", pPkt.ExtTimestamp,
"pktHistory", r.pktHistory,
"redHeader", pkt.Packet.Payload[:10],
"payloadSize", len(pkt.Packet.Payload),
"lastSeq", r.lastSeq,
"highestFowardedExtSequenceNumber", r.highestForwardedExtSequenceNumber,
"highestFowardedExtTimestamp", r.highestForwardedExtTimestamp,
)
continue // drop the packet which causes the inversion
}
}
r.highestForwardedExtSequenceNumber = max(r.highestForwardedExtSequenceNumber, pPkt.ExtSequenceNumber)
r.highestForwardedExtTimestamp = max(r.highestForwardedExtTimestamp, pPkt.ExtTimestamp)
// not modify the ExtPacket.RawPacket here for performance since it is not used by the DownTrack,
// otherwise it should be set to the correct value (marshal the primary rtp packet)
count += r.downTrackSpreader.Broadcast(func(dt TrackSender) {
_ = dt.WriteRTP(&pPkt, spatialLayer)
r.downTrackSpreader.Broadcast(func(dt TrackSender) {
writeCount.Add(dt.WriteRTP(&pPkt, spatialLayer))
})
}
return count
return writeCount.Load()
}
func (r *RedPrimaryReceiver) ForwardRTCPSenderReport(
payloadType webrtc.PayloadType,
layer int32,
publisherSRData *livekit.RTCPSenderReportState,
) {
r.downTrackSpreader.Broadcast(func(dt TrackSender) {
_ = dt.HandleRTCPSenderReportData(payloadType, layer, publisherSRData)
})
}
func (r *RedPrimaryReceiver) AddDownTrack(track TrackSender) error {
@@ -127,6 +175,12 @@ func (r *RedPrimaryReceiver) ResyncDownTracks() {
})
}
func (r *RedPrimaryReceiver) OnStreamRestart() {
r.downTrackSpreader.Broadcast(func(dt TrackSender) {
dt.ReceiverRestart(r)
})
}
func (r *RedPrimaryReceiver) IsClosed() bool {
return r.closed.Load()
}
@@ -191,7 +245,7 @@ func (r *RedPrimaryReceiver) getSendPktsFromRed(rtp *rtp.Packet) ([]*rtp.Packet,
var recoverBits byte
if needRecover {
bitIndex := r.lastSeq - rtp.SequenceNumber
for i := 0; i < maxRedCount; i++ {
for i := range maxRedCount {
if bitIndex > 7 {
break
}
@@ -205,6 +259,8 @@ func (r *RedPrimaryReceiver) getSendPktsFromRed(rtp *rtp.Packet) ([]*rtp.Packet,
return extractPktsFromRed(rtp, recoverBits)
}
// ---------------------------------------
type block struct {
tsOffset uint32
length int
pkg/sfu/redreceiver.go
+33 -9
View File
@@ -21,13 +21,17 @@ import (
"go.uber.org/atomic"
"github.com/pion/rtp"
"github.com/pion/webrtc/v4"
"github.com/livekit/livekit-server/pkg/sfu/buffer"
"github.com/livekit/livekit-server/pkg/sfu/utils"
"github.com/livekit/mediatransportutil/pkg/bucket"
"github.com/livekit/protocol/livekit"
"github.com/livekit/protocol/logger"
)
var _ REDTransformer = (*RedReceiver)(nil)
const (
maxRedCount = 2
mtuSize = 1500
@@ -42,22 +46,22 @@ const (
type RedReceiver struct {
TrackReceiver
downTrackSpreader *DownTrackSpreader
downTrackSpreader *utils.DownTrackSpreader[TrackSender]
logger logger.Logger
closed atomic.Bool
pktBuff [maxRedCount]*rtp.Packet
redPayloadBuf [mtuSize]byte
}
func NewRedReceiver(receiver TrackReceiver, dsp DownTrackSpreaderParams) *RedReceiver {
func NewRedReceiver(receiver TrackReceiver, dsp utils.DownTrackSpreaderParams) REDTransformer {
return &RedReceiver{
TrackReceiver: receiver,
downTrackSpreader: NewDownTrackSpreader(dsp),
downTrackSpreader: utils.NewDownTrackSpreader[TrackSender](dsp),
logger: dsp.Logger,
}
}
func (r *RedReceiver) ForwardRTP(pkt *buffer.ExtPacket, spatialLayer int32) int {
func (r *RedReceiver) ForwardRTP(pkt *buffer.ExtPacket, spatialLayer int32) int32 {
// encode RED payload from primary payload and forward to downtracks
if r.downTrackSpreader.DownTrackCount() == 0 {
return 0
@@ -65,9 +69,11 @@ func (r *RedReceiver) ForwardRTP(pkt *buffer.ExtPacket, spatialLayer int32) int
// fallback to primary codec if payload size exceeds redundant block length
if len(pkt.Packet.Payload) >= maxRedPayload {
return r.downTrackSpreader.Broadcast(func(dt TrackSender) {
_ = dt.WriteRTP(pkt, spatialLayer)
var writeCount atomic.Int32
r.downTrackSpreader.Broadcast(func(dt TrackSender) {
writeCount.Add(dt.WriteRTP(pkt, spatialLayer))
})
return writeCount.Load()
}
redLen, err := r.encodeRedForPrimary(pkt.Packet, r.redPayloadBuf[:])
@@ -78,14 +84,26 @@ func (r *RedReceiver) ForwardRTP(pkt *buffer.ExtPacket, spatialLayer int32) int
pPkt := *pkt
redRtpPacket := *pkt.Packet
redRtpPacket.PayloadType = 63
redRtpPacket.PayloadType = opusRedPT
redRtpPacket.Payload = r.redPayloadBuf[:redLen]
pPkt.Packet = &redRtpPacket
// not modify the ExtPacket.RawPacket here for performance since it is not used by the DownTrack,
// otherwise it should be set to the correct value (marshal the primary rtp packet)
return r.downTrackSpreader.Broadcast(func(dt TrackSender) {
_ = dt.WriteRTP(&pPkt, spatialLayer)
var writeCount atomic.Int32
r.downTrackSpreader.Broadcast(func(dt TrackSender) {
writeCount.Add(dt.WriteRTP(&pPkt, spatialLayer))
})
return writeCount.Load()
}
func (r *RedReceiver) ForwardRTCPSenderReport(
payloadType webrtc.PayloadType,
layer int32,
publisherSRData *livekit.RTCPSenderReportState,
) {
r.downTrackSpreader.Broadcast(func(dt TrackSender) {
_ = dt.HandleRTCPSenderReportData(payloadType, layer, publisherSRData)
})
}
@@ -122,6 +140,12 @@ func (r *RedReceiver) ResyncDownTracks() {
})
}
func (r *RedReceiver) OnStreamRestart() {
r.downTrackSpreader.Broadcast(func(dt TrackSender) {
dt.ReceiverRestart(r)
})
}
func (r *RedReceiver) CanClose() bool {
return r.closed.Load() || r.downTrackSpreader.DownTrackCount() == 0
}
pkg/sfu/redreceiver_test.go
+68 -39
View File
@@ -26,8 +26,7 @@ import (
)
const (
tsStep = uint32(48000 / 1000 * 10)
opusREDPT = 63
tsStep = uint32(48000 / 1000 * 10)
)
type dummyDowntrack struct {
@@ -36,24 +35,26 @@ type dummyDowntrack struct {
receivedPkts []*rtp.Packet
}
func (dt *dummyDowntrack) WriteRTP(p *buffer.ExtPacket, _ int32) error {
func (dt *dummyDowntrack) WriteRTP(p *buffer.ExtPacket, _ int32) int32 {
dt.lastReceivedPkt = p.Packet
dt.receivedPkts = append(dt.receivedPkts, p.Packet)
return nil
return 1
}
func (dt *dummyDowntrack) TrackInfoAvailable() {}
func TestRedReceiver(t *testing.T) {
dt := &dummyDowntrack{TrackSender: &DownTrack{}}
t.Run("normal", func(t *testing.T) {
w := &WebRTCReceiver{
isRED: true,
kind: webrtc.RTPCodecTypeAudio,
logger: logger.GetLogger(),
ReceiverBase: &ReceiverBase{
params: ReceiverBaseParams{
Kind: webrtc.RTPCodecTypeAudio,
Logger: logger.GetLogger(),
},
isRED: true,
},
}
require.Equal(t, w.GetRedReceiver(), w)
require.Equal(t, w.GetRedReceiver(), w.ReceiverBase)
w.isRED = false
red := w.GetRedReceiver().(*RedReceiver)
require.NotNil(t, red)
@@ -75,15 +76,19 @@ func TestRedReceiver(t *testing.T) {
t.Run("packet lost and jump", func(t *testing.T) {
w := &WebRTCReceiver{
kind: webrtc.RTPCodecTypeAudio,
logger: logger.GetLogger(),
ReceiverBase: &ReceiverBase{
params: ReceiverBaseParams{
Kind: webrtc.RTPCodecTypeAudio,
Logger: logger.GetLogger(),
},
},
}
red := w.GetRedReceiver().(*RedReceiver)
require.NoError(t, red.AddDownTrack(dt))
header := rtp.Header{SequenceNumber: 65534, Timestamp: (uint32(1) << 31) - 2*tsStep, PayloadType: 111}
expectPkt := make([]*rtp.Packet, 0, maxRedCount+1)
for i := 0; i < 10; i++ {
for i := range 10 {
if i%2 == 0 {
header.SequenceNumber++
header.Timestamp += tsStep
@@ -126,8 +131,12 @@ func TestRedReceiver(t *testing.T) {
t.Run("unorder and repeat", func(t *testing.T) {
w := &WebRTCReceiver{
kind: webrtc.RTPCodecTypeAudio,
logger: logger.GetLogger(),
ReceiverBase: &ReceiverBase{
params: ReceiverBaseParams{
Kind: webrtc.RTPCodecTypeAudio,
Logger: logger.GetLogger(),
},
},
}
red := w.GetRedReceiver().(*RedReceiver)
require.NoError(t, red.AddDownTrack(dt))
@@ -158,11 +167,15 @@ func TestRedReceiver(t *testing.T) {
t.Run("encoding exceed space", func(t *testing.T) {
w := &WebRTCReceiver{
isRED: true,
kind: webrtc.RTPCodecTypeAudio,
logger: logger.GetLogger(),
ReceiverBase: &ReceiverBase{
params: ReceiverBaseParams{
Kind: webrtc.RTPCodecTypeAudio,
Logger: logger.GetLogger(),
},
isRED: true,
},
}
require.Equal(t, w.GetRedReceiver(), w)
require.Equal(t, w.GetRedReceiver(), w.ReceiverBase)
w.isRED = false
red := w.GetRedReceiver().(*RedReceiver)
require.NotNil(t, red)
@@ -183,11 +196,15 @@ func TestRedReceiver(t *testing.T) {
t.Run("large timestamp gap", func(t *testing.T) {
w := &WebRTCReceiver{
isRED: true,
kind: webrtc.RTPCodecTypeAudio,
logger: logger.GetLogger(),
ReceiverBase: &ReceiverBase{
params: ReceiverBaseParams{
Kind: webrtc.RTPCodecTypeAudio,
Logger: logger.GetLogger(),
},
isRED: true,
},
}
require.Equal(t, w.GetRedReceiver(), w)
require.Equal(t, w.GetRedReceiver(), w.ReceiverBase)
w.isRED = false
red := w.GetRedReceiver().(*RedReceiver)
require.NotNil(t, red)
@@ -247,7 +264,7 @@ func verifyEncodingEqual(t *testing.T, p1, p2 *rtp.Packet) {
func generatePkts(header rtp.Header, count int, tsStep uint32) []*rtp.Packet {
pkts := make([]*rtp.Packet, 0, count)
for i := 0; i < count; i++ {
for range count {
hbuf, _ := header.Marshal()
pkts = append(pkts, &rtp.Packet{
Header: header,
@@ -271,7 +288,7 @@ func generateRedPkts(t *testing.T, pkts []*rtp.Packet, redCount int) []*rtp.Pack
}
buf := make([]byte, mtuSize)
redPkt := *pkt
redPkt.PayloadType = opusREDPT
redPkt.PayloadType = opusRedPT
encoded, err := encodeRedForPrimary(encodingPkts, pkt, buf)
require.NoError(t, err)
redPkt.Payload = buf[:encoded]
@@ -283,11 +300,15 @@ func generateRedPkts(t *testing.T, pkts []*rtp.Packet, redCount int) []*rtp.Pack
func testRedRedPrimaryReceiver(t *testing.T, maxPktCount, redCount int, sendPktIdx, expectPktIdx []int) {
dt := &dummyDowntrack{TrackSender: &DownTrack{}}
w := &WebRTCReceiver{
kind: webrtc.RTPCodecTypeAudio,
logger: logger.GetLogger(),
codec: webrtc.RTPCodecParameters{PayloadType: opusREDPT, RTPCodecCapability: webrtc.RTPCodecCapability{MimeType: "audio/red"}},
ReceiverBase: &ReceiverBase{
params: ReceiverBaseParams{
Kind: webrtc.RTPCodecTypeAudio,
Logger: logger.GetLogger(),
Codec: webrtc.RTPCodecParameters{PayloadType: opusRedPT, RTPCodecCapability: webrtc.RTPCodecCapability{MimeType: "audio/red"}},
},
},
}
require.Equal(t, w.GetPrimaryReceiverForRed(), w)
require.Equal(t, w.GetPrimaryReceiverForRed(), w.ReceiverBase)
w.isRED = true
red := w.GetPrimaryReceiverForRed().(*RedPrimaryReceiver)
require.NotNil(t, red)
@@ -313,10 +334,14 @@ func testRedRedPrimaryReceiver(t *testing.T, maxPktCount, redCount int, sendPktI
func TestRedPrimaryReceiver(t *testing.T) {
w := &WebRTCReceiver{
kind: webrtc.RTPCodecTypeAudio,
logger: logger.GetLogger(),
ReceiverBase: &ReceiverBase{
params: ReceiverBaseParams{
Kind: webrtc.RTPCodecTypeAudio,
Logger: logger.GetLogger(),
},
},
}
require.Equal(t, w.GetPrimaryReceiverForRed(), w)
require.Equal(t, w.GetPrimaryReceiverForRed(), w.ReceiverBase)
w.isRED = true
red := w.GetPrimaryReceiverForRed().(*RedPrimaryReceiver)
require.NotNil(t, red)
@@ -324,7 +349,7 @@ func TestRedPrimaryReceiver(t *testing.T) {
t.Run("packet should send only once", func(t *testing.T) {
maxPktCount := 19
var sendPktIndex []int
for i := 0; i < maxPktCount; i++ {
for i := range maxPktCount {
sendPktIndex = append(sendPktIndex, i)
}
testRedRedPrimaryReceiver(t, maxPktCount, maxRedCount, sendPktIndex, sendPktIndex)
@@ -333,7 +358,7 @@ func TestRedPrimaryReceiver(t *testing.T) {
t.Run("packet duplicate and unorder", func(t *testing.T) {
maxPktCount := 19
var sendPktIndex []int
for i := 0; i < maxPktCount; i++ {
for i := range maxPktCount {
sendPktIndex = append(sendPktIndex, i)
if i > 0 {
sendPktIndex = append(sendPktIndex, i-1)
@@ -346,7 +371,7 @@ func TestRedPrimaryReceiver(t *testing.T) {
t.Run("full recover", func(t *testing.T) {
maxPktCount := 19
var sendPktIndex, recvPktIndex []int
for i := 0; i < maxPktCount; i++ {
for i := range maxPktCount {
recvPktIndex = append(recvPktIndex, i)
// drop packets covered by red encoding
@@ -383,11 +408,15 @@ func TestRedPrimaryReceiver(t *testing.T) {
t.Run("mixed primary codec", func(t *testing.T) {
dt := &dummyDowntrack{TrackSender: &DownTrack{}}
w := &WebRTCReceiver{
kind: webrtc.RTPCodecTypeAudio,
logger: logger.GetLogger(),
codec: webrtc.RTPCodecParameters{PayloadType: opusREDPT, RTPCodecCapability: webrtc.RTPCodecCapability{MimeType: "audio/red"}},
ReceiverBase: &ReceiverBase{
params: ReceiverBaseParams{
Kind: webrtc.RTPCodecTypeAudio,
Logger: logger.GetLogger(),
Codec: webrtc.RTPCodecParameters{PayloadType: opusRedPT, RTPCodecCapability: webrtc.RTPCodecCapability{MimeType: "audio/red"}},
},
},
}
require.Equal(t, w.GetPrimaryReceiverForRed(), w)
require.Equal(t, w.GetPrimaryReceiverForRed(), w.ReceiverBase)
w.isRED = true
red := w.GetPrimaryReceiverForRed().(*RedPrimaryReceiver)
require.NotNil(t, red)
pkg/sfu/rtpextension/dependencydescriptor/bitstreamwriter.go
+1 -1
View File
@@ -103,7 +103,7 @@ func (w *BitStreamWriter) writePartialByte(source uint8, sourceBitCount int, tar
}
func (w *BitStreamWriter) WriteNonSymmetric(val, numValues uint32) error {
if !(val < numValues && numValues <= 1<<31) {
if val >= numValues || numValues > 1<<31 {
return fmt.Errorf("invalid argument, val %d, numValues %d", val, numValues)
}
if numValues == 1 {
pkg/sfu/rtpextension/dependencydescriptor/dependencydescriptorreader.go
+4 -3
View File
@@ -229,12 +229,13 @@ func (r *DependencyDescriptorReader) readTemplateLayers() error {
}
nextLayerIdc = nextLayerIdcType(idc)
if nextLayerIdc == nextTemporalLayer {
switch nextLayerIdc {
case nextTemporalLayer:
temporalId++
if temporalId >= MaxTemporalIds {
return ErrDDReaderTooManyTemporalLayers
}
} else if nextLayerIdc == nextSpatialLayer {
case nextSpatialLayer:
spatialId++
temporalId = 0
if spatialId >= MaxSpatialIds {
@@ -242,7 +243,7 @@ func (r *DependencyDescriptorReader) readTemplateLayers() error {
}
}
if !(nextLayerIdc != noMoreLayer && r.buffer.Ok()) {
if nextLayerIdc == noMoreLayer || !r.buffer.Ok() {
break
}
}
pkg/sfu/rtpextension/dependencydescriptor/dependencydescriptorwriter.go
+5 -6
View File
@@ -16,8 +16,7 @@ package dependencydescriptor
import (
"fmt"
"golang.org/x/exp/slices"
"slices"
)
type TemplateMatch struct {
@@ -246,8 +245,8 @@ func (w *DependencyDescriptorWriter) writeExtendedFields() error {
}
func (w *DependencyDescriptorWriter) writeTemplateDependencyStructure() error {
if !(w.structure.StructureId >= 0 && w.structure.StructureId < MaxTemplates &&
w.structure.NumDecodeTargets > 0 && w.structure.NumDecodeTargets <= MaxDecodeTargets) {
if w.structure.StructureId < 0 || w.structure.StructureId >= MaxTemplates ||
w.structure.NumDecodeTargets <= 0 || w.structure.NumDecodeTargets > MaxDecodeTargets {
return fmt.Errorf("invalid arguments, structureId: %d, numDecodeTargets: %d", w.structure.StructureId, w.structure.NumDecodeTargets)
}
@@ -283,8 +282,8 @@ func (w *DependencyDescriptorWriter) writeTemplateDependencyStructure() error {
}
func (w *DependencyDescriptorWriter) writeTemplateLayers() error {
if !(len(w.structure.Templates) > 0 && len(w.structure.Templates) <= MaxTemplates &&
w.structure.Templates[0].SpatialId == 0 && w.structure.Templates[0].TemporalId == 0) {
if len(w.structure.Templates) <= 0 || len(w.structure.Templates) > MaxTemplates ||
w.structure.Templates[0].SpatialId != 0 || w.structure.Templates[0].TemporalId != 0 {
return fmt.Errorf("invalid templates, len %d, templates[0]: spatialId %d, temporalId %d", len(w.structure.Templates), w.structure.Templates[0].SpatialId, w.structure.Templates[0].TemporalId)
}
for i := 1; i < len(w.structure.Templates); i++ {
pkg/sfu/rtpmunger.go
+21 -10
View File
@@ -77,8 +77,8 @@ func NewRTPMunger(logger logger.Logger) *RTPMunger {
}
}
func (r *RTPMunger) DebugInfo() map[string]interface{} {
return map[string]interface{}{
func (r *RTPMunger) DebugInfo() map[string]any {
return map[string]any{
"ExtHighestIncomingSN": r.extHighestIncomingSN,
"ExtLastSN": r.extLastSN,
"ExtSecondLastSN": r.extSecondLastSN,
@@ -102,6 +102,10 @@ func (r *RTPMunger) GetState() *livekit.RTPMungerState {
}
}
func (r *RTPMunger) GetExtLastTimestamp() uint64 {
return r.extLastTS
}
func (r *RTPMunger) GetTSOffset() uint64 {
return r.tsOffset
}
@@ -185,7 +189,7 @@ func (r *RTPMunger) UpdateAndGetSnTs(extPkt *buffer.ExtPacket, marker bool) (Tra
r.secondLastMarker = r.lastMarker
r.lastMarker = marker
if extPkt.KeyFrame {
if extPkt.IsKeyFrame {
r.extRtxGateSn = extMungedSN
r.isInRtxGateRegion = true
}
@@ -207,7 +211,7 @@ func (r *RTPMunger) UpdateAndGetSnTs(extPkt *buffer.ExtPacket, marker bool) (Tra
if err != nil {
return TranslationParamsRTP{
snOrdering: SequenceNumberOrderingOutOfOrder,
}, ErrOutOfOrderSequenceNumberCacheMiss
}, errOutOfOrderSequenceNumberCacheMiss
}
extSequenceNumber := extPkt.ExtSequenceNumber - snOffset
@@ -223,7 +227,7 @@ func (r *RTPMunger) UpdateAndGetSnTs(extPkt *buffer.ExtPacket, marker bool) (Tra
)
return TranslationParamsRTP{
snOrdering: SequenceNumberOrderingOutOfOrder,
}, ErrOutOfOrderSequenceNumberCacheMiss
}, errOutOfOrderSequenceNumberCacheMiss
}
return TranslationParamsRTP{
@@ -245,13 +249,13 @@ func (r *RTPMunger) UpdateAndGetSnTs(extPkt *buffer.ExtPacket, marker bool) (Tra
return TranslationParamsRTP{
snOrdering: SequenceNumberOrderingContiguous,
}, ErrPaddingOnlyPacket
}, errPaddingOnlyPacket
}
// can get duplicate packet due to FEC
return TranslationParamsRTP{
snOrdering: SequenceNumberOrderingDuplicate,
}, ErrDuplicatePacket
}, errDuplicatePacket
}
func (r *RTPMunger) FilterRTX(nacks []uint16) []uint16 {
@@ -269,7 +273,13 @@ func (r *RTPMunger) FilterRTX(nacks []uint16) []uint16 {
return filtered
}
func (r *RTPMunger) UpdateAndGetPaddingSnTs(num int, clockRate uint32, frameRate uint32, forceMarker bool, extRtpTimestamp uint64) ([]SnTs, error) {
func (r *RTPMunger) UpdateAndGetPaddingSnTs(
num int,
clockRate uint32,
frameRate uint32,
forceMarker bool,
extRtpTimestamp uint64,
) ([]SnTs, error) {
if num == 0 {
return nil, nil
}
@@ -278,7 +288,7 @@ func (r *RTPMunger) UpdateAndGetPaddingSnTs(num int, clockRate uint32, frameRate
tsOffset := 0
if !r.lastMarker {
if !forceMarker {
return nil, ErrPaddingNotOnFrameBoundary
return nil, errPaddingNotOnFrameBoundary
}
// if forcing frame end, use timestamp of latest received frame for the first one
@@ -289,7 +299,7 @@ func (r *RTPMunger) UpdateAndGetPaddingSnTs(num int, clockRate uint32, frameRate
extLastSN := r.extLastSN
extLastTS := r.extLastTS
vals := make([]SnTs, num)
for i := 0; i < num; i++ {
for i := range num {
extLastSN++
vals[i].extSequenceNumber = extLastSN
@@ -322,6 +332,7 @@ func (r *RTPMunger) UpdateAndGetPaddingSnTs(num int, clockRate uint32, frameRate
r.tsOffset -= extLastTS - r.extLastTS
r.extLastTS = extLastTS
r.secondLastMarker = r.lastMarker
if forceMarker {
r.lastMarker = true
}
pkg/sfu/rtpmunger_test.go
+20 -18
View File
@@ -44,9 +44,9 @@ func TestSetLastSnTs(t *testing.T) {
require.Equal(t, uint64(23332), r.extHighestIncomingSN)
require.Equal(t, uint64(23333), r.extLastSN)
require.Equal(t, uint64(0xabcdef), r.extLastTS)
snOffset, err := r.snRangeMap.GetValue(r.extHighestIncomingSN)
_, err = r.snRangeMap.GetValue(r.extHighestIncomingSN)
require.Error(t, err)
snOffset, err = r.snRangeMap.GetValue(r.extLastSN)
snOffset, err := r.snRangeMap.GetValue(r.extLastSN)
require.NoError(t, err)
require.Equal(t, uint64(0), snOffset)
require.Equal(t, uint64(0), r.snOffset)
@@ -96,9 +96,9 @@ func TestPacketDropped(t *testing.T) {
require.Equal(t, uint64(23332), r.extHighestIncomingSN)
require.Equal(t, uint64(23333), r.extLastSN)
require.Equal(t, uint64(0xabcdef), r.extLastTS)
snOffset, err := r.snRangeMap.GetValue(r.extHighestIncomingSN)
_, err := r.snRangeMap.GetValue(r.extHighestIncomingSN)
require.Error(t, err)
snOffset, err = r.snRangeMap.GetValue(r.extLastSN)
snOffset, err := r.snRangeMap.GetValue(r.extLastSN)
require.NoError(t, err)
require.Equal(t, uint64(0), snOffset)
require.Equal(t, uint64(0), r.tsOffset)
@@ -133,7 +133,7 @@ func TestPacketDropped(t *testing.T) {
r.PacketDropped(extPkt)
require.Equal(t, uint64(23333), r.extLastSN)
snOffset, err = r.snRangeMap.GetValue(r.extHighestIncomingSN)
_, err = r.snRangeMap.GetValue(r.extHighestIncomingSN)
require.Error(t, err)
snOffset, err = r.snRangeMap.GetValue(r.extHighestIncomingSN + 1)
require.NoError(t, err)
@@ -180,9 +180,10 @@ func TestOutOfOrderSequenceNumber(t *testing.T) {
}
extPkt, _ = testutils.GetTestExtPacket(params)
tp, err := r.UpdateAndGetSnTs(extPkt, extPkt.Packet.Marker)
_, err = r.UpdateAndGetSnTs(extPkt, extPkt.Packet.Marker)
require.Error(t, err)
var tp TranslationParamsRTP
// add a missing sequence number to the cache
err = r.snRangeMap.ExcludeRange(23334, 23335)
require.NoError(t, err)
@@ -228,7 +229,7 @@ func TestOutOfOrderSequenceNumber(t *testing.T) {
}
tp, err = r.UpdateAndGetSnTs(extPkt, extPkt.Packet.Marker)
require.Error(t, err, ErrOutOfOrderSequenceNumberCacheMiss)
require.Error(t, err, errOutOfOrderSequenceNumberCacheMiss)
require.Equal(t, tpExpected, tp)
}
@@ -252,7 +253,7 @@ func TestDuplicateSequenceNumber(t *testing.T) {
}
tp, err := r.UpdateAndGetSnTs(extPkt, extPkt.Packet.Marker)
require.ErrorIs(t, err, ErrDuplicatePacket)
require.ErrorIs(t, err, errDuplicatePacket)
require.Equal(t, tpExpected, tp)
}
@@ -274,13 +275,14 @@ func TestPaddingOnlyPacket(t *testing.T) {
tp, err := r.UpdateAndGetSnTs(extPkt, extPkt.Packet.Marker)
require.Error(t, err)
require.ErrorIs(t, err, ErrPaddingOnlyPacket)
require.ErrorIs(t, err, errPaddingOnlyPacket)
require.Equal(t, tpExpected, tp)
require.Equal(t, uint64(23333), r.extHighestIncomingSN)
require.Equal(t, uint64(23333), r.extLastSN)
snOffset, err := r.snRangeMap.GetValue(r.extHighestIncomingSN)
_, err = r.snRangeMap.GetValue(r.extHighestIncomingSN)
require.Error(t, err)
var snOffset uint64
// padding only packet with a gap should not report an error
params = &testutils.TestExtPacketParams{
SequenceNumber: 23335,
@@ -366,11 +368,11 @@ func TestGapInSequenceNumber(t *testing.T) {
}
tp, err = r.UpdateAndGetSnTs(extPkt, extPkt.Packet.Marker)
require.ErrorIs(t, err, ErrPaddingOnlyPacket)
require.ErrorIs(t, err, errPaddingOnlyPacket)
require.Equal(t, tpExpected, tp)
require.Equal(t, uint64(65536+2), r.extHighestIncomingSN)
require.Equal(t, uint64(65536+1), r.extLastSN)
snOffset, err = r.snRangeMap.GetValue(r.extHighestIncomingSN)
_, err = r.snRangeMap.GetValue(r.extHighestIncomingSN)
require.Error(t, err)
// a packet with a gap should be adjusting for dropped padding packet
@@ -417,11 +419,11 @@ func TestGapInSequenceNumber(t *testing.T) {
}
tp, err = r.UpdateAndGetSnTs(extPkt, extPkt.Packet.Marker)
require.ErrorIs(t, err, ErrPaddingOnlyPacket)
require.ErrorIs(t, err, errPaddingOnlyPacket)
require.Equal(t, tpExpected, tp)
require.Equal(t, uint64(65536+5), r.extHighestIncomingSN)
require.Equal(t, uint64(65536+3), r.extLastSN)
snOffset, err = r.snRangeMap.GetValue(r.extHighestIncomingSN)
_, err = r.snRangeMap.GetValue(r.extHighestIncomingSN)
require.Error(t, err)
// a packet with a gap should be adjusting for dropped packets
@@ -521,7 +523,7 @@ func TestUpdateAndGetPaddingSnTs(t *testing.T) {
// getting padding without forcing marker should fail
_, err := r.UpdateAndGetPaddingSnTs(10, 10, 5, false, 0)
require.Error(t, err)
require.ErrorIs(t, err, ErrPaddingNotOnFrameBoundary)
require.ErrorIs(t, err, errPaddingNotOnFrameBoundary)
// forcing a marker should not error out.
// And timestamp on first padding should be the same as the last one.
@@ -529,7 +531,7 @@ func TestUpdateAndGetPaddingSnTs(t *testing.T) {
clockRate := uint64(10)
frameRate := uint64(5)
var sntsExpected = make([]SnTs, numPadding)
for i := 0; i < numPadding; i++ {
for i := range numPadding {
sntsExpected[i] = SnTs{
extSequenceNumber: uint64(params.SequenceNumber) + uint64(i) + 1,
extTimestamp: uint64(params.Timestamp) + ((uint64(i)*clockRate)+frameRate-1)/frameRate,
@@ -540,7 +542,7 @@ func TestUpdateAndGetPaddingSnTs(t *testing.T) {
require.Equal(t, sntsExpected, snts)
// now that there is a marker, timestamp should jump on first padding when asked again
for i := 0; i < numPadding; i++ {
for i := range numPadding {
sntsExpected[i] = SnTs{
extSequenceNumber: uint64(params.SequenceNumber) + uint64(len(snts)) + uint64(i) + 1,
extTimestamp: snts[len(snts)-1].extTimestamp + ((uint64(i+1)*clockRate)+frameRate-1)/frameRate,
@@ -570,7 +572,7 @@ func TestIsOnFrameBoundary(t *testing.T) {
// packet with RTP marker
params = &testutils.TestExtPacketParams{
SetMarker: true,
Marker: true,
SequenceNumber: 23334,
Timestamp: 0xabcdef,
SSRC: 0x12345678,
pkg/sfu/rtpstats/rtpstats_base.go
+90 -56
View File
@@ -25,13 +25,12 @@ import (
"github.com/livekit/protocol/livekit"
"github.com/livekit/protocol/utils"
"github.com/livekit/protocol/utils/mono"
"github.com/livekit/protocol/utils/rtputil"
)
const (
cFirstPacketTimeAdjustWindow = 2 * time.Minute
cFirstPacketTimeAdjustThreshold = 15 * 1e9
cSequenceNumberLargeJumpThreshold = 100
)
// -------------------------------------------------------
@@ -192,6 +191,8 @@ func (w WrappedRTCPSenderReportStateLogger) MarshalLogObject(e zapcore.ObjectEnc
return nil
}
// ------------------------------------------------------------------
func RTCPSenderReportPropagationDelay(rsrs *livekit.RTCPSenderReportState, passThrough bool) time.Duration {
if passThrough {
return 0
@@ -205,6 +206,8 @@ func RTCPSenderReportPropagationDelay(rsrs *livekit.RTCPSenderReportState, passT
type rtpStatsBase struct {
*rtpStatsBaseLite
rtpConverter *rtputil.RTPConverter
firstTime int64
firstTimeAdjustment time.Duration
highestTime int64
@@ -251,6 +254,18 @@ func newRTPStatsBase(params RTPStatsParams) *rtpStatsBase {
}
}
func (r *rtpStatsBase) SetClockRate(clockRate uint32) {
r.lock.Lock()
defer r.lock.Unlock()
r.setClockRateLocked(clockRate)
}
func (r *rtpStatsBase) setClockRateLocked(clockRate uint32) {
r.rtpConverter = rtputil.NewRTPConverter(int64(clockRate))
r.rtpStatsBaseLite.setClockRateLocked(clockRate)
}
func (r *rtpStatsBase) seed(from *rtpStatsBase) bool {
if !r.rtpStatsBaseLite.seed(from.rtpStatsBaseLite) {
return false
@@ -346,6 +361,13 @@ func (r *rtpStatsBase) UpdateKeyFrame(kfCount uint32) {
r.lastKeyFrame = time.Now()
}
func (r *rtpStatsBase) KeyFrame() (uint32, time.Time) {
r.lock.RLock()
defer r.lock.RUnlock()
return r.keyFrames, r.lastKeyFrame
}
func (r *rtpStatsBase) UpdateRtt(rtt uint32) {
r.lock.Lock()
defer r.lock.Unlock()
@@ -374,7 +396,11 @@ func (r *rtpStatsBase) GetRtt() uint32 {
return r.rtt
}
func (r *rtpStatsBase) maybeAdjustFirstPacketTime(srData *livekit.RTCPSenderReportState, tsOffset uint64, extStartTS uint64) (err error, loggingFields []interface{}) {
func (r *rtpStatsBase) maybeAdjustFirstPacketTime(
srData *livekit.RTCPSenderReportState,
tsOffset uint64,
extStartTS uint64,
) (adjustment int64, loggingFields []any, err error) {
nowNano := mono.UnixNano()
if time.Duration(nowNano-r.startTime) > cFirstPacketTimeAdjustWindow {
return
@@ -387,49 +413,52 @@ func (r *rtpStatsBase) maybeAdjustFirstPacketTime(srData *livekit.RTCPSenderRepo
// in some network element along the way), push back first time
// to an earlier instance.
timeSinceReceive := time.Duration(nowNano - srData.AtAdjusted)
extNowTS := srData.RtpTimestampExt - tsOffset + uint64(timeSinceReceive.Nanoseconds()*int64(r.params.ClockRate)/1e9)
extNowTS := srData.RtpTimestampExt - tsOffset + r.rtpConverter.ToRTPExt(timeSinceReceive)
samplesDiff := int64(extNowTS - extStartTS)
if samplesDiff < 0 {
// out-of-order, skip
return
}
samplesDuration := time.Duration(float64(samplesDiff) / float64(r.params.ClockRate) * float64(time.Second))
samplesDuration := r.rtpConverter.ToDurationExt(uint64(samplesDiff))
timeSinceFirst := time.Duration(nowNano - r.firstTime)
now := r.firstTime + timeSinceFirst.Nanoseconds()
firstTime := now - samplesDuration.Nanoseconds()
getFields := func() []interface{} {
return []interface{}{
"startTime", time.Unix(0, r.startTime),
"nowTime", time.Unix(0, now),
"before", time.Unix(0, r.firstTime),
"after", time.Unix(0, firstTime),
"adjustment", time.Duration(r.firstTime - firstTime),
"extNowTS", extNowTS,
"extStartTS", extStartTS,
"srData", WrappedRTCPSenderReportStateLogger{srData},
"tsOffset", tsOffset,
"timeSinceReceive", timeSinceReceive,
"timeSinceFirst", timeSinceFirst,
"samplesDiff", samplesDiff,
"samplesDuration", samplesDuration,
}
}
adjustment = r.firstTime - firstTime
if firstTime < r.firstTime {
if r.firstTime-firstTime > cFirstPacketTimeAdjustThreshold {
if adjustment > cFirstPacketTimeAdjustThreshold {
err = errors.New("adjusting first packet time, too big, ignoring")
loggingFields = getFields()
loggingFields = r.firstPacketTimeAdjustLogFields(now, firstTime, adjustment, extNowTS, extStartTS, srData, tsOffset, timeSinceReceive, timeSinceFirst, samplesDuration, samplesDiff)
} else {
r.logger.Debugw("adjusting first packet time", getFields()...)
r.firstTimeAdjustment += time.Duration(r.firstTime - firstTime)
r.firstTimeAdjustment += time.Duration(adjustment)
r.logger.Debugw("adjusting first packet time", r.firstPacketTimeAdjustLogFields(now, firstTime, adjustment, extNowTS, extStartTS, srData, tsOffset, timeSinceReceive, timeSinceFirst, samplesDuration, samplesDiff)...)
r.firstTime = firstTime
}
}
return
}
func (r *rtpStatsBase) firstPacketTimeAdjustLogFields(now, firstTime int64, adjustment int64, extNowTS, extStartTS uint64, srData *livekit.RTCPSenderReportState, tsOffset uint64, timeSinceReceive, timeSinceFirst, samplesDuration time.Duration, samplesDiff int64) []any {
return []any{
"startTime", time.Unix(0, r.startTime),
"nowTime", time.Unix(0, now),
"before", time.Unix(0, r.firstTime),
"after", time.Unix(0, firstTime),
"adjustment", time.Duration(adjustment),
"firstTimeAdjustment", r.firstTimeAdjustment,
"extNowTS", extNowTS,
"extStartTS", extStartTS,
"srData", WrappedRTCPSenderReportStateLogger{srData},
"tsOffset", tsOffset,
"timeSinceReceive", timeSinceReceive,
"timeSinceFirst", timeSinceFirst,
"samplesDiff", samplesDiff,
"samplesDuration", samplesDuration,
}
}
func (r *rtpStatsBase) getPacketsSeenMinusPadding(extStartSN, extHighestSN uint64) uint64 {
packetsSeen := r.getPacketsSeen(extStartSN, extHighestSN)
if r.packetsPadding > packetsSeen {
@@ -447,9 +476,13 @@ func (r *rtpStatsBase) deltaInfo(
snapshotID uint32,
extStartSN uint64,
extHighestSN uint64,
) (deltaInfo *RTPDeltaInfo, err error, loggingFields []interface{}) {
then, now := r.getAndResetSnapshot(snapshotID, extStartSN, extHighestSN)
if now == nil || then == nil {
) (deltaInfo *RTPDeltaInfo, loggingFields []any, err error) {
if r.clockRate == 0 {
return
}
then, now, ok := r.getAndResetSnapshot(snapshotID, extStartSN, extHighestSN)
if !ok {
return
}
@@ -461,7 +494,7 @@ func (r *rtpStatsBase) deltaInfo(
packetsExpected = 0
}
if packetsExpected > cNumSequenceNumbers {
loggingFields = []interface{}{
loggingFields = []any{
"snapshotID", snapshotID,
"snapshotNow", now,
"snapshotThen", then,
@@ -487,7 +520,7 @@ func (r *rtpStatsBase) deltaInfo(
// padding packets delta could be higher than expected due to out-of-order padding packets
packetsPadding := now.packetsPadding - then.packetsPadding
if packetsExpected < packetsPadding {
loggingFields = []interface{}{
loggingFields = []any{
"snapshotID", snapshotID,
"snapshotNow", now,
"snapshotThen", then,
@@ -518,7 +551,7 @@ func (r *rtpStatsBase) deltaInfo(
PacketsOutOfOrder: uint32(now.packetsOutOfOrder - then.packetsOutOfOrder),
Frames: now.frames - then.frames,
RttMax: then.maxRtt,
JitterMax: then.maxJitter / float64(r.params.ClockRate) * 1e6,
JitterMax: then.maxJitter / float64(r.clockRate) * 1e6,
Nacks: now.nacks - then.nacks,
Plis: now.plis - then.plis,
Firs: now.firs - then.firs,
@@ -614,8 +647,8 @@ func (r *rtpStatsBase) toProto(
p.KeyFrames = r.keyFrames
p.LastKeyFrame = timestamppb.New(r.lastKeyFrame)
p.JitterCurrent = jitter / float64(r.params.ClockRate) * 1e6
p.JitterMax = maxJitter / float64(r.params.ClockRate) * 1e6
p.JitterCurrent = jitter / float64(r.clockRate) * 1e6
p.JitterMax = maxJitter / float64(r.clockRate) * 1e6
p.Firs = r.firs
p.LastFir = timestamppb.New(r.lastFir)
@@ -635,9 +668,9 @@ func (r *rtpStatsBase) updateJitter(ets uint64, packetTime int64) float64 {
// p1f1 -> p1f2 -> p2f1
// In this case, p2f1 (packet 2, frame 1) will still be used in jitter calculation
// although it is the second packet of a frame because of out-of-order receival.
if r.lastJitterExtTimestamp != ets {
if r.lastJitterExtTimestamp != ets && r.rtpConverter != nil {
timeSinceFirst := packetTime - r.firstTime
packetTimeRTP := uint64(timeSinceFirst * int64(r.params.ClockRate) / 1e9)
packetTimeRTP := r.rtpConverter.ToRTPExt(time.Duration(timeSinceFirst))
transit := packetTimeRTP - ets
if r.lastTransit != 0 {
@@ -661,9 +694,9 @@ func (r *rtpStatsBase) updateJitter(ets uint64, packetTime int64) float64 {
return r.jitter
}
func (r *rtpStatsBase) getAndResetSnapshot(snapshotID uint32, extStartSN uint64, extHighestSN uint64) (*snapshot, *snapshot) {
if !r.initialized {
return nil, nil
func (r *rtpStatsBase) getAndResetSnapshot(snapshotID uint32, extStartSN uint64, extHighestSN uint64) (snapshot, snapshot, bool) {
if !r.initialized || snapshotID < cFirstSnapshotID {
return snapshot{}, snapshot{}, false
}
idx := snapshotID - cFirstSnapshotID
@@ -676,7 +709,7 @@ func (r *rtpStatsBase) getAndResetSnapshot(snapshotID uint32, extStartSN uint64,
// snapshot now
now := r.getSnapshot(mono.UnixNano(), extHighestSN+1)
r.snapshots[idx] = now
return &then, &now
return then, now, true
}
func (r *rtpStatsBase) getDrift(extStartTS, extHighestTS uint64) (
@@ -686,21 +719,20 @@ func (r *rtpStatsBase) getDrift(extStartTS, extHighestTS uint64) (
rebasedReportDrift *livekit.RTPDrift,
) {
if r.firstTime != 0 {
elapsed := r.highestTime - r.firstTime
elapsed := time.Duration(r.highestTime - r.firstTime)
rtpClockTicks := extHighestTS - extStartTS
driftSamples := int64(rtpClockTicks - uint64(elapsed*int64(r.params.ClockRate)/1e9))
driftSamples := int64(rtpClockTicks - r.rtpConverter.ToRTPExt(elapsed))
if elapsed > 0 {
elapsedSeconds := time.Duration(elapsed).Seconds()
packetDrift = &livekit.RTPDrift{
StartTime: timestamppb.New(time.Unix(0, r.firstTime)),
EndTime: timestamppb.New(time.Unix(0, r.highestTime)),
Duration: elapsedSeconds,
Duration: elapsed.Seconds(),
StartTimestamp: extStartTS,
EndTimestamp: extHighestTS,
RtpClockTicks: rtpClockTicks,
DriftSamples: driftSamples,
DriftMs: (float64(driftSamples) * 1000) / float64(r.params.ClockRate),
ClockRate: float64(rtpClockTicks) / elapsedSeconds,
DriftMs: (float64(driftSamples) * 1000) / float64(r.clockRate),
ClockRate: float64(rtpClockTicks) / elapsed.Seconds(),
}
}
}
@@ -708,25 +740,27 @@ func (r *rtpStatsBase) getDrift(extStartTS, extHighestTS uint64) (
if r.srFirst != nil && r.srNewest != nil && r.srFirst.RtpTimestamp != r.srNewest.RtpTimestamp {
rtpClockTicks := r.srNewest.RtpTimestampExt - r.srFirst.RtpTimestampExt
elapsed := mediatransportutil.NtpTime(r.srNewest.NtpTimestamp).Time().Sub(mediatransportutil.NtpTime(r.srFirst.NtpTimestamp).Time())
srFirstTime := mediatransportutil.NtpTime(r.srFirst.NtpTimestamp).Time()
srNewestTime := mediatransportutil.NtpTime(r.srNewest.NtpTimestamp).Time()
elapsed := srNewestTime.Sub(srFirstTime)
if elapsed.Seconds() > 0.0 {
driftSamples := int64(rtpClockTicks - uint64(elapsed.Nanoseconds()*int64(r.params.ClockRate)/1e9))
driftSamples := int64(rtpClockTicks - r.rtpConverter.ToRTPExt(elapsed))
ntpReportDrift = &livekit.RTPDrift{
StartTime: timestamppb.New(mediatransportutil.NtpTime(r.srFirst.NtpTimestamp).Time()),
EndTime: timestamppb.New(mediatransportutil.NtpTime(r.srNewest.NtpTimestamp).Time()),
StartTime: timestamppb.New(srFirstTime),
EndTime: timestamppb.New(srNewestTime),
Duration: elapsed.Seconds(),
StartTimestamp: r.srFirst.RtpTimestampExt,
EndTimestamp: r.srNewest.RtpTimestampExt,
RtpClockTicks: rtpClockTicks,
DriftSamples: driftSamples,
DriftMs: (float64(driftSamples) * 1000) / float64(r.params.ClockRate),
DriftMs: (float64(driftSamples) * 1000) / float64(r.clockRate),
ClockRate: float64(rtpClockTicks) / elapsed.Seconds(),
}
}
elapsed = time.Duration(r.srNewest.At - r.srFirst.At)
if elapsed.Seconds() > 0.0 {
driftSamples := int64(rtpClockTicks - uint64(elapsed.Nanoseconds()*int64(r.params.ClockRate)/1e9))
driftSamples := int64(rtpClockTicks - r.rtpConverter.ToRTPExt(elapsed))
receivedReportDrift = &livekit.RTPDrift{
StartTime: timestamppb.New(time.Unix(0, r.srFirst.At)),
EndTime: timestamppb.New(time.Unix(0, r.srNewest.At)),
@@ -735,14 +769,14 @@ func (r *rtpStatsBase) getDrift(extStartTS, extHighestTS uint64) (
EndTimestamp: r.srNewest.RtpTimestampExt,
RtpClockTicks: rtpClockTicks,
DriftSamples: driftSamples,
DriftMs: (float64(driftSamples) * 1000) / float64(r.params.ClockRate),
DriftMs: (float64(driftSamples) * 1000) / float64(r.clockRate),
ClockRate: float64(rtpClockTicks) / elapsed.Seconds(),
}
}
elapsed = time.Duration(r.srNewest.AtAdjusted - r.srFirst.AtAdjusted)
if elapsed.Seconds() > 0.0 {
driftSamples := int64(rtpClockTicks - uint64(elapsed.Nanoseconds()*int64(r.params.ClockRate)/1e9))
driftSamples := int64(rtpClockTicks - r.rtpConverter.ToRTPExt(elapsed))
rebasedReportDrift = &livekit.RTPDrift{
StartTime: timestamppb.New(time.Unix(0, r.srFirst.AtAdjusted)),
EndTime: timestamppb.New(time.Unix(0, r.srNewest.AtAdjusted)),
@@ -751,7 +785,7 @@ func (r *rtpStatsBase) getDrift(extStartTS, extHighestTS uint64) (
EndTimestamp: r.srNewest.RtpTimestampExt,
RtpClockTicks: rtpClockTicks,
DriftSamples: driftSamples,
DriftMs: (float64(driftSamples) * 1000) / float64(r.params.ClockRate),
DriftMs: (float64(driftSamples) * 1000) / float64(r.clockRate),
ClockRate: float64(rtpClockTicks) / elapsed.Seconds(),
}
}
pkg/sfu/rtpstats/rtpstats_base_lite.go
+44 -44
View File
@@ -17,6 +17,7 @@ package rtpstats
import (
"errors"
"fmt"
"strings"
"sync"
"time"
@@ -95,14 +96,13 @@ func (s *snapshotLite) MarshalLogObject(e zapcore.ObjectEncoder) error {
// ------------------------------------------------------------------
type RTPStatsParams struct {
ClockRate uint32
IsRTX bool
Logger logger.Logger
IsRTX bool
}
type rtpStatsBaseLite struct {
params RTPStatsParams
logger logger.Logger
params RTPStatsParams
clockRate uint32
logger logger.Logger
lock sync.RWMutex
@@ -134,7 +134,7 @@ type rtpStatsBaseLite struct {
func newRTPStatsBaseLite(params RTPStatsParams) *rtpStatsBaseLite {
return &rtpStatsBaseLite{
params: params,
logger: params.Logger,
logger: logger.GetLogger(),
nextSnapshotLiteID: cFirstSnapshotID,
snapshotLites: make([]snapshotLite, 2),
}
@@ -172,7 +172,21 @@ func (r *rtpStatsBaseLite) seed(from *rtpStatsBaseLite) bool {
return true
}
func (r *rtpStatsBaseLite) SetClockRate(clockRate uint32) {
r.lock.Lock()
defer r.lock.Unlock()
r.setClockRateLocked(clockRate)
}
func (r *rtpStatsBaseLite) setClockRateLocked(clockRate uint32) {
r.clockRate = clockRate
}
func (r *rtpStatsBaseLite) SetLogger(logger logger.Logger) {
r.lock.Lock()
defer r.lock.Unlock()
r.logger = logger
}
@@ -305,9 +319,9 @@ func (r *rtpStatsBaseLite) deltaInfoLite(
snapshotLiteID uint32,
extStartSN uint64,
extHighestSN uint64,
) (deltaInfoLite *RTPDeltaInfoLite, err error, loggingFields []interface{}) {
then, now := r.getAndResetSnapshotLite(snapshotLiteID, extStartSN, extHighestSN)
if now == nil || then == nil {
) (deltaInfoLite *RTPDeltaInfoLite, loggingFields []any, err error) {
then, now, ok := r.getAndResetSnapshotLite(snapshotLiteID, extStartSN, extHighestSN)
if !ok {
return
}
@@ -319,7 +333,7 @@ func (r *rtpStatsBaseLite) deltaInfoLite(
packetsExpected = 0
}
if packetsExpected > cNumSequenceNumbers {
loggingFields = []interface{}{
loggingFields = []any{
"snapshotLiteID", snapshotLiteID,
"snapshotLiteNow", now,
"snapshotLiteThen", then,
@@ -342,7 +356,7 @@ func (r *rtpStatsBaseLite) deltaInfoLite(
packetsLost = 0
}
if packetsLost > packetsExpected {
loggingFields = []interface{}{
loggingFields = []any{
"snapshotLiteID", snapshotLiteID,
"snapshotLiteNow", now,
"snapshotLiteThen", then,
@@ -366,7 +380,7 @@ func (r *rtpStatsBaseLite) deltaInfoLite(
}
func (r *rtpStatsBaseLite) marshalLogObject(e zapcore.ObjectEncoder, packetsExpected, packetsSeenMinusPadding uint64) (float64, error) {
if r == nil || !r.initialized {
if r == nil || !r.initialized || r.clockRate == 0 {
return 0, errors.New("not initialized")
}
@@ -399,25 +413,18 @@ func (r *rtpStatsBaseLite) marshalLogObject(e zapcore.ObjectEncoder, packetsExpe
e.AddFloat32("packetLostPercentage", float32(r.packetsLost)/float32(packetsExpected)*100.0)
}
hasLoss := false
first := true
str := "["
var sb strings.Builder
for burst, count := range r.gapHistogram {
if count == 0 {
continue
}
hasLoss = true
if !first {
str += ", "
if sb.Len() > 0 {
sb.WriteString(", ")
}
first = false
str += fmt.Sprintf("%d:%d", burst+1, count)
fmt.Fprintf(&sb, "%d:%d", burst+1, count)
}
str += "]"
if hasLoss {
e.AddString("gapHistogram", str)
if sb.Len() > 0 {
e.AddString("gapHistogram", "["+sb.String()+"]")
}
e.AddUint32("nacks", r.nacks)
@@ -431,7 +438,7 @@ func (r *rtpStatsBaseLite) marshalLogObject(e zapcore.ObjectEncoder, packetsExpe
}
func (r *rtpStatsBaseLite) toProto(packetsExpected, packetsSeenMinusPadding, packetsLost uint64) *livekit.RTPStats {
if r.startTime == 0 {
if r.startTime == 0 || r.clockRate == 0 {
return nil
}
@@ -473,33 +480,22 @@ func (r *rtpStatsBaseLite) toProto(packetsExpected, packetsSeenMinusPadding, pac
LastPli: timestamppb.New(time.Unix(0, r.lastPli)),
}
gapsPresent := false
for i := 0; i < len(r.gapHistogram); i++ {
for i := range len(r.gapHistogram) {
if r.gapHistogram[i] == 0 {
continue
}
gapsPresent = true
break
}
if gapsPresent {
p.GapHistogram = make(map[int32]uint32, len(r.gapHistogram))
for i := 0; i < len(r.gapHistogram); i++ {
if r.gapHistogram[i] == 0 {
continue
}
p.GapHistogram[int32(i+1)] = r.gapHistogram[i]
if p.GapHistogram == nil {
p.GapHistogram = make(map[int32]uint32, len(r.gapHistogram))
}
p.GapHistogram[int32(i+1)] = r.gapHistogram[i]
}
return p
}
func (r *rtpStatsBaseLite) getAndResetSnapshotLite(snapshotLiteID uint32, extStartSN uint64, extHighestSN uint64) (*snapshotLite, *snapshotLite) {
func (r *rtpStatsBaseLite) getAndResetSnapshotLite(snapshotLiteID uint32, extStartSN uint64, extHighestSN uint64) (snapshotLite, snapshotLite, bool) {
if !r.initialized {
return nil, nil
return snapshotLite{}, snapshotLite{}, false
}
idx := snapshotLiteID - cFirstSnapshotID
@@ -512,7 +508,7 @@ func (r *rtpStatsBaseLite) getAndResetSnapshotLite(snapshotLiteID uint32, extSta
// snapshot now
now := r.getSnapshotLite(mono.UnixNano(), extHighestSN+1)
r.snapshotLites[idx] = now
return &then, &now
return then, now, true
}
func (r *rtpStatsBaseLite) updateGapHistogram(gap int) {
@@ -554,4 +550,8 @@ func getPacketsExpected(extStartSN, extHighestSN uint64) uint64 {
return extHighestSN - extStartSN + 1
}
func shouldLog(count int) bool {
return count < 20 || count%200 == 0
}
// ----------------------------------
pkg/sfu/rtpstats/rtpstats_receiver.go
+434 -129
View File
@@ -20,10 +20,12 @@ import (
"time"
"github.com/pion/rtcp"
"go.uber.org/zap"
"go.uber.org/zap/zapcore"
"github.com/livekit/livekit-server/pkg/sfu/utils"
"github.com/livekit/mediatransportutil"
"github.com/livekit/mediatransportutil/pkg/latency"
"github.com/livekit/mediatransportutil/pkg/utils"
"github.com/livekit/protocol/livekit"
"github.com/livekit/protocol/logger"
protoutils "github.com/livekit/protocol/utils"
@@ -37,12 +39,53 @@ const (
cReportSlack = float64(60.0)
cTSJumpTooHighFactor = float64(1.5)
restartThreshold = 5
)
// ---------------------------------------------------------------------
type RTPFlowUnhandledReason int
const (
RTPFlowUnhandledReasonNone RTPFlowUnhandledReason = iota
RTPFlowUnhandledReasonEnded
RTPFlowUnhandledReasonUnconfigured
RTPFlowUnhandledReasonPaddingOnly
RTPFlowUnhandledReasonPreStartTimestamp
RTPFlowUnhandledReasonOldTimestamp
RTPFlowUnhandledReasonPreStartSequenceNumber
RTPFlowUnhandledReasonOldSequenceNumber
RTPFlowUnhandledReasonRestart
)
func (r RTPFlowUnhandledReason) String() string {
switch r {
case RTPFlowUnhandledReasonNone:
return "NONE"
case RTPFlowUnhandledReasonEnded:
return "ENDED"
case RTPFlowUnhandledReasonUnconfigured:
return "UNCONFIGURED"
case RTPFlowUnhandledReasonPaddingOnly:
return "PADDING_ONLY"
case RTPFlowUnhandledReasonPreStartTimestamp:
return "PRE_START_TIMESTAMP"
case RTPFlowUnhandledReasonOldTimestamp:
return "OLD_TIMESTAMP"
case RTPFlowUnhandledReasonPreStartSequenceNumber:
return "PRE_START_SEQUENCE_NUMBER"
case RTPFlowUnhandledReasonOldSequenceNumber:
return "OLD_SEQUENCE_NUMBER"
case RTPFlowUnhandledReasonRestart:
return "RESTART"
default:
return fmt.Sprintf("UNKNOWN: %d", int(r))
}
}
type RTPFlowState struct {
IsNotHandled bool
UnhandledReason RTPFlowUnhandledReason
LossStartInclusive uint64
LossEndExclusive uint64
@@ -59,7 +102,7 @@ func (r *RTPFlowState) MarshalLogObject(e zapcore.ObjectEncoder) error {
return nil
}
e.AddBool("IsNotHandled", r.IsNotHandled)
e.AddString("UnhandledReason", r.UnhandledReason.String())
e.AddUint64("LossStartInclusive", r.LossStartInclusive)
e.AddUint64("LossEndExclusive", r.LossEndExclusive)
e.AddBool("IsDuplicate", r.IsDuplicate)
@@ -71,6 +114,62 @@ func (r *RTPFlowState) MarshalLogObject(e zapcore.ObjectEncoder) error {
// ---------------------------------------------------------------------
type packet struct {
sequenceNumber uint16
timestamp uint32
}
func (p packet) MarshalLogObject(e zapcore.ObjectEncoder) error {
e.AddUint16("sequenceNumber", p.sequenceNumber)
e.AddUint32("timestamp", p.timestamp)
return nil
}
// ---------------------------------------------------------------------
type receiverUpdateLoggingFields struct {
packetTime int64
sequenceNumber uint16
timestamp uint32
marker bool
hdrSize int
payloadSize int
paddingSize int
resSN utils.WrapAroundUpdateResult[uint64]
gapSN int64
resTS utils.WrapAroundUpdateResult[uint64]
gapTS int64
snRolloverCount int
expectedTSJump int64
tsRolloverCount int
timeSinceHighest int64
rtpStats *RTPStatsReceiver
}
func (rulf *receiverUpdateLoggingFields) MarshalLogObject(e zapcore.ObjectEncoder) error {
if rulf != nil {
e.AddObject("resSN", &rulf.resSN)
e.AddInt64("gapSN", rulf.gapSN)
e.AddObject("resTS", &rulf.resTS)
e.AddInt64("gapTS", rulf.gapTS)
e.AddInt("snRolloverCount", rulf.snRolloverCount)
e.AddInt64("expectedTSJump", rulf.expectedTSJump)
e.AddInt("tsRolloverCount", rulf.tsRolloverCount)
e.AddTime("packetTime", time.Unix(0, rulf.packetTime))
e.AddDuration("timeSinceHighest", time.Duration(rulf.timeSinceHighest))
e.AddUint16("sequenceNumber", rulf.sequenceNumber)
e.AddUint32("timestamp", rulf.timestamp)
e.AddBool("marker", rulf.marker)
e.AddInt("hdrSize", rulf.hdrSize)
e.AddInt("payloadSize", rulf.payloadSize)
e.AddInt("paddingSize", rulf.paddingSize)
e.AddObject("rtpStats", lockedRTPStatsReceiverLogEncoder{rulf.rtpStats})
}
return nil
}
// ---------------------------------------------------------------------
type RTPStatsReceiver struct {
*rtpStatsBase
@@ -81,27 +180,40 @@ type RTPStatsReceiver struct {
history *protoutils.Bitmap[uint64]
propagationDelayEstimator *utils.OWDEstimator
propagationDelayEstimator *latency.OWDEstimator
clockSkewCount int
clockSkewMediaPathCount int
outOfOrderSenderReportCount int
largeJumpCount int
largeJumpNegativeCount int
timeReversedCount int
packetsDroppedPreStartTimestamp int
packetsDroppedOldTimestamp int
packetsDroppedPreStartSequenceNumber int
packetsDroppedOldSequenceNumber int
restartPacketsBuf [restartThreshold]packet
restartPacketsN int
}
func NewRTPStatsReceiver(params RTPStatsParams) *RTPStatsReceiver {
return &RTPStatsReceiver{
rtpStatsBase: newRTPStatsBase(params),
sequenceNumber: utils.NewWrapAround[uint16, uint64](utils.WrapAroundParams{IsRestartAllowed: false}),
tsRolloverThreshold: (1 << 31) * 1e9 / int64(params.ClockRate),
timestamp: utils.NewWrapAround[uint32, uint64](utils.WrapAroundParams{IsRestartAllowed: false}),
history: protoutils.NewBitmap[uint64](cHistorySize),
propagationDelayEstimator: utils.NewOWDEstimator(utils.OWDEstimatorParamsDefault),
propagationDelayEstimator: latency.NewOWDEstimator(latency.OWDEstimatorParamsDefault),
}
}
func (r *RTPStatsReceiver) SetClockRate(clockRate uint32) {
r.lock.Lock()
defer r.lock.Unlock()
r.tsRolloverThreshold = (1 << 31) * 1e9 / int64(clockRate)
r.rtpStatsBase.setClockRateLocked(clockRate)
}
func (r *RTPStatsReceiver) NewSnapshotId() uint32 {
r.lock.Lock()
defer r.lock.Unlock()
@@ -115,17 +227,19 @@ func (r *RTPStatsReceiver) getTSRolloverCount(diffNano int64, ts uint32) int {
return -1
}
excess := (diffNano - r.tsRolloverThreshold*2) * int64(r.params.ClockRate) / 1e9
roc := excess / (1 << 32)
if roc < 0 {
roc = 0
}
excess := (diffNano - r.tsRolloverThreshold*2) * int64(r.clockRate) / 1e9
roc := max(excess/(1<<32), 0)
if r.timestamp.GetHighest() > ts {
roc++
}
return int(roc)
}
func (r *RTPStatsReceiver) undoUpdatesLocked(resSN utils.WrapAroundUpdateResult[uint64], resTS utils.WrapAroundUpdateResult[uint64]) {
r.sequenceNumber.UndoUpdate(resSN)
r.timestamp.UndoUpdate(resTS)
}
func (r *RTPStatsReceiver) Update(
packetTime int64,
sequenceNumber uint16,
@@ -139,7 +253,11 @@ func (r *RTPStatsReceiver) Update(
defer r.lock.Unlock()
if r.endTime != 0 {
flowState.IsNotHandled = true
flowState.UnhandledReason = RTPFlowUnhandledReasonEnded
return
}
if r.clockRate == 0 {
flowState.UnhandledReason = RTPFlowUnhandledReasonUnconfigured
return
}
@@ -152,31 +270,10 @@ func (r *RTPStatsReceiver) Update(
var tsRolloverCount int
var snRolloverCount int
logger := func() logger.UnlikelyLogger {
return r.logger.WithUnlikelyValues(
"resSN", resSN,
"gapSN", gapSN,
"resTS", resTS,
"gapTS", gapTS,
"snRolloverCount", snRolloverCount,
"expectedTSJump", expectedTSJump,
"tsRolloverCount", tsRolloverCount,
"packetTime", time.Unix(0, packetTime),
"timeSinceHighest", time.Duration(timeSinceHighest),
"sequenceNumber", sequenceNumber,
"timestamp", timestamp,
"marker", marker,
"hdrSize", hdrSize,
"payloadSize", payloadSize,
"paddingSize", paddingSize,
"rtpStats", lockedRTPStatsReceiverLogEncoder{r},
)
}
if !r.initialized {
if payloadSize == 0 {
// do not start on a padding only packet
flowState.IsNotHandled = true
flowState.UnhandledReason = RTPFlowUnhandledReasonPaddingOnly
return
}
@@ -201,57 +298,164 @@ func (r *RTPStatsReceiver) Update(
)
} else {
resSN = r.sequenceNumber.Update(sequenceNumber)
if resSN.IsUnhandled {
flowState.IsNotHandled = true
return
}
gapSN = int64(resSN.ExtendedVal - resSN.PreExtendedHighest)
timeSinceHighest = packetTime - r.highestTime
expectedTSJump = int64(r.rtpConverter.ToRTPExt(time.Duration(timeSinceHighest)))
tsRolloverCount = r.getTSRolloverCount(timeSinceHighest, timestamp)
if tsRolloverCount >= 0 {
logger().Warnw("potential time stamp roll over", nil)
}
resTS = r.timestamp.Rollover(timestamp, tsRolloverCount)
gapTS = int64(resTS.ExtendedVal - resTS.PreExtendedHighest)
if resTS.IsUnhandled {
flowState.IsNotHandled = true
r.undoUpdatesLocked(resSN, resTS)
r.packetsDroppedPreStartTimestamp++
rulf := &receiverUpdateLoggingFields{
packetTime: packetTime,
sequenceNumber: sequenceNumber,
timestamp: timestamp,
marker: marker,
hdrSize: hdrSize,
payloadSize: payloadSize,
paddingSize: paddingSize,
resSN: resSN,
gapSN: gapSN,
resTS: resTS,
gapTS: gapTS,
snRolloverCount: snRolloverCount,
expectedTSJump: expectedTSJump,
tsRolloverCount: tsRolloverCount,
timeSinceHighest: timeSinceHighest,
rtpStats: r,
}
if shouldLog(r.packetsDroppedPreStartTimestamp) {
r.logger.Warnw("dropping packet, pre-start timestamp", nil, zap.Inline(rulf))
}
if r.maybeRestart(sequenceNumber, timestamp, payloadSize) {
r.logger.Infow("potential restart", zap.Inline(rulf))
r.resetRestart()
flowState.UnhandledReason = RTPFlowUnhandledReasonRestart
} else {
flowState.UnhandledReason = RTPFlowUnhandledReasonPreStartTimestamp
}
return
}
gapTS = int64(resTS.ExtendedVal - resTS.PreExtendedHighest)
// it is possible to receive old packets in two different scenarios
// as it is not possible to detect how far to roll back, ignore old packets
//
// Case 1:
// Very old time stamp, happens under the following conditions
// - resume after long mute, big time stamp jump
// - an out of order packet from before the mute arrives (unsure what causes this
// very old packet to be trasmitted from remote), causing time stamp to jump back
// to before mute, but it appears like it has rolled over.
// Use a threshold against expected to ignore these.
if gapSN < 0 && gapTS > 0 {
expectedTSJump = timeSinceHighest * int64(r.params.ClockRate) / 1e9
if gapTS > int64(float64(expectedTSJump)*cTSJumpTooHighFactor) {
r.sequenceNumber.UndoUpdate(resSN)
r.timestamp.UndoUpdate(resTS)
logger().Warnw("dropping old packet, timestamp", nil)
flowState.IsNotHandled = true
if tsRolloverCount >= 0 && payloadSize > 0 {
rulf := &receiverUpdateLoggingFields{
packetTime: packetTime,
sequenceNumber: sequenceNumber,
timestamp: timestamp,
marker: marker,
hdrSize: hdrSize,
payloadSize: payloadSize,
paddingSize: paddingSize,
resSN: resSN,
gapSN: gapSN,
resTS: resTS,
gapTS: gapTS,
snRolloverCount: snRolloverCount,
expectedTSJump: expectedTSJump,
tsRolloverCount: tsRolloverCount,
timeSinceHighest: timeSinceHighest,
rtpStats: r,
}
r.logger.Warnw("potential time stamp roll over", nil, zap.Inline(rulf))
}
if !resSN.IsUnhandled {
// it is possible to receive old packets in two different scenarios
// as it is not possible to detect how far to roll back, ignore old packets
//
// Case 1:
// Very old time stamp, happens under the following conditions
// - resume after long mute, this casues big time stamp jump ahead for the packets
// after unmute
// - an out of order packet from before the mute arrives (unsure what causes this
// very old packet to be transmitted from remote), causing time stamp to jump back
// to before mute, but it appears like it has rolled over.
// Use a threshold against expected to ignore these.
if gapSN < 0 && gapTS > 0 {
if gapTS > int64(float64(expectedTSJump)*cTSJumpTooHighFactor) {
r.undoUpdatesLocked(resSN, resTS)
r.packetsDroppedOldTimestamp++
rulf := &receiverUpdateLoggingFields{
packetTime: packetTime,
sequenceNumber: sequenceNumber,
timestamp: timestamp,
marker: marker,
hdrSize: hdrSize,
payloadSize: payloadSize,
paddingSize: paddingSize,
resSN: resSN,
gapSN: gapSN,
resTS: resTS,
gapTS: gapTS,
snRolloverCount: snRolloverCount,
expectedTSJump: expectedTSJump,
tsRolloverCount: tsRolloverCount,
timeSinceHighest: timeSinceHighest,
rtpStats: r,
}
if shouldLog(r.packetsDroppedOldTimestamp) {
r.logger.Warnw("dropping packet, old timestamp", nil, zap.Inline(rulf))
}
if r.maybeRestart(sequenceNumber, timestamp, payloadSize) {
r.logger.Infow("potential restart", zap.Inline(rulf))
r.resetRestart()
flowState.UnhandledReason = RTPFlowUnhandledReasonRestart
} else {
flowState.UnhandledReason = RTPFlowUnhandledReasonOldTimestamp
}
return
}
}
// Case 2:
// Sequence number looks like it is moving forward, but it is actually a very old packet.
if gapTS < 0 && gapSN > 0 {
r.undoUpdatesLocked(resSN, resTS)
expectedTSJump = int64(r.rtpConverter.ToRTPExt(time.Duration(timeSinceHighest)))
r.packetsDroppedOldSequenceNumber++
rulf := &receiverUpdateLoggingFields{
packetTime: packetTime,
sequenceNumber: sequenceNumber,
timestamp: timestamp,
marker: marker,
hdrSize: hdrSize,
payloadSize: payloadSize,
paddingSize: paddingSize,
resSN: resSN,
gapSN: gapSN,
resTS: resTS,
gapTS: gapTS,
snRolloverCount: snRolloverCount,
expectedTSJump: expectedTSJump,
tsRolloverCount: tsRolloverCount,
timeSinceHighest: timeSinceHighest,
rtpStats: r,
}
if shouldLog(r.packetsDroppedOldSequenceNumber) {
r.logger.Warnw("dropping packet, old sequence number", nil, zap.Inline(rulf))
}
if r.maybeRestart(sequenceNumber, timestamp, payloadSize) {
r.logger.Infow("potential restart", zap.Inline(rulf))
r.resetRestart()
flowState.UnhandledReason = RTPFlowUnhandledReasonRestart
} else {
flowState.UnhandledReason = RTPFlowUnhandledReasonOldSequenceNumber
}
return
}
}
// Case 2:
// Sequence number looks like it is moving forward, but it is actually a very old packet.
if gapTS < 0 && gapSN > 0 {
r.sequenceNumber.UndoUpdate(resSN)
r.timestamp.UndoUpdate(resTS)
logger().Warnw("dropping old packet, sequence number", nil)
flowState.IsNotHandled = true
return
}
// it is possible that sequence number has rolled over too
if gapSN < 0 && gapTS > 0 && payloadSize > 0 {
if (gapSN < 0 || gapSN > (1<<15)) && gapTS > 0 && payloadSize > 0 {
// not possible to know how many cycles of sequence number roll over could have happened,
// ensure that it at least does not go backwards
snRolloverCount = 0
@@ -259,12 +463,63 @@ func (r *RTPStatsReceiver) Update(
snRolloverCount = 1
}
resSN = r.sequenceNumber.Rollover(sequenceNumber, snRolloverCount)
if resSN.IsUnhandled {
flowState.IsNotHandled = true
return
if !resSN.IsUnhandled {
rulf := &receiverUpdateLoggingFields{
packetTime: packetTime,
sequenceNumber: sequenceNumber,
timestamp: timestamp,
marker: marker,
hdrSize: hdrSize,
payloadSize: payloadSize,
paddingSize: paddingSize,
resSN: resSN,
gapSN: gapSN,
resTS: resTS,
gapTS: gapTS,
snRolloverCount: snRolloverCount,
expectedTSJump: expectedTSJump,
tsRolloverCount: tsRolloverCount,
timeSinceHighest: timeSinceHighest,
rtpStats: r,
}
r.logger.Warnw("forcing sequence number rollover", nil, zap.Inline(rulf))
}
}
if resSN.IsUnhandled {
r.undoUpdatesLocked(resSN, resTS)
r.packetsDroppedPreStartSequenceNumber++
rulf := &receiverUpdateLoggingFields{
packetTime: packetTime,
sequenceNumber: sequenceNumber,
timestamp: timestamp,
marker: marker,
hdrSize: hdrSize,
payloadSize: payloadSize,
paddingSize: paddingSize,
resSN: resSN,
gapSN: gapSN,
resTS: resTS,
gapTS: gapTS,
snRolloverCount: snRolloverCount,
expectedTSJump: expectedTSJump,
tsRolloverCount: tsRolloverCount,
timeSinceHighest: timeSinceHighest,
rtpStats: r,
}
if shouldLog(r.packetsDroppedPreStartSequenceNumber) {
r.logger.Warnw("dropping packet, pre-start sequence number", nil, zap.Inline(rulf))
}
logger().Warnw("forcing sequence number rollover", nil)
if r.maybeRestart(sequenceNumber, timestamp, payloadSize) {
r.logger.Infow("potential restart", zap.Inline(rulf))
r.resetRestart()
flowState.UnhandledReason = RTPFlowUnhandledReasonRestart
} else {
flowState.UnhandledReason = RTPFlowUnhandledReasonPreStartSequenceNumber
}
return
}
}
gapSN = int64(resSN.ExtendedVal - resSN.PreExtendedHighest)
@@ -287,32 +542,31 @@ func (r *RTPStatsReceiver) Update(
}
flowState.IsOutOfOrder = true
if !flowState.IsDuplicate && -gapSN >= cSequenceNumberLargeJumpThreshold {
r.largeJumpNegativeCount++
if (r.largeJumpNegativeCount-1)%100 == 0 {
logger().Warnw(
"large sequence number gap negative", nil,
"count", r.largeJumpNegativeCount,
)
}
}
} else { // in-order
if gapSN >= cSequenceNumberLargeJumpThreshold {
r.largeJumpCount++
if (r.largeJumpCount-1)%100 == 0 {
logger().Warnw(
"large sequence number gap", nil,
"count", r.largeJumpCount,
)
}
}
if resTS.ExtendedVal < resTS.PreExtendedHighest {
if resTS.ExtendedVal < resTS.PreExtendedHighest && r.bytes > 0 {
r.timeReversedCount++
if (r.timeReversedCount-1)%100 == 0 {
logger().Warnw(
if shouldLog(r.timeReversedCount) {
rulf := &receiverUpdateLoggingFields{
packetTime: packetTime,
sequenceNumber: sequenceNumber,
timestamp: timestamp,
marker: marker,
hdrSize: hdrSize,
payloadSize: payloadSize,
paddingSize: paddingSize,
resSN: resSN,
gapSN: gapSN,
resTS: resTS,
gapTS: gapTS,
snRolloverCount: snRolloverCount,
expectedTSJump: expectedTSJump,
tsRolloverCount: tsRolloverCount,
timeSinceHighest: timeSinceHighest,
rtpStats: r,
}
r.logger.Warnw(
"time reversed", nil,
zap.Inline(rulf),
"count", r.timeReversedCount,
)
}
@@ -355,6 +609,7 @@ func (r *RTPStatsReceiver) Update(
r.updateJitter(resTS.ExtendedVal, packetTime)
}
}
r.resetRestart()
return
}
@@ -363,10 +618,12 @@ func (r *RTPStatsReceiver) getExtendedSenderReport(srData *livekit.RTCPSenderRep
if r.srNewest != nil {
// use time since last sender report to ensure long gaps where the time stamp might
// jump more than half the range
timeSinceLastReport := mediatransportutil.NtpTime(srData.NtpTimestamp).Time().Sub(mediatransportutil.NtpTime(r.srNewest.NtpTimestamp).Time())
expectedRTPTimestampExt := r.srNewest.RtpTimestampExt + uint64(timeSinceLastReport.Nanoseconds()*int64(r.params.ClockRate)/1e9)
lbound := expectedRTPTimestampExt - uint64(cReportSlack*float64(r.params.ClockRate))
ubound := expectedRTPTimestampExt + uint64(cReportSlack*float64(r.params.ClockRate))
srTime := mediatransportutil.NtpTime(srData.NtpTimestamp).Time()
srNewestTime := mediatransportutil.NtpTime(r.srNewest.NtpTimestamp).Time()
timeSinceLastReport := srTime.Sub(srNewestTime)
expectedRTPTimestampExt := r.srNewest.RtpTimestampExt + r.rtpConverter.ToRTPExt(timeSinceLastReport)
lbound := expectedRTPTimestampExt - uint64(cReportSlack*float64(r.clockRate))
ubound := expectedRTPTimestampExt + uint64(cReportSlack*float64(r.clockRate))
isInRange := (srData.RtpTimestamp-uint32(lbound) < (1 << 31)) && (uint32(ubound)-srData.RtpTimestamp < (1 << 31))
if isInRange {
lbTSCycles := lbound & 0xFFFF_FFFF_0000_0000
@@ -409,7 +666,7 @@ func (r *RTPStatsReceiver) checkOutOfOrderSenderReport(srData *livekit.RTCPSende
// Or it could be due bad report generation.
// In any case, ignore out-of-order reports.
r.outOfOrderSenderReportCount++
if (r.outOfOrderSenderReportCount-1)%10 == 0 {
if shouldLog(r.outOfOrderSenderReportCount) {
r.logger.Infow(
"received sender report, out-of-order, skipping",
"current", WrappedRTCPSenderReportStateLogger{srData},
@@ -428,18 +685,22 @@ func (r *RTPStatsReceiver) checkRTPClockSkewForSenderReport(srData *livekit.RTCP
return
}
timeSinceLast := mediatransportutil.NtpTime(srData.NtpTimestamp).Time().Sub(mediatransportutil.NtpTime(r.srNewest.NtpTimestamp).Time()).Seconds()
srTime := mediatransportutil.NtpTime(srData.NtpTimestamp).Time()
srNewestTime := mediatransportutil.NtpTime(r.srNewest.NtpTimestamp).Time()
srFirstTime := mediatransportutil.NtpTime(r.srFirst.NtpTimestamp).Time()
timeSinceLast := srTime.Sub(srNewestTime).Seconds()
rtpDiffSinceLast := srData.RtpTimestampExt - r.srNewest.RtpTimestampExt
calculatedClockRateFromLast := float64(rtpDiffSinceLast) / timeSinceLast
timeSinceFirst := mediatransportutil.NtpTime(srData.NtpTimestamp).Time().Sub(mediatransportutil.NtpTime(r.srFirst.NtpTimestamp).Time()).Seconds()
timeSinceFirst := srTime.Sub(srFirstTime).Seconds()
rtpDiffSinceFirst := srData.RtpTimestampExt - r.srFirst.RtpTimestampExt
calculatedClockRateFromFirst := float64(rtpDiffSinceFirst) / timeSinceFirst
if (timeSinceLast > 0.2 && math.Abs(float64(r.params.ClockRate)-calculatedClockRateFromLast) > 0.2*float64(r.params.ClockRate)) ||
(timeSinceFirst > 0.2 && math.Abs(float64(r.params.ClockRate)-calculatedClockRateFromFirst) > 0.2*float64(r.params.ClockRate)) {
if (timeSinceLast > 0.2 && math.Abs(float64(r.clockRate)-calculatedClockRateFromLast) > 0.2*float64(r.clockRate)) ||
(timeSinceFirst > 0.2 && math.Abs(float64(r.clockRate)-calculatedClockRateFromFirst) > 0.2*float64(r.clockRate)) {
r.clockSkewCount++
if (r.clockSkewCount-1)%100 == 0 {
if shouldLog(r.clockSkewCount) {
r.logger.Infow(
"received sender report, clock skew",
"current", WrappedRTCPSenderReportStateLogger{srData},
@@ -463,20 +724,20 @@ func (r *RTPStatsReceiver) checkRTPClockSkewAgainstMediaPathForSenderReport(srDa
nowNano := mono.UnixNano()
timeSinceSR := time.Duration(nowNano - srData.AtAdjusted)
extNowTSSR := srData.RtpTimestampExt + uint64(timeSinceSR.Nanoseconds()*int64(r.params.ClockRate)/1e9)
extNowTSSR := srData.RtpTimestampExt + r.rtpConverter.ToRTPExt(timeSinceSR)
timeSinceHighest := time.Duration(nowNano - r.highestTime)
extNowTSHighest := r.timestamp.GetExtendedHighest() + uint64(timeSinceHighest.Nanoseconds()*int64(r.params.ClockRate)/1e9)
extNowTSHighest := r.timestamp.GetExtendedHighest() + r.rtpConverter.ToRTPExt(timeSinceHighest)
diffHighest := extNowTSSR - extNowTSHighest
timeSinceFirst := time.Duration(nowNano - r.firstTime)
extNowTSFirst := r.timestamp.GetExtendedStart() + uint64(timeSinceFirst.Nanoseconds()*int64(r.params.ClockRate)/1e9)
extNowTSFirst := r.timestamp.GetExtendedStart() + r.rtpConverter.ToRTPExt(timeSinceFirst)
diffFirst := extNowTSSR - extNowTSFirst
// is it more than 5 seconds off?
if uint32(math.Abs(float64(int64(diffHighest)))) > 5*r.params.ClockRate || uint32(math.Abs(float64(int64(diffFirst)))) > 5*r.params.ClockRate {
if uint32(math.Abs(float64(int64(diffHighest)))) > 5*r.clockRate || uint32(math.Abs(float64(int64(diffFirst)))) > 5*r.clockRate {
r.clockSkewMediaPathCount++
if (r.clockSkewMediaPathCount-1)%100 == 0 {
if shouldLog(r.clockSkewMediaPathCount) {
r.logger.Infow(
"received sender report, clock skew against media path",
"current", WrappedRTCPSenderReportStateLogger{srData},
@@ -496,7 +757,8 @@ func (r *RTPStatsReceiver) checkRTPClockSkewAgainstMediaPathForSenderReport(srDa
}
func (r *RTPStatsReceiver) updatePropagationDelayAndRecordSenderReport(srData *livekit.RTCPSenderReportState) {
senderClockTime := mediatransportutil.NtpTime(srData.NtpTimestamp).Time().UnixNano()
srTime := mediatransportutil.NtpTime(srData.NtpTimestamp).Time()
senderClockTime := srTime.UnixNano()
estimatedPropagationDelay, stepChange := r.propagationDelayEstimator.Update(senderClockTime, srData.At)
if stepChange {
r.logger.Debugw(
@@ -518,7 +780,7 @@ func (r *RTPStatsReceiver) SetRtcpSenderReportData(srData *livekit.RTCPSenderRep
r.lock.Lock()
defer r.lock.Unlock()
if srData == nil || !r.initialized {
if srData == nil || !r.initialized || r.clockRate == 0 {
return false
}
@@ -542,9 +804,11 @@ func (r *RTPStatsReceiver) SetRtcpSenderReportData(srData *livekit.RTCPSenderRep
r.updatePropagationDelayAndRecordSenderReport(srDataExt)
r.checkRTPClockSkewAgainstMediaPathForSenderReport(srDataExt)
if err, loggingFields := r.maybeAdjustFirstPacketTime(r.srNewest, 0, r.timestamp.GetExtendedStart()); err != nil {
adjustment, loggingFields, err := r.maybeAdjustFirstPacketTime(r.srNewest, 0, r.timestamp.GetExtendedStart())
if err != nil {
r.logger.Infow(err.Error(), append(loggingFields, "rtpStats", lockedRTPStatsReceiverLogEncoder{r})...)
}
r.propagationDelayEstimator.InitialAdjustment(adjustment)
return true
}
@@ -571,8 +835,8 @@ func (r *RTPStatsReceiver) GetRtcpReceptionReport(ssrc uint32, proxyFracLost uin
defer r.lock.Unlock()
extHighestSN := r.sequenceNumber.GetExtendedHighest()
then, now := r.getAndResetSnapshot(snapshotID, r.sequenceNumber.GetExtendedStart(), extHighestSN)
if now == nil || then == nil {
then, now, ok := r.getAndResetSnapshot(snapshotID, r.sequenceNumber.GetExtendedStart(), extHighestSN)
if !ok {
return nil
}
@@ -594,15 +858,9 @@ func (r *RTPStatsReceiver) GetRtcpReceptionReport(ssrc uint32, proxyFracLost uin
packetsLost = 0
}
lossRate := float32(packetsLost) / float32(packetsExpected)
fracLost := uint8(lossRate * 256.0)
if proxyFracLost > fracLost {
fracLost = proxyFracLost
}
fracLost := max(proxyFracLost, uint8(lossRate*256.0))
totalLost := r.packetsLost
if totalLost > 0xffffff { // 24-bits max
totalLost = 0xffffff
}
totalLost := min(r.packetsLost, 0xffffff) // 24-bits max
lastSR := uint32(0)
dlsr := uint32(0)
@@ -629,7 +887,7 @@ func (r *RTPStatsReceiver) DeltaInfo(snapshotID uint32) *RTPDeltaInfo {
r.lock.Lock()
defer r.lock.Unlock()
deltaInfo, err, loggingFields := r.deltaInfo(
deltaInfo, loggingFields, err := r.deltaInfo(
snapshotID,
r.sequenceNumber.GetExtendedStart(),
r.sequenceNumber.GetExtendedHighest(),
@@ -653,6 +911,10 @@ func (r *RTPStatsReceiver) MarshalLogObject(e zapcore.ObjectEncoder) error {
}
func (r *RTPStatsReceiver) ToProto() *livekit.RTPStats {
if r == nil {
return nil
}
r.lock.RLock()
defer r.lock.RUnlock()
@@ -696,6 +958,37 @@ func (r *RTPStatsReceiver) ExtendedHighestSequenceNumber() uint64 {
return r.sequenceNumber.GetExtendedHighest()
}
func (r *RTPStatsReceiver) maybeRestart(sn uint16, ts uint32, payloadSize int) bool {
if payloadSize > 0 {
if r.restartPacketsN < restartThreshold {
r.restartPacketsBuf[r.restartPacketsN] = packet{sn, ts}
r.restartPacketsN++
} else {
// keep last restartThreshold entries: shift left and append
copy(r.restartPacketsBuf[:], r.restartPacketsBuf[1:])
r.restartPacketsBuf[restartThreshold-1] = packet{sn, ts}
}
}
if r.restartPacketsN < restartThreshold {
return false
}
// check for contiguous sequence numbers and equal or increasing timestamps
for i := 1; i < r.restartPacketsN; i++ {
p := &r.restartPacketsBuf[i]
prev := &r.restartPacketsBuf[i-1]
if p.sequenceNumber != prev.sequenceNumber+1 || (p.timestamp-prev.timestamp) > (1<<31) {
return false
}
}
return true
}
func (r *RTPStatsReceiver) resetRestart() {
r.restartPacketsN = 0
}
// ----------------------------------
type lockedRTPStatsReceiverLogEncoder struct {
@@ -725,6 +1018,18 @@ func (r lockedRTPStatsReceiverLogEncoder) MarshalLogObject(e zapcore.ObjectEncod
e.AddUint64("extHighestTS", extHighestTS)
e.AddObject("propagationDelayEstimator", r.propagationDelayEstimator)
e.AddInt("clockSkewCount", r.clockSkewCount)
e.AddInt("clockSkewMediaPathCount", r.clockSkewMediaPathCount)
e.AddInt("outOfOrderSenderReportCount", r.outOfOrderSenderReportCount)
e.AddInt("timeReversedCount", r.timeReversedCount)
e.AddInt("packetsDroppedPreStartTimestamp", r.packetsDroppedPreStartTimestamp)
e.AddInt("packetsDroppedOldTimestamp", r.packetsDroppedOldTimestamp)
e.AddInt("packetsDroppedPreStartSequenceNumber", r.packetsDroppedPreStartSequenceNumber)
e.AddInt("packetsDroppedOldSequenceNumber", r.packetsDroppedOldSequenceNumber)
e.AddArray("restartPackets", logger.ObjectSlice(r.restartPacketsBuf[:r.restartPacketsN]))
return nil
}
pkg/sfu/rtpstats/rtpstats_receiver_lite.go
+5 -3
View File
@@ -17,7 +17,7 @@ package rtpstats
import (
"go.uber.org/zap/zapcore"
"github.com/livekit/livekit-server/pkg/sfu/utils"
"github.com/livekit/mediatransportutil/pkg/utils"
"github.com/livekit/protocol/livekit"
"github.com/livekit/protocol/utils/mono"
)
@@ -102,7 +102,9 @@ func (r *RTPStatsReceiverLite) Update(packetTime int64, packetSize int, sequence
gapSN := int64(resSN.ExtendedVal - resSN.PreExtendedHighest)
if gapSN <= 0 { // duplicate OR out-of-order
r.packetsOutOfOrder++ // counting duplicate as out-of-order
r.packetsLost--
if gapSN != 0 && r.packetsLost > 0 {
r.packetsLost--
}
} else { // in-order
r.updateGapHistogram(int(gapSN))
r.packetsLost += uint64(gapSN - 1)
@@ -119,7 +121,7 @@ func (r *RTPStatsReceiverLite) DeltaInfoLite(snapshotLiteID uint32) *RTPDeltaInf
r.lock.Lock()
defer r.lock.Unlock()
deltaInfoLite, err, loggingFields := r.deltaInfoLite(
deltaInfoLite, loggingFields, err := r.deltaInfoLite(
snapshotLiteID,
r.sequenceNumber.GetExtendedStart(),
r.sequenceNumber.GetExtendedHighest(),
pkg/sfu/rtpstats/rtpstats_sender.go
+241 -135
View File
@@ -21,12 +21,12 @@ import (
"time"
"github.com/pion/rtcp"
"go.uber.org/zap"
"go.uber.org/zap/zapcore"
"google.golang.org/protobuf/types/known/timestamppb"
"github.com/livekit/mediatransportutil"
"github.com/livekit/protocol/livekit"
"github.com/livekit/protocol/logger"
"github.com/livekit/protocol/utils/mono"
)
@@ -40,6 +40,9 @@ const (
snInfoFlagOutOfOrder
)
// max number of receiver reports kept per sender snapshot for logging
const cMaxProcessedReceptionReports = 64
type snInfo struct {
pktSize uint16
hdrSize uint8
@@ -103,7 +106,8 @@ type wrappedReceptionReportsLogger struct {
}
func (w wrappedReceptionReportsLogger) MarshalLogObject(e zapcore.ObjectEncoder) error {
for i, rr := range w.senderSnapshotReceiverView.processedReceptionReports {
for i := 0; i < w.senderSnapshotReceiverView.processedReceptionReportsSize; i++ {
rr := w.senderSnapshotReceiverView.processedReceptionReports[(w.senderSnapshotReceiverView.processedReceptionReportsHead+i)%cMaxProcessedReceptionReports]
e.AddReflected(fmt.Sprintf("%d", i), rr)
}
@@ -192,9 +196,13 @@ type senderSnapshotReceiverView struct {
maxRtt uint32
maxJitter float64
extLastRRSN uint64
intervalStats intervalStats
processedReceptionReports []rtcp.ReceptionReport
extLastRRSN uint64
intervalStats intervalStats
processedReceptionReports [cMaxProcessedReceptionReports]rtcp.ReceptionReport
processedReceptionReportsHead int
processedReceptionReportsSize int
metadataCacheOverflowCount int
}
@@ -272,6 +280,70 @@ func (s *senderSnapshot) maybeUpdateMaxJitter(jitter float64) {
// -------------------------------------------------------------------
type senderUpdateLoggingFields struct {
packetTime int64
extSequenceNumber uint64
extTimestamp uint64
marker bool
hdrSize int
payloadSize int
paddingSize int
gapSN int64
gapTS int64
timeSinceHighest int64
rtpStats *RTPStatsSender
}
func (sulf *senderUpdateLoggingFields) MarshalLogObject(e zapcore.ObjectEncoder) error {
if sulf != nil {
e.AddUint64("currSN", sulf.extSequenceNumber)
e.AddInt64("gapSN", sulf.gapSN)
e.AddUint64("currTS", sulf.extTimestamp)
e.AddInt64("gapTS", sulf.gapTS)
e.AddTime("packetTime", time.Unix(0, sulf.packetTime))
e.AddDuration("timeSinceHighest", time.Duration(sulf.timeSinceHighest))
e.AddBool("marker", sulf.marker)
e.AddInt("hdrSize", sulf.hdrSize)
e.AddInt("payloadSize", sulf.payloadSize)
e.AddInt("paddingSize", sulf.paddingSize)
e.AddObject("rtpStats", lockedRTPStatsSenderLogEncoder{sulf.rtpStats})
}
return nil
}
// -------------------------------------------------------------------
type senderReportLoggingFields struct {
srData *livekit.RTCPSenderReportState
publisherSRData *livekit.RTCPSenderReportState
tsOffset uint64
reportTime int64
reportTimeAdjusted int64
nowNano int64
nowRTPExt uint64
rtpStats *RTPStatsSender
}
func (srlf *senderReportLoggingFields) MarshalLogObject(e zapcore.ObjectEncoder) error {
if srlf != nil {
e.AddObject("curr", WrappedRTCPSenderReportStateLogger{srlf.srData})
e.AddObject("feed", WrappedRTCPSenderReportStateLogger{srlf.publisherSRData})
e.AddUint64("tsOffset", srlf.tsOffset)
e.AddTime("timeNow", time.Now())
e.AddInt64("reportTime", srlf.reportTime)
e.AddInt64("reportTimeAdjusted", srlf.reportTimeAdjusted)
e.AddDuration("timeSinceHighest", time.Duration(srlf.nowNano-srlf.rtpStats.highestTime))
e.AddDuration("timeSinceFirst", time.Duration(srlf.nowNano-srlf.rtpStats.firstTime))
e.AddDuration("timeSincePublisherSRAdjusted", time.Duration(srlf.nowNano-srlf.publisherSRData.AtAdjusted))
e.AddDuration("timeSincePublisherSR", time.Duration(srlf.nowNano-srlf.publisherSRData.At))
e.AddUint64("nowRTPExt", srlf.nowRTPExt)
e.AddObject("rtpStats", lockedRTPStatsSenderLogEncoder{srlf.rtpStats})
}
return nil
}
// -------------------------------------------------------------------
type rttMarker struct {
ntpTime mediatransportutil.NtpTime
sentAt time.Time
@@ -308,10 +380,8 @@ type RTPStatsSender struct {
nextSenderSnapshotID uint32
senderSnapshots []senderSnapshot
clockSkewCount int
largeJumpNegativeCount int
largeJumpCount int
timeReversedCount int
clockSkewCount int
timeReversedCount int
}
func NewRTPStatsSender(params RTPStatsParams, cacheSize int) *RTPStatsSender {
@@ -399,7 +469,7 @@ func (r *RTPStatsSender) Update(
r.lock.Lock()
defer r.lock.Unlock()
if r.endTime != 0 {
if r.endTime != 0 || r.clockRate == 0 {
return
}
@@ -444,21 +514,6 @@ func (r *RTPStatsSender) Update(
pktSize := uint64(hdrSize + payloadSize + paddingSize)
isDuplicate := false
gapSN := int64(extSequenceNumber - r.extHighestSN)
ulgr := func() logger.UnlikelyLogger {
return r.logger.WithUnlikelyValues(
"currSN", extSequenceNumber,
"gapSN", gapSN,
"currTS", extTimestamp,
"gapTS", int64(extTimestamp-r.extHighestTS),
"packetTime", time.Unix(0, packetTime),
"timeSinceHighest", time.Duration(packetTime-r.highestTime),
"marker", marker,
"hdrSize", hdrSize,
"payloadSize", payloadSize,
"paddingSize", paddingSize,
"rtpStats", lockedRTPStatsSenderLogEncoder{r},
)
}
if gapSN <= 0 { // duplicate OR out-of-order
if payloadSize == 0 && extSequenceNumber < r.extStartSN && !r.params.IsRTX {
// do not start on a padding only packet
@@ -479,8 +534,22 @@ func (r *RTPStatsSender) Update(
r.senderSnapshots[i].maybeReinit(r.extStartSN, extSequenceNumber)
}
ulgr().Infow(
sulf := &senderUpdateLoggingFields{
packetTime: packetTime,
extSequenceNumber: extSequenceNumber,
extTimestamp: extTimestamp,
marker: marker,
hdrSize: hdrSize,
payloadSize: payloadSize,
paddingSize: paddingSize,
gapSN: gapSN,
gapTS: int64(extTimestamp - r.extHighestTS),
timeSinceHighest: packetTime - r.highestTime,
rtpStats: r,
}
r.logger.Infow(
"adjusting start sequence number",
zap.Inline(sulf),
"snAfter", extSequenceNumber,
"tsAfter", extTimestamp,
)
@@ -500,32 +569,26 @@ func (r *RTPStatsSender) Update(
r.packetsLost--
r.setSnInfo(extSequenceNumber, r.extHighestSN, uint16(pktSize), uint8(hdrSize), uint16(payloadSize), marker, true)
}
if !isDuplicate && -gapSN >= cSequenceNumberLargeJumpThreshold {
r.largeJumpNegativeCount++
if (r.largeJumpNegativeCount-1)%100 == 0 {
ulgr().Warnw(
"large sequence number gap negative", nil,
"count", r.largeJumpNegativeCount,
)
}
}
} else { // in-order
if gapSN >= cSequenceNumberLargeJumpThreshold {
r.largeJumpCount++
if (r.largeJumpCount-1)%100 == 0 {
ulgr().Warnw(
"large sequence number gap", nil,
"count", r.largeJumpCount,
)
}
}
if extTimestamp < r.extHighestTS {
r.timeReversedCount++
if (r.timeReversedCount-1)%100 == 0 {
ulgr().Warnw(
if shouldLog(r.timeReversedCount) {
sulf := &senderUpdateLoggingFields{
packetTime: packetTime,
extSequenceNumber: extSequenceNumber,
extTimestamp: extTimestamp,
marker: marker,
hdrSize: hdrSize,
payloadSize: payloadSize,
paddingSize: paddingSize,
gapSN: gapSN,
gapTS: int64(extTimestamp - r.extHighestTS),
timeSinceHighest: packetTime - r.highestTime,
rtpStats: r,
}
r.logger.Warnw(
"time reversed", nil,
zap.Inline(sulf),
"count", r.timeReversedCount,
)
}
@@ -544,8 +607,22 @@ func (r *RTPStatsSender) Update(
}
if extTimestamp < r.extStartTS {
ulgr().Infow(
sulf := &senderUpdateLoggingFields{
packetTime: packetTime,
extSequenceNumber: extSequenceNumber,
extTimestamp: extTimestamp,
marker: marker,
hdrSize: hdrSize,
payloadSize: payloadSize,
paddingSize: paddingSize,
gapSN: gapSN,
gapTS: int64(extTimestamp - r.extHighestTS),
timeSinceHighest: packetTime - r.highestTime,
rtpStats: r,
}
r.logger.Infow(
"adjusting start timestamp",
zap.Inline(sulf),
"snAfter", extSequenceNumber,
"tsAfter", extTimestamp,
)
@@ -628,14 +705,24 @@ func (r *RTPStatsSender) UpdateFromReceiverReport(rr rtcp.ReceptionReport) (rtt
r.lastRR = rr
}()
timeSinceLastRR := func() time.Duration {
if r.lastRRTime != 0 {
return time.Duration(nowNano - r.lastRRTime)
}
return time.Duration(nowNano - r.startTime)
timeSinceLastRR := time.Duration(nowNano - r.startTime)
if r.lastRRTime != 0 {
timeSinceLastRR = time.Duration(nowNano - r.lastRRTime)
}
extReceivedRRSN := extHighestSNFromRR + (r.extStartSN & 0xFFFF_FFFF_FFFF_0000)
if int64(r.extHighestSN-extReceivedRRSN) < 0 || int64(r.extHighestSN-extReceivedRRSN) > (1<<32) {
r.logger.Infow(
"receiver report runaway, dropping",
"timeSinceLastRR", timeSinceLastRR,
"receivedRR", rr,
"extHighestSNFromRR", extHighestSNFromRR,
"extReceivedRRSN", extReceivedRRSN,
"rtpStats", lockedRTPStatsSenderLogEncoder{r},
)
return
}
if r.extHighestSNFromRR != extHighestSNFromRR && int64(r.extHighestSN-extReceivedRRSN) >= (1<<16) {
// there are cases where remote does not send RTCP Receiver Report for extended periods of time,
// some times several minutes, in that interval the sequence number rolls over,
@@ -645,7 +732,7 @@ func (r *RTPStatsSender) UpdateFromReceiverReport(rr rtcp.ReceptionReport) (rtt
// should not roll over for long time, for e. g. it will approximately take 100 days at 500 pps).
// So, there seems to be a remote reporter issue where the sequence number rollover is missed.
//
// catch up till diffrence between highest sent and highest received via receiver report is
// catch up till difference between highest sent and highest received via receiver report is
// less than full 16-bit range.
//
// in a different flavor, there are clients that do not report properly,
@@ -654,7 +741,7 @@ func (r *RTPStatsSender) UpdateFromReceiverReport(rr rtcp.ReceptionReport) (rtt
// RTCP RR does not change.
r.logger.Infow(
"receiver report missed rollover, adjusting",
"timeSinceLastRR", timeSinceLastRR(),
"timeSinceLastRR", timeSinceLastRR,
"receivedRR", rr,
"extHighestSNFromRR", extHighestSNFromRR,
"extReceivedRRSN", extReceivedRRSN,
@@ -667,7 +754,7 @@ func (r *RTPStatsSender) UpdateFromReceiverReport(rr rtcp.ReceptionReport) (rtt
}
r.logger.Infow(
"receiver report missed rollover, adjusted",
"timeSinceLastRR", timeSinceLastRR(),
"timeSinceLastRR", timeSinceLastRR,
"receivedRR", rr,
"extHighestSNFromRR", extHighestSNFromRR,
"extReceivedRRSN", extReceivedRRSN,
@@ -679,7 +766,7 @@ func (r *RTPStatsSender) UpdateFromReceiverReport(rr rtcp.ReceptionReport) (rtt
// if remote adjusts somehow, roll back alignment
r.logger.Infow(
"receiver report caught up rollover, adjusting",
"timeSinceLastRR", timeSinceLastRR(),
"timeSinceLastRR", timeSinceLastRR,
"receivedRR", rr,
"extHighestSNFromRR", extHighestSNFromRR,
"extReceivedRRSN", extReceivedRRSN,
@@ -692,7 +779,7 @@ func (r *RTPStatsSender) UpdateFromReceiverReport(rr rtcp.ReceptionReport) (rtt
}
r.logger.Infow(
"receiver report caught up rollover, adjusted",
"timeSinceLastRR", timeSinceLastRR(),
"timeSinceLastRR", timeSinceLastRR,
"receivedRR", rr,
"extHighestSNFromRR", extHighestSNFromRR,
"extReceivedRRSN", extReceivedRRSN,
@@ -701,9 +788,9 @@ func (r *RTPStatsSender) UpdateFromReceiverReport(rr rtcp.ReceptionReport) (rtt
}
if r.extHighestSNFromRR > extHighestSNFromRR {
r.logger.Infow(
r.logger.Debugw(
"receiver report out-of-order, dropping",
"timeSinceLastRR", timeSinceLastRR(),
"timeSinceLastRR", timeSinceLastRR,
"receivedRR", rr,
"extHighestSNFromRR", extHighestSNFromRR,
"extReceivedRRSN", extReceivedRRSN,
@@ -768,7 +855,7 @@ func (r *RTPStatsSender) UpdateFromReceiverReport(rr rtcp.ReceptionReport) (rtt
"metadata cache overflow",
"senderSnapshotID", i+cFirstSnapshotID,
"senderSnapshot", s,
"timeSinceLastRR", timeSinceLastRR(),
"timeSinceLastRR", timeSinceLastRR,
"receivedRR", rr,
"extReceivedRRSN", extReceivedRRSN,
"packetsInInterval", extReceivedRRSN-s.receiverView.extLastRRSN,
@@ -779,7 +866,16 @@ func (r *RTPStatsSender) UpdateFromReceiverReport(rr rtcp.ReceptionReport) (rtt
}
}
s.receiverView.extLastRRSN = extReceivedRRSN
s.receiverView.processedReceptionReports = append(s.receiverView.processedReceptionReports, rr)
s.receiverView.processedReceptionReports[s.receiverView.processedReceptionReportsHead] = rr
s.receiverView.processedReceptionReportsHead++
if s.receiverView.processedReceptionReportsHead >= cMaxProcessedReceptionReports {
s.receiverView.processedReceptionReportsHead = 0
}
s.receiverView.processedReceptionReportsSize++
if s.receiverView.processedReceptionReportsSize > cMaxProcessedReceptionReports {
s.receiverView.processedReceptionReportsSize = cMaxProcessedReceptionReports
}
}
return
@@ -796,11 +892,11 @@ func (r *RTPStatsSender) MaybeAdjustFirstPacketTime(publisherSRData *livekit.RTC
r.lock.Lock()
defer r.lock.Unlock()
if !r.initialized || publisherSRData == nil {
if !r.initialized || publisherSRData == nil || r.clockRate == 0 {
return
}
if err, loggingFields := r.maybeAdjustFirstPacketTime(publisherSRData, tsOffset, r.extStartTS); err != nil {
if _, loggingFields, err := r.maybeAdjustFirstPacketTime(publisherSRData, tsOffset, r.extStartTS); err != nil {
r.logger.Infow(err.Error(), append(loggingFields, "rtpStats", lockedRTPStatsSenderLogEncoder{r})...)
}
}
@@ -809,14 +905,13 @@ func (r *RTPStatsSender) GetExpectedRTPTimestamp(at time.Time) (expectedTSExt ui
r.lock.RLock()
defer r.lock.RUnlock()
if r.firstTime == 0 {
if r.firstTime == 0 || r.clockRate == 0 {
err = errors.New("uninitialized")
return
}
timeDiff := at.Sub(time.Unix(0, r.firstTime))
expectedRTPDiff := timeDiff.Nanoseconds() * int64(r.params.ClockRate) / 1e9
expectedTSExt = r.extStartTS + uint64(expectedRTPDiff)
expectedTSExt = r.extStartTS + r.rtpConverter.ToRTPExt(timeDiff)
return
}
@@ -824,7 +919,7 @@ func (r *RTPStatsSender) GetRtcpSenderReport(ssrc uint32, publisherSRData *livek
r.lock.Lock()
defer r.lock.Unlock()
if !r.initialized || publisherSRData == nil {
if !r.initialized || publisherSRData == nil || r.clockRate == 0 {
return nil
}
@@ -834,19 +929,21 @@ func (r *RTPStatsSender) GetRtcpSenderReport(ssrc uint32, publisherSRData *livek
nowNTP mediatransportutil.NtpTime
nowRTPExt uint64
)
nowNano := mono.UnixNano()
if passThrough {
reportTime = publisherSRData.At
reportTimeAdjusted = publisherSRData.AtAdjusted
timeSincePublisherSR := time.Duration(nowNano - publisherSRData.At)
reportTime = publisherSRData.At + timeSincePublisherSR.Nanoseconds()
reportTimeAdjusted = publisherSRData.AtAdjusted + timeSincePublisherSR.Nanoseconds()
nowNTP = mediatransportutil.NtpTime(publisherSRData.NtpTimestamp)
nowRTPExt = publisherSRData.RtpTimestampExt - tsOffset
nowNTP = mediatransportutil.ToNtpTime(mediatransportutil.NtpTime(publisherSRData.NtpTimestamp).Time().Add(timeSincePublisherSR))
nowRTPExt = publisherSRData.RtpTimestampExt - tsOffset + r.rtpConverter.ToRTPExt(timeSincePublisherSR)
} else {
timeSincePublisherSRAdjusted := time.Duration(mono.UnixNano() - publisherSRData.AtAdjusted)
timeSincePublisherSRAdjusted := time.Duration(nowNano - publisherSRData.AtAdjusted)
reportTimeAdjusted = publisherSRData.AtAdjusted + timeSincePublisherSRAdjusted.Nanoseconds()
reportTime = reportTimeAdjusted
nowNTP = mediatransportutil.ToNtpTime(time.Unix(0, reportTime))
nowRTPExt = publisherSRData.RtpTimestampExt - tsOffset + uint64(timeSincePublisherSRAdjusted.Nanoseconds()*int64(r.params.ClockRate)/1e9)
nowRTPExt = publisherSRData.RtpTimestampExt - tsOffset + r.rtpConverter.ToRTPExt(timeSincePublisherSRAdjusted)
}
packetCount := uint32(r.getPacketsSeenPlusDuplicates(r.extStartSN, r.extHighestSN))
@@ -861,33 +958,28 @@ func (r *RTPStatsSender) GetRtcpSenderReport(ssrc uint32, publisherSRData *livek
Octets: octetCount,
}
ulgr := func() logger.UnlikelyLogger {
nowNano := mono.UnixNano()
return r.logger.WithUnlikelyValues(
"curr", WrappedRTCPSenderReportStateLogger{srData},
"feed", WrappedRTCPSenderReportStateLogger{publisherSRData},
"tsOffset", tsOffset,
"timeNow", mono.Now(),
"reportTime", time.Unix(0, reportTime),
"reportTimeAdjusted", time.Unix(0, reportTimeAdjusted),
"timeSinceHighest", time.Duration(nowNano-r.highestTime),
"timeSinceFirst", time.Duration(nowNano-r.firstTime),
"timeSincePublisherSRAdjusted", time.Duration(nowNano-publisherSRData.AtAdjusted),
"timeSincePublisherSR", time.Duration(nowNano-publisherSRData.At),
"nowRTPExt", nowRTPExt,
"rtpStats", lockedRTPStatsSenderLogEncoder{r},
)
}
if r.srNewest != nil && nowRTPExt >= r.srNewest.RtpTimestampExt {
timeSinceLastReport := nowNTP.Time().Sub(mediatransportutil.NtpTime(r.srNewest.NtpTimestamp).Time())
if r.srNewest != nil && nowRTPExt >= r.srNewest.RtpTimestampExt && r.clockRate != 0 {
nowTime := nowNTP.Time()
srNewestTime := mediatransportutil.NtpTime(r.srNewest.NtpTimestamp).Time()
timeSinceLastReport := nowTime.Sub(srNewestTime)
rtpDiffSinceLastReport := nowRTPExt - r.srNewest.RtpTimestampExt
windowClockRate := float64(rtpDiffSinceLastReport) / timeSinceLastReport.Seconds()
if timeSinceLastReport.Seconds() > 0.2 && math.Abs(float64(r.params.ClockRate)-windowClockRate) > 0.2*float64(r.params.ClockRate) {
if timeSinceLastReport.Seconds() > 0.2 && math.Abs(float64(r.clockRate)-windowClockRate) > 0.2*float64(r.clockRate) {
r.clockSkewCount++
if (r.clockSkewCount-1)%100 == 0 {
ulgr().Infow(
if shouldLog(r.clockSkewCount) {
srlf := &senderReportLoggingFields{
srData: srData,
publisherSRData: publisherSRData,
tsOffset: tsOffset,
reportTime: reportTime,
reportTimeAdjusted: reportTimeAdjusted,
nowNano: nowNano,
nowRTPExt: nowRTPExt,
rtpStats: r,
}
r.logger.Infow(
"sending sender report, clock skew",
zap.Inline(srlf),
"timeSinceLastReport", timeSinceLastReport,
"rtpDiffSinceLastReport", rtpDiffSinceLastReport,
"windowClockRate", windowClockRate,
@@ -900,7 +992,20 @@ func (r *RTPStatsSender) GetRtcpSenderReport(ssrc uint32, publisherSRData *livek
if r.srNewest != nil && nowRTPExt < r.srNewest.RtpTimestampExt {
// If report being generated is behind the last report, skip it.
// Should not happen.
ulgr().Infow("sending sender report, out-of-order, skipping")
srlf := &senderReportLoggingFields{
srData: srData,
publisherSRData: publisherSRData,
tsOffset: tsOffset,
reportTime: reportTime,
reportTimeAdjusted: reportTimeAdjusted,
nowNano: nowNano,
nowRTPExt: nowRTPExt,
rtpStats: r,
}
r.logger.Infow(
"sending sender report, out-of-order, skipping",
zap.Inline(srlf),
)
return nil
}
@@ -911,7 +1016,7 @@ func (r *RTPStatsSender) GetRtcpSenderReport(ssrc uint32, publisherSRData *livek
r.rttMarker = rttMarker{
ntpTime: nowNTP,
sentAt: mono.Now(),
sentAt: time.Now(),
}
return &rtcp.SenderReport{
@@ -927,7 +1032,7 @@ func (r *RTPStatsSender) DeltaInfo(snapshotID uint32) *RTPDeltaInfo {
r.lock.Lock()
defer r.lock.Unlock()
deltaInfo, err, loggingFields := r.deltaInfo(
deltaInfo, loggingFields, err := r.deltaInfo(
snapshotID,
r.extStartSN,
r.extHighestSN,
@@ -943,9 +1048,13 @@ func (r *RTPStatsSender) DeltaInfoSender(senderSnapshotID uint32) (*RTPDeltaInfo
r.lock.Lock()
defer r.lock.Unlock()
if r.clockRate == 0 {
return nil, nil
}
var deltaStatsSenderView *RTPDeltaInfo
thenSenderView, nowSenderView := r.getAndResetSenderSnapshotWindow(senderSnapshotID)
if thenSenderView != nil && nowSenderView != nil {
thenSenderView, nowSenderView, ok := r.getAndResetSenderSnapshotWindow(senderSnapshotID)
if ok {
startTime := thenSenderView.startTime
endTime := nowSenderView.startTime
@@ -979,7 +1088,7 @@ func (r *RTPStatsSender) DeltaInfoSender(senderSnapshotID uint32) (*RTPDeltaInfo
packetsLostFeed = packetsExpected
}
maxJitterTime := thenSenderView.maxJitterFeed / float64(r.params.ClockRate) * 1e6
maxJitterTime := thenSenderView.maxJitterFeed / float64(r.clockRate) * 1e6
deltaStatsSenderView = &RTPDeltaInfo{
StartTime: time.Unix(0, startTime),
@@ -1007,8 +1116,8 @@ func (r *RTPStatsSender) DeltaInfoSender(senderSnapshotID uint32) (*RTPDeltaInfo
var deltaStatsReceiverView *RTPDeltaInfo
if r.lastRRTime != 0 {
thenReceiverView, nowReceiverView := r.getAndResetSenderSnapshotReceiverView(senderSnapshotID)
if thenReceiverView != nil && nowReceiverView != nil {
thenReceiverView, nowReceiverView, ok := r.getAndResetSenderSnapshotReceiverView(senderSnapshotID)
if ok {
startTime := thenReceiverView.startTime
endTime := nowReceiverView.startTime
@@ -1048,12 +1157,7 @@ func (r *RTPStatsSender) DeltaInfoSender(senderSnapshotID uint32) (*RTPDeltaInfo
packetsLost = packetsExpected
}
// discount jitter from publisher side + internal processing
maxJitter := thenReceiverView.maxJitter - thenReceiverView.maxJitterFeed
if maxJitter < 0.0 {
maxJitter = 0.0
}
maxJitterTime := maxJitter / float64(r.params.ClockRate) * 1e6
maxJitterTime := thenReceiverView.maxJitter / float64(r.clockRate) * 1e6
deltaStatsReceiverView = &RTPDeltaInfo{
StartTime: time.Unix(0, startTime),
@@ -1117,9 +1221,9 @@ func (r *RTPStatsSender) ToProto() *livekit.RTPStats {
return p
}
func (r *RTPStatsSender) getAndResetSenderSnapshotWindow(senderSnapshotID uint32) (*senderSnapshotWindow, *senderSnapshotWindow) {
if !r.initialized {
return nil, nil
func (r *RTPStatsSender) getAndResetSenderSnapshotWindow(senderSnapshotID uint32) (senderSnapshotWindow, senderSnapshotWindow, bool) {
if !r.initialized || senderSnapshotID < cFirstSnapshotID {
return senderSnapshotWindow{}, senderSnapshotWindow{}, false
}
idx := senderSnapshotID - cFirstSnapshotID
@@ -1131,7 +1235,7 @@ func (r *RTPStatsSender) getAndResetSenderSnapshotWindow(senderSnapshotID uint32
// snapshot now
r.senderSnapshots[idx].senderView = r.getSenderSnapshotWindow(mono.UnixNano())
return &then.senderView, &r.senderSnapshots[idx].senderView
return then.senderView, r.senderSnapshots[idx].senderView, true
}
func (r *RTPStatsSender) getSenderSnapshotWindow(startTime int64) senderSnapshotWindow {
@@ -1158,9 +1262,9 @@ func (r *RTPStatsSender) getSenderSnapshotWindow(startTime int64) senderSnapshot
}
}
func (r *RTPStatsSender) getAndResetSenderSnapshotReceiverView(senderSnapshotID uint32) (*senderSnapshotReceiverView, *senderSnapshotReceiverView) {
if !r.initialized || r.lastRRTime == 0 {
return nil, nil
func (r *RTPStatsSender) getAndResetSenderSnapshotReceiverView(senderSnapshotID uint32) (senderSnapshotReceiverView, senderSnapshotReceiverView, bool) {
if !r.initialized || r.lastRRTime == 0 || senderSnapshotID < cFirstSnapshotID {
return senderSnapshotReceiverView{}, senderSnapshotReceiverView{}, false
}
idx := senderSnapshotID - cFirstSnapshotID
@@ -1172,7 +1276,7 @@ func (r *RTPStatsSender) getAndResetSenderSnapshotReceiverView(senderSnapshotID
// snapshot now
r.senderSnapshots[idx].receiverView = r.getSenderSnapshotReceiverView(r.lastRRTime, &then.receiverView)
return &then.receiverView, &r.senderSnapshots[idx].receiverView
return then.receiverView, r.senderSnapshots[idx].receiverView, true
}
func (r *RTPStatsSender) getSenderSnapshotReceiverView(startTime int64, s *senderSnapshotReceiverView) senderSnapshotReceiverView {
@@ -1251,11 +1355,13 @@ func (r *RTPStatsSender) clearSnInfos(extStartInclusive uint64, extEndExclusive
return
}
if extEndExclusive-extStartInclusive > uint64(len(r.snInfos)) {
clear(r.snInfos)
return
}
for esn := extStartInclusive; esn != extEndExclusive; esn++ {
snInfo := &r.snInfos[int(esn)%len(r.snInfos)]
snInfo.pktSize = 0
snInfo.hdrSize = 0
snInfo.flags = 0
r.snInfos[int(esn)%len(r.snInfos)] = snInfo{}
}
}
@@ -1273,14 +1379,18 @@ func (r *RTPStatsSender) getIntervalStats(
extEndExclusive uint64,
ehsn uint64,
) (intervalStats intervalStats) {
processESN := func(esn uint64, ehsn uint64) {
slot := r.getSnInfoOutOfOrderSlot(esn, ehsn)
if slot < 0 {
intervalStats.packetsNotFoundMetadata++
return
}
upperBound := ehsn + 1
lowerBound := uint64(0)
if n := uint64(len(r.snInfos)); n != 0 && ehsn >= n-1 {
lowerBound = ehsn - n + 1
}
extStartInclusiveClamped := max(min(extStartInclusive, upperBound), lowerBound)
extEndExclusiveClamped := max(min(extEndExclusive, upperBound), extStartInclusiveClamped)
snInfo := &r.snInfos[slot]
intervalStats.packetsNotFoundMetadata = (extEndExclusive - extStartInclusive) - (extEndExclusiveClamped - extStartInclusiveClamped)
for esn := extStartInclusiveClamped; esn != extEndExclusiveClamped; esn++ {
snInfo := &r.snInfos[int(esn)%len(r.snInfos)]
switch {
case snInfo.pktSize == 0:
intervalStats.packetsLostFeed++
@@ -1303,10 +1413,6 @@ func (r *RTPStatsSender) getIntervalStats(
intervalStats.frames++
}
}
for esn := extStartInclusive; esn != extEndExclusive; esn++ {
processESN(esn, ehsn)
}
return
}
pkg/sfu/rtpstats/rtpstats_receiver_test.go → pkg/sfu/rtpstats/rtpstats_test.go
+67 -9
View File
@@ -21,8 +21,6 @@ import (
"github.com/pion/rtp"
"github.com/stretchr/testify/require"
"github.com/livekit/protocol/logger"
)
func getPacket(sn uint16, ts uint32, payloadSize int) *rtp.Packet {
@@ -37,10 +35,8 @@ func getPacket(sn uint16, ts uint32, payloadSize int) *rtp.Packet {
func Test_RTPStatsReceiver_Update(t *testing.T) {
clockRate := uint32(90000)
r := NewRTPStatsReceiver(RTPStatsParams{
ClockRate: clockRate,
Logger: logger.GetLogger(),
})
r := NewRTPStatsReceiver(RTPStatsParams{})
r.SetClockRate(clockRate)
sequenceNumber := uint16(rand.Float64() * float64(1<<16))
timestamp := uint32(rand.Float64() * float64(1<<32))
@@ -89,7 +85,7 @@ func Test_RTPStatsReceiver_Update(t *testing.T) {
len(packet.Payload),
0,
)
require.True(t, flowState.IsNotHandled)
require.Equal(t, RTPFlowUnhandledReasonPreStartTimestamp, flowState.UnhandledReason)
require.Equal(t, sequenceNumber, r.sequenceNumber.GetHighest())
require.Equal(t, sequenceNumber, uint16(r.sequenceNumber.GetExtendedHighest()))
require.Equal(t, timestamp, r.timestamp.GetHighest())
@@ -108,7 +104,7 @@ func Test_RTPStatsReceiver_Update(t *testing.T) {
len(packet.Payload),
0,
)
require.True(t, flowState.IsNotHandled)
require.Equal(t, RTPFlowUnhandledReasonPreStartTimestamp, flowState.UnhandledReason)
require.Equal(t, sequenceNumber, r.sequenceNumber.GetHighest())
require.Equal(t, sequenceNumber, uint16(r.sequenceNumber.GetExtendedHighest()))
require.Equal(t, timestamp, r.timestamp.GetHighest())
@@ -218,7 +214,7 @@ func Test_RTPStatsReceiver_Update(t *testing.T) {
len(packet.Payload),
0,
)
require.True(t, flowState.IsNotHandled)
require.Equal(t, RTPFlowUnhandledReasonOldSequenceNumber, flowState.UnhandledReason)
require.Equal(t, sequenceNumber, r.sequenceNumber.GetHighest())
require.Equal(t, sequenceNumber, uint16(r.sequenceNumber.GetExtendedHighest()))
require.Equal(t, timestamp, r.timestamp.GetHighest())
@@ -226,3 +222,65 @@ func Test_RTPStatsReceiver_Update(t *testing.T) {
r.Stop()
}
func Test_RTPStatsReceiver_Restart(t *testing.T) {
clockRate := uint32(90000)
r := NewRTPStatsReceiver(RTPStatsParams{})
r.SetClockRate(clockRate)
// should not restart till there are at least threshold packets
require.False(t, r.maybeRestart(10, 20, 1000))
require.False(t, r.maybeRestart(11, 20, 1000))
require.False(t, r.maybeRestart(13, 20, 1000))
require.False(t, r.maybeRestart(14, 20, 1000))
// although adding 5th packet should have enough packets for a check,
// still should not restart as there is a sequence number gap between 11 and 13
require.False(t, r.maybeRestart(15, 20, 1000))
require.False(t, r.maybeRestart(16, 19, 1000))
// has enough packets, but still cannot restart because timestamps are not increasing
require.False(t, r.maybeRestart(17, 21, 1000))
require.False(t, r.maybeRestart(18, 21, 1000))
require.False(t, r.maybeRestart(19, 21, 1000))
// can restart as there are enough packets with proper sequencing
require.True(t, r.maybeRestart(20, 21, 1000))
require.Equal(t, restartThreshold, r.restartPacketsN)
r.resetRestart()
require.Zero(t, r.restartPacketsN)
r.Stop()
}
func Test_RTPStatsSender_getIntervalStats(t *testing.T) {
t.Run("packetsNotFoundMetadata should match lost packets", func(t *testing.T) {
r := NewRTPStatsSender(RTPStatsParams{}, 1024)
stats := r.getIntervalStats(0, 10000, 10000)
require.EqualValues(t, 8977, stats.packetsNotFoundMetadata)
})
}
func BenchmarkRTPStatsReceiver_Update(b *testing.B) {
const clockRate = 90000
const hdrSize = 12
const payloadSize = 1200
const paddingSize = 0
const tsIncrement = 3000 // ~33ms at 90kHz
r := NewRTPStatsReceiver(RTPStatsParams{})
r.SetClockRate(clockRate)
// initialize before running benchmark loop
packetTime := time.Now().UnixNano()
sn := uint16(0)
ts := uint32(0)
r.Update(packetTime, sn, ts, false, hdrSize, payloadSize, paddingSize)
b.ResetTimer()
for i := 0; i < b.N; i++ {
sn++
ts += tsIncrement
packetTime += 33 * int64(time.Millisecond) // ~30fps
marker := (i+1)%100 == 0 // marker every 100 packets
r.Update(packetTime, sn, ts, marker, hdrSize, payloadSize, paddingSize)
}
}
pkg/sfu/sequencer.go
+4
View File
@@ -119,6 +119,10 @@ type sequencer struct {
}
func newSequencer(size int, maybeSparse bool, logger logger.Logger) *sequencer {
if size == 0 {
return nil
}
s := &sequencer{
size: size,
startTime: time.Now().UnixNano(),
pkg/sfu/sfu.go
+10 -7
View File
@@ -16,18 +16,21 @@ package sfu
import (
"sync"
"github.com/pion/rtp"
)
var (
PacketFactory *sync.Pool
)
func init() {
// Init packet factory
PacketFactory = &sync.Pool{
New: func() interface{} {
New: func() any {
b := make([]byte, 1460)
return &b
},
}
}
RTPHeaderFactory = &sync.Pool{
New: func() any {
return &rtp.Header{}
},
}
)
pkg/sfu/streamallocator/streamallocator.go
+155 -128
View File
@@ -16,6 +16,7 @@ package streamallocator
import (
"fmt"
"slices"
"sort"
"sync"
"time"
@@ -37,21 +38,24 @@ import (
)
const (
ChannelCapacityInfinity = 100 * 1000 * 1000 // 100 Mbps
cChannelCapacityInfinity = 100 * 1000 * 1000 // 100 Mbps
PriorityMin = uint8(1)
PriorityMax = uint8(255)
PriorityDefaultScreenshare = PriorityMax
PriorityDefaultVideo = PriorityMin
cPriorityMin = uint8(1)
cPriorityMax = uint8(255)
cPriorityDefaultScreenshare = cPriorityMax
cPriorityDefaultVideo = cPriorityMin
FlagAllowOvershootWhileOptimal = true
FlagAllowOvershootWhileDeficient = false
FlagAllowOvershootExemptTrackWhileDeficient = true
FlagAllowOvershootInProbe = true
FlagAllowOvershootInCatchup = false
FlagAllowOvershootInBoost = true
cFlagAllowOvershootWhileOptimal = true
cFlagAllowOvershootWhileDeficient = false
cFlagAllowOvershootExemptTrackWhileDeficient = true
cFlagAllowOvershootInProbe = true
cFlagAllowOvershootInCatchup = false
cFlagAllowOvershootInBoost = true
cRTTPullInterval = 30 * time.Second
cPingLong = cRTTPullInterval / 2
cPingShort = 100 * time.Millisecond
)
// ---------------------------------------------------------------------------
@@ -131,13 +135,45 @@ func (s streamAllocatorSignal) String() string {
type Event struct {
*StreamAllocator
Signal streamAllocatorSignal
TrackID livekit.TrackID
Data interface{}
signal streamAllocatorSignal
// only one of these fields will be set based on event type
trackID livekit.TrackID
allowPause bool
channelCapacity int64
twccFeedback *rtcp.TransportLayerCC
congestionStateChangeData congestionStateChangeData
probeBytesToSend int
probeClusterId ccutils.ProbeClusterId
probeClusterInfo ccutils.ProbeClusterInfo
}
func (e Event) String() string {
return fmt.Sprintf("StreamAllocator:Event{signal: %s, trackID: %s, data: %+v}", e.Signal, e.TrackID, e.Data)
value := ""
switch e.signal {
case streamAllocatorSignalAllocateTrack, streamAllocatorSignalResume:
value = fmt.Sprintf("trackID: %s", e.trackID)
case streamAllocatorSignalEstimate, streamAllocatorSignalSetChannelCapacity:
value = fmt.Sprintf("channelCapacity: %d", e.channelCapacity)
case streamAllocatorSignalFeedback:
value = fmt.Sprintf("twccFeedback: %+v", e.twccFeedback)
case streamAllocatorSignalProbeClusterSwitch:
value = fmt.Sprintf("probeClusterInfo: %+v", e.probeClusterInfo)
case streamAllocatorSignalSendProbe:
value = fmt.Sprintf("probeBytesToSend: %d", e.probeBytesToSend)
case streamAllocatorSignalPacerProbeObserverClusterComplete:
value = fmt.Sprintf("probeClusterId: %v", e.probeClusterId)
case streamAllocatorSignalSetAllowPause:
value = fmt.Sprintf("allowPause: %v", e.allowPause)
case streamAllocatorSignalCongestionStateChange:
value = fmt.Sprintf("congestStatecChangeData: %+v", e.congestionStateChangeData)
}
if value == "" {
return fmt.Sprintf("StreamAllocator:Event{signal: %s}", e.signal)
}
return fmt.Sprintf("StreamAllocator:Event{signal: %s, %s}", e.signal, value)
}
// ---------------------------------------------------------------------------
@@ -202,6 +238,7 @@ type StreamAllocator struct {
videoTracksMu sync.RWMutex
videoTracks map[livekit.TrackID]*Track
videoTracksShadow []*Track
isAllocateAllPending bool
rembTrackingSSRC uint32
@@ -215,6 +252,8 @@ type StreamAllocator struct {
lastRTTTime time.Time
pingGeneration atomic.Uint32
isStopped atomic.Bool
}
@@ -247,7 +286,7 @@ func NewStreamAllocator(params StreamAllocatorParams, enabled bool, allowPause b
func (s *StreamAllocator) Start() {
s.eventsQueue.Start()
go s.ping()
go s.ping(s.pingGeneration.Inc(), cPingLong)
}
func (s *StreamAllocator) Stop() {
@@ -273,10 +312,10 @@ func (s *StreamAllocator) SetSendSideBWEInterceptor(sendSideBWEInterceptor cc.Ba
}
type AddTrackParams struct {
Source livekit.TrackSource
Priority uint8
IsSimulcast bool
PublisherID livekit.ParticipantID
Source livekit.TrackSource
Priority uint8
IsMultiLayered bool
PublisherID livekit.ParticipantID
}
func (s *StreamAllocator) AddTrack(downTrack *sfu.DownTrack, params AddTrackParams) {
@@ -284,13 +323,14 @@ func (s *StreamAllocator) AddTrack(downTrack *sfu.DownTrack, params AddTrackPara
return
}
track := NewTrack(downTrack, params.Source, params.IsSimulcast, params.PublisherID, s.params.Logger)
track := NewTrack(downTrack, params.Source, params.IsMultiLayered, params.PublisherID, s.params.Logger)
track.SetPriority(params.Priority)
trackID := livekit.TrackID(downTrack.ID())
s.videoTracksMu.Lock()
oldTrack := s.videoTracks[trackID]
s.videoTracks[trackID] = track
s.shadowVideoTracksLocked()
s.videoTracksMu.Unlock()
if oldTrack != nil {
@@ -307,12 +347,13 @@ func (s *StreamAllocator) RemoveTrack(downTrack *sfu.DownTrack) {
s.videoTracksMu.Lock()
if existing := s.videoTracks[livekit.TrackID(downTrack.ID())]; existing != nil && existing.DownTrack() == downTrack {
delete(s.videoTracks, livekit.TrackID(downTrack.ID()))
s.shadowVideoTracksLocked()
}
s.videoTracksMu.Unlock()
// STREAM-ALLOCATOR-TODO: use any saved bandwidth to re-distribute
s.postEvent(Event{
Signal: streamAllocatorSignalAdjustState,
signal: streamAllocatorSignalAdjustState,
})
}
@@ -324,7 +365,7 @@ func (s *StreamAllocator) SetTrackPriority(downTrack *sfu.DownTrack, priority ui
// do a full allocation on a track priority change to keep it simple
s.isAllocateAllPending = true
s.postEvent(Event{
Signal: streamAllocatorSignalAllocateAllTracks,
signal: streamAllocatorSignalAllocateAllTracks,
})
}
}
@@ -333,15 +374,15 @@ func (s *StreamAllocator) SetTrackPriority(downTrack *sfu.DownTrack, priority ui
func (s *StreamAllocator) SetAllowPause(allowPause bool) {
s.postEvent(Event{
Signal: streamAllocatorSignalSetAllowPause,
Data: allowPause,
signal: streamAllocatorSignalSetAllowPause,
allowPause: allowPause,
})
}
func (s *StreamAllocator) SetChannelCapacity(channelCapacity int64) {
s.postEvent(Event{
Signal: streamAllocatorSignalSetChannelCapacity,
Data: channelCapacity,
signal: streamAllocatorSignalSetChannelCapacity,
channelCapacity: channelCapacity,
})
}
@@ -385,13 +426,7 @@ func (s *StreamAllocator) OnREMB(downTrack *sfu.DownTrack, remb *rtcp.ReceiverEs
downTrackSSRCRTX = track.DownTrack().SSRCRTX()
}
found := false
for _, ssrc := range remb.SSRCs {
if ssrc == s.rembTrackingSSRC {
found = true
break
}
}
found := slices.Contains(remb.SSRCs, s.rembTrackingSSRC)
if !found {
if len(remb.SSRCs) == 0 {
s.params.Logger.Warnw("stream allocator: no SSRC to track REMB", nil)
@@ -429,24 +464,24 @@ func (s *StreamAllocator) OnREMB(downTrack *sfu.DownTrack, remb *rtcp.ReceiverEs
s.videoTracksMu.Unlock()
s.postEvent(Event{
Signal: streamAllocatorSignalEstimate,
Data: int64(remb.Bitrate),
signal: streamAllocatorSignalEstimate,
channelCapacity: int64(remb.Bitrate),
})
}
// called when a new transport-cc feedback is received
func (s *StreamAllocator) OnTransportCCFeedback(downTrack *sfu.DownTrack, fb *rtcp.TransportLayerCC) {
s.postEvent(Event{
Signal: streamAllocatorSignalFeedback,
Data: fb,
signal: streamAllocatorSignalFeedback,
twccFeedback: fb,
})
}
// called when target bitrate changes (send side bandwidth estimation)
func (s *StreamAllocator) onTargetBitrateChange(bitrate int) {
s.postEvent(Event{
Signal: streamAllocatorSignalEstimate,
Data: int64(bitrate),
signal: streamAllocatorSignalEstimate,
channelCapacity: int64(bitrate),
})
}
@@ -460,8 +495,8 @@ type congestionStateChangeData struct {
// BWEListener implementation
func (s *StreamAllocator) OnCongestionStateChange(fromState bwe.CongestionState, toState bwe.CongestionState, estimatedAvailableChannelCapacity int64) {
s.postEvent(Event{
Signal: streamAllocatorSignalCongestionStateChange,
Data: congestionStateChangeData{fromState, toState, estimatedAvailableChannelCapacity},
signal: streamAllocatorSignalCongestionStateChange,
congestionStateChangeData: congestionStateChangeData{fromState, toState, estimatedAvailableChannelCapacity},
})
}
@@ -503,8 +538,8 @@ func (s *StreamAllocator) OnSubscribedLayerChanged(downTrack *sfu.DownTrack, lay
if shouldPost {
s.postEvent(Event{
Signal: streamAllocatorSignalAllocateTrack,
TrackID: livekit.TrackID(downTrack.ID()),
signal: streamAllocatorSignalAllocateTrack,
trackID: livekit.TrackID(downTrack.ID()),
})
}
}
@@ -512,32 +547,32 @@ func (s *StreamAllocator) OnSubscribedLayerChanged(downTrack *sfu.DownTrack, lay
// called when forwarder resumes a track
func (s *StreamAllocator) OnResume(downTrack *sfu.DownTrack) {
s.postEvent(Event{
Signal: streamAllocatorSignalResume,
TrackID: livekit.TrackID(downTrack.ID()),
signal: streamAllocatorSignalResume,
trackID: livekit.TrackID(downTrack.ID()),
})
}
// called when probe cluster changes
func (s *StreamAllocator) OnProbeClusterSwitch(pci ccutils.ProbeClusterInfo) {
s.postEvent(Event{
Signal: streamAllocatorSignalProbeClusterSwitch,
Data: pci,
signal: streamAllocatorSignalProbeClusterSwitch,
probeClusterInfo: pci,
})
}
// called when prober wants to send packet(s)
func (s *StreamAllocator) OnSendProbe(bytesToSend int) {
s.postEvent(Event{
Signal: streamAllocatorSignalSendProbe,
Data: bytesToSend,
signal: streamAllocatorSignalSendProbe,
probeBytesToSend: bytesToSend,
})
}
// called when pacer probe observer observes a cluster completion
func (s *StreamAllocator) OnPacerProbeObserverClusterComplete(probeClusterId ccutils.ProbeClusterId) {
s.postEvent(Event{
Signal: streamAllocatorSignalPacerProbeObserverClusterComplete,
Data: probeClusterId,
signal: streamAllocatorSignalPacerProbeObserverClusterComplete,
probeClusterId: probeClusterId,
})
}
@@ -557,6 +592,10 @@ func (s *StreamAllocator) IsBWEEnabled(downTrack *sfu.DownTrack) bool {
return true
}
func (s *StreamAllocator) BWEType() bwe.BWEType {
return s.params.BWE.Type()
}
// called to check if track subscription mute can be applied
func (s *StreamAllocator) IsSubscribeMutable(downTrack *sfu.DownTrack) bool {
s.videoTracksMu.Lock()
@@ -579,24 +618,24 @@ func (s *StreamAllocator) maybePostEventAllocateTrack(downTrack *sfu.DownTrack)
if shouldPost {
s.postEvent(Event{
Signal: streamAllocatorSignalAllocateTrack,
TrackID: livekit.TrackID(downTrack.ID()),
signal: streamAllocatorSignalAllocateTrack,
trackID: livekit.TrackID(downTrack.ID()),
})
}
}
func (s *StreamAllocator) ping() {
ticker := time.NewTicker(100 * time.Millisecond)
func (s *StreamAllocator) ping(pingGeneration uint32, interval time.Duration) {
ticker := time.NewTicker(interval)
defer ticker.Stop()
for {
<-ticker.C
if s.isStopped.Load() {
if s.isStopped.Load() || (pingGeneration != s.pingGeneration.Load()) {
return
}
s.postEvent(Event{
Signal: streamAllocatorSignalPeriodicPing,
signal: streamAllocatorSignalPeriodicPing,
})
}
}
@@ -604,7 +643,7 @@ func (s *StreamAllocator) ping() {
func (s *StreamAllocator) postEvent(event Event) {
event.StreamAllocator = s
s.eventsQueue.Enqueue(func(event Event) {
switch event.Signal {
switch event.signal {
case streamAllocatorSignalAllocateTrack:
event.handleSignalAllocateTrack(event)
case streamAllocatorSignalAllocateAllTracks:
@@ -637,7 +676,7 @@ func (s *StreamAllocator) postEvent(event Event) {
func (s *StreamAllocator) handleSignalAllocateTrack(event Event) {
s.videoTracksMu.Lock()
track := s.videoTracks[event.TrackID]
track := s.videoTracks[event.trackID]
if track != nil {
track.SetDirty(false)
}
@@ -663,13 +702,11 @@ func (s *StreamAllocator) handleSignalAdjustState(Event) {
}
func (s *StreamAllocator) handleSignalEstimate(event Event) {
receivedEstimate := event.Data.(int64)
// always update NACKs
packetDelta, repeatedNackDelta := s.getNackDelta()
s.params.BWE.HandleREMB(
receivedEstimate,
event.channelCapacity,
s.getExpectedBandwidthUsage(),
packetDelta,
repeatedNackDelta,
@@ -677,7 +714,7 @@ func (s *StreamAllocator) handleSignalEstimate(event Event) {
}
func (s *StreamAllocator) handleSignalFeedback(event Event) {
fb := event.Data.(*rtcp.TransportLayerCC)
fb := event.twccFeedback
if s.sendSideBWEInterceptor != nil {
s.sendSideBWEInterceptor.WriteRTCP([]rtcp.Packet{fb}, nil)
}
@@ -745,7 +782,7 @@ func (s *StreamAllocator) handleSignalPeriodicPing(Event) {
}
func (s *StreamAllocator) handleSignalProbeClusterSwitch(event Event) {
pci := event.Data.(ccutils.ProbeClusterInfo)
pci := event.probeClusterInfo
s.activeProbeClusterId = pci.Id
s.activeProbeGoalReached = false
s.activeProbeCongesting = false
@@ -754,19 +791,19 @@ func (s *StreamAllocator) handleSignalProbeClusterSwitch(event Event) {
s.params.Pacer.StartProbeCluster(pci)
for _, t := range s.getTracks() {
for _, t := range s.getVideoTracks() {
t.DownTrack().SetProbeClusterId(pci.Id)
}
}
func (s *StreamAllocator) handleSignalSendProbe(event Event) {
bytesToSend := event.Data.(int)
bytesToSend := event.probeBytesToSend
if bytesToSend <= 0 {
return
}
bytesSent := 0
for _, track := range s.getTracks() {
for _, track := range s.getVideoTracks() {
sent := track.WriteProbePackets(bytesToSend)
bytesSent += sent
bytesToSend -= sent
@@ -779,10 +816,9 @@ func (s *StreamAllocator) handleSignalSendProbe(event Event) {
}
func (s *StreamAllocator) handleSignalPacerProbeObserverClusterComplete(event Event) {
probeClusterId, _ := event.Data.(ccutils.ProbeClusterId)
pci := s.params.Pacer.EndProbeCluster(probeClusterId)
pci := s.params.Pacer.EndProbeCluster(event.probeClusterId)
for _, t := range s.getTracks() {
for _, t := range s.getVideoTracks() {
t.DownTrack().SwapProbeClusterId(pci.Id, ccutils.ProbeClusterIdInvalid)
}
@@ -792,7 +828,7 @@ func (s *StreamAllocator) handleSignalPacerProbeObserverClusterComplete(event Ev
func (s *StreamAllocator) handleSignalResume(event Event) {
s.videoTracksMu.Lock()
track := s.videoTracks[event.TrackID]
track := s.videoTracks[event.trackID]
updated := track != nil && track.SetStreamState(StreamStateActive)
s.videoTracksMu.Unlock()
@@ -804,11 +840,11 @@ func (s *StreamAllocator) handleSignalResume(event Event) {
}
func (s *StreamAllocator) handleSignalSetAllowPause(event Event) {
s.allowPause = event.Data.(bool)
s.allowPause = event.allowPause
}
func (s *StreamAllocator) handleSignalSetChannelCapacity(event Event) {
s.overriddenChannelCapacity = event.Data.(int64)
s.overriddenChannelCapacity = event.channelCapacity
if s.overriddenChannelCapacity > 0 {
s.params.Logger.Infow("allocating on override channel capacity", "override", s.overriddenChannelCapacity)
s.allocateAllTracks()
@@ -818,7 +854,7 @@ func (s *StreamAllocator) handleSignalSetChannelCapacity(event Event) {
}
func (s *StreamAllocator) handleSignalCongestionStateChange(event Event) {
cscd := event.Data.(congestionStateChangeData)
cscd := event.congestionStateChangeData
if cscd.toState != bwe.CongestionStateNone {
// end/abort any running probe if channel is not clear
s.maybeStopProbe()
@@ -828,8 +864,8 @@ func (s *StreamAllocator) handleSignalCongestionStateChange(event Event) {
// during early warning hold (if there was one)
if isHoldableCongestionState(cscd.fromState) && cscd.toState == bwe.CongestionStateNone && s.state == streamAllocatorStateStable {
update := NewStreamStateUpdate()
for _, track := range s.getTracks() {
allocation := track.AllocateOptimal(FlagAllowOvershootWhileOptimal, false)
for _, track := range s.getVideoTracks() {
allocation := track.AllocateOptimal(cFlagAllowOvershootWhileOptimal, false)
updateStreamStateChange(track, allocation, update)
}
s.maybeSendUpdate(update)
@@ -875,11 +911,14 @@ func (s *StreamAllocator) setState(state streamAllocatorState) {
s.params.BWE.Reset()
s.activeProbeClusterId = ccutils.ProbeClusterIdInvalid
go s.ping(s.pingGeneration.Inc(), cPingLong)
} else {
go s.ping(s.pingGeneration.Inc(), cPingShort)
}
}
func (s *StreamAllocator) adjustState() {
for _, track := range s.getTracks() {
for _, track := range s.getVideoTracks() {
if track.IsDeficient() {
s.setState(streamAllocatorStateDeficient)
return
@@ -897,7 +936,7 @@ func (s *StreamAllocator) allocateTrack(track *Track) {
bweCongestionState := s.params.BWE.CongestionState()
if !s.enabled || (s.state == streamAllocatorStateStable && !isDeficientCongestionState(bweCongestionState)) || !track.IsManaged() {
update := NewStreamStateUpdate()
allocation := track.AllocateOptimal(FlagAllowOvershootWhileOptimal, isHoldableCongestionState(bweCongestionState))
allocation := track.AllocateOptimal(cFlagAllowOvershootWhileOptimal, isHoldableCongestionState(bweCongestionState))
updateStreamStateChange(track, allocation, update)
s.maybeSendUpdate(update)
return
@@ -923,7 +962,7 @@ func (s *StreamAllocator) allocateTrack(track *Track) {
// track needing change is not currently streaming, i. e. it has to be resumed.
//
track.ProvisionalAllocatePrepare()
transition := track.ProvisionalAllocateGetCooperativeTransition(FlagAllowOvershootWhileDeficient)
transition := track.ProvisionalAllocateGetCooperativeTransition(cFlagAllowOvershootWhileDeficient)
// downgrade, giving back bits
if transition.From.GreaterThan(transition.To) {
@@ -969,7 +1008,7 @@ func (s *StreamAllocator) allocateTrack(track *Track) {
availableChannelCapacity,
layer,
s.allowPause,
FlagAllowOvershootWhileDeficient,
cFlagAllowOvershootWhileDeficient,
)
if availableChannelCapacity < usedChannelCapacity {
break alloc_loop
@@ -995,14 +1034,14 @@ func (s *StreamAllocator) allocateTrack(track *Track) {
}
track.ProvisionalAllocateReset()
transition = track.ProvisionalAllocateGetCooperativeTransition(FlagAllowOvershootWhileDeficient) // get transition again to reset above allocation attempt using available headroom
transition = track.ProvisionalAllocateGetCooperativeTransition(cFlagAllowOvershootWhileDeficient) // get transition again to reset above allocation attempt using available headroom
}
// if there is not enough headroom, try to redistribute starting with tracks that are closest to their desired.
bandwidthAcquired := int64(0)
var contributingTracks []*Track
minDistanceSorted := s.getMinDistanceSorted(track)
minDistanceSorted := s.getMinDistanceSortedVideoTracks(track)
for _, t := range minDistanceSorted {
t.ProvisionalAllocatePrepare()
}
@@ -1052,7 +1091,7 @@ func (s *StreamAllocator) maybeStopProbe() {
pci := s.params.Pacer.EndProbeCluster(s.activeProbeClusterId)
for _, t := range s.getTracks() {
for _, t := range s.getVideoTracks() {
t.DownTrack().SwapProbeClusterId(pci.Id, ccutils.ProbeClusterIdInvalid)
}
@@ -1074,11 +1113,11 @@ func (s *StreamAllocator) maybeBoostDeficientTracks() {
update := NewStreamStateUpdate()
sortedTracks := s.getMaxDistanceSortedDeficient()
sortedTracks := s.getMaxDistanceSortedDeficientVideoTracks()
boost_loop:
for {
for idx, track := range sortedTracks {
allocation, boosted := track.AllocateNextHigher(availableChannelCapacity, FlagAllowOvershootInCatchup)
allocation, boosted := track.AllocateNextHigher(availableChannelCapacity, cFlagAllowOvershootInCatchup)
if !boosted {
if idx == len(sortedTracks)-1 {
// all tracks tried
@@ -1096,7 +1135,7 @@ boost_loop:
break // sort again below as the track that was just boosted could still be farthest from its desired
}
sortedTracks = s.getMaxDistanceSortedDeficient()
sortedTracks = s.getMaxDistanceSortedDeficientVideoTracks()
if len(sortedTracks) == 0 {
break // nothing available to boost
}
@@ -1133,13 +1172,13 @@ func (s *StreamAllocator) allocateAllTracks() {
// This pass is to find out if there is any leftover channel capacity after allocating exempt tracks.
// Exempt tracks are given optimal allocation (i. e. no bandwidth constraint) so that they do not fail allocation.
//
videoTracks := s.getTracks()
videoTracks := s.getVideoTracks()
for _, track := range videoTracks {
if track.IsManaged() {
continue
}
allocation := track.AllocateOptimal(FlagAllowOvershootExemptTrackWhileDeficient, false)
allocation := track.AllocateOptimal(cFlagAllowOvershootExemptTrackWhileDeficient, false)
updateStreamStateChange(track, allocation, update)
// STREAM-ALLOCATOR-TODO: optimistic allocation before bitrate is available will return 0. How to account for that?
@@ -1162,7 +1201,7 @@ func (s *StreamAllocator) allocateAllTracks() {
updateStreamStateChange(track, allocation, update)
}
} else {
sorted := s.getSorted()
sorted := s.getSortedVideoTracks()
for _, track := range sorted {
track.ProvisionalAllocatePrepare()
}
@@ -1175,7 +1214,7 @@ func (s *StreamAllocator) allocateAllTracks() {
}
for _, track := range sorted {
_, usedChannelCapacity := track.ProvisionalAllocate(availableChannelCapacity, layer, s.allowPause, FlagAllowOvershootWhileDeficient)
_, usedChannelCapacity := track.ProvisionalAllocate(availableChannelCapacity, layer, s.allowPause, cFlagAllowOvershootWhileDeficient)
availableChannelCapacity -= usedChannelCapacity
if availableChannelCapacity < 0 {
availableChannelCapacity = 0
@@ -1239,7 +1278,7 @@ func (s *StreamAllocator) getAvailableChannelCapacity(allowOverride bool) int64
func (s *StreamAllocator) getExpectedBandwidthUsage() int64 {
expected := int64(0)
for _, track := range s.getTracks() {
for _, track := range s.getVideoTracks() {
expected += track.BandwidthRequested()
}
@@ -1248,14 +1287,8 @@ func (s *StreamAllocator) getExpectedBandwidthUsage() int64 {
func (s *StreamAllocator) getExpectedBandwidthUsageWithoutTracks(filteredTracks []*Track) int64 {
expected := int64(0)
for _, track := range s.getTracks() {
filtered := false
for _, ft := range filteredTracks {
if ft == track {
filtered = true
break
}
}
for _, track := range s.getVideoTracks() {
filtered := slices.Contains(filteredTracks, track)
if !filtered {
expected += track.BandwidthRequested()
}
@@ -1275,7 +1308,7 @@ func (s *StreamAllocator) getAvailableHeadroomWithoutTracks(allowOverride bool,
func (s *StreamAllocator) getNackDelta() (uint32, uint32) {
aggPacketDelta := uint32(0)
aggRepeatedNackDelta := uint32(0)
for _, track := range s.getTracks() {
for _, track := range s.getVideoTracks() {
packetDelta, nackDelta := track.GetNackDelta()
aggPacketDelta += packetDelta
aggRepeatedNackDelta += nackDelta
@@ -1305,8 +1338,8 @@ func (s *StreamAllocator) maybeProbe() {
func (s *StreamAllocator) maybeProbeWithMedia() {
// boost deficient track farthest from desired layer
for _, track := range s.getMaxDistanceSortedDeficient() {
allocation, boosted := track.AllocateNextHigher(ChannelCapacityInfinity, FlagAllowOvershootInBoost)
for _, track := range s.getMaxDistanceSortedDeficientVideoTracks() {
allocation, boosted := track.AllocateNextHigher(cChannelCapacityInfinity, cFlagAllowOvershootInBoost)
if !boosted {
continue
}
@@ -1322,17 +1355,14 @@ func (s *StreamAllocator) maybeProbeWithMedia() {
func (s *StreamAllocator) maybeProbeWithPadding() {
// use deficient track farthest from desired layer to find how much to probe
for _, track := range s.getMaxDistanceSortedDeficient() {
transition, available := track.GetNextHigherTransition(FlagAllowOvershootInProbe)
for _, track := range s.getMaxDistanceSortedDeficientVideoTracks() {
transition, available := track.GetNextHigherTransition(cFlagAllowOvershootInProbe)
if !available || transition.BandwidthDelta < 0 {
continue
}
// overshoot a bit to account for noise (in measurement/estimate etc)
desiredIncreaseBps := (transition.BandwidthDelta * s.params.Config.ProbeOveragePct) / 100
if desiredIncreaseBps < s.params.Config.ProbeMinBps {
desiredIncreaseBps = s.params.Config.ProbeMinBps
}
desiredIncreaseBps := max((transition.BandwidthDelta*s.params.Config.ProbeOveragePct)/100, s.params.Config.ProbeMinBps)
expectedBandwidthUsage := s.getExpectedBandwidthUsage()
pci := s.prober.AddCluster(
ccutils.ProbeClusterModeUniform,
@@ -1351,68 +1381,65 @@ func (s *StreamAllocator) maybeProbeWithPadding() {
}
}
func (s *StreamAllocator) getTracks() []*Track {
func (s *StreamAllocator) getVideoTracks() []*Track {
s.videoTracksMu.RLock()
tracks := make([]*Track, 0, len(s.videoTracks))
for _, track := range s.videoTracks {
tracks = append(tracks, track)
}
s.videoTracksMu.RUnlock()
defer s.videoTracksMu.RUnlock()
return tracks
return s.videoTracksShadow
}
func (s *StreamAllocator) getSorted() TrackSorter {
s.videoTracksMu.RLock()
func (s *StreamAllocator) getSortedVideoTracks() TrackSorter {
var trackSorter TrackSorter
for _, track := range s.videoTracks {
for _, track := range s.getVideoTracks() {
if !track.IsManaged() {
continue
}
trackSorter = append(trackSorter, track)
}
s.videoTracksMu.RUnlock()
sort.Sort(trackSorter)
return trackSorter
}
func (s *StreamAllocator) getMinDistanceSorted(exclude *Track) MinDistanceSorter {
s.videoTracksMu.RLock()
func (s *StreamAllocator) getMinDistanceSortedVideoTracks(exclude *Track) MinDistanceSorter {
var minDistanceSorter MinDistanceSorter
for _, track := range s.videoTracks {
for _, track := range s.getVideoTracks() {
if !track.IsManaged() || track == exclude {
continue
}
minDistanceSorter = append(minDistanceSorter, track)
}
s.videoTracksMu.RUnlock()
sort.Sort(minDistanceSorter)
return minDistanceSorter
}
func (s *StreamAllocator) getMaxDistanceSortedDeficient() MaxDistanceSorter {
s.videoTracksMu.RLock()
func (s *StreamAllocator) getMaxDistanceSortedDeficientVideoTracks() MaxDistanceSorter {
var maxDistanceSorter MaxDistanceSorter
for _, track := range s.videoTracks {
for _, track := range s.getVideoTracks() {
if !track.IsManaged() || !track.IsDeficient() {
continue
}
maxDistanceSorter = append(maxDistanceSorter, track)
}
s.videoTracksMu.RUnlock()
sort.Sort(maxDistanceSorter)
return maxDistanceSorter
}
func (s *StreamAllocator) shadowVideoTracksLocked() {
s.videoTracksShadow = make([]*Track, 0, len(s.videoTracks))
for _, t := range s.videoTracks {
s.videoTracksShadow = append(s.videoTracksShadow, t)
}
}
// ------------------------------------------------
func updateStreamStateChange(track *Track, allocation sfu.VideoAllocation, update *StreamStateUpdate) {
pkg/sfu/streamallocator/track.go
+16 -16
View File
@@ -23,12 +23,12 @@ import (
)
type Track struct {
downTrack *sfu.DownTrack
source livekit.TrackSource
isSimulcast bool
priority uint8
publisherID livekit.ParticipantID
logger logger.Logger
downTrack *sfu.DownTrack
source livekit.TrackSource
isMultiLayered bool
priority uint8
publisherID livekit.ParticipantID
logger logger.Logger
maxLayer buffer.VideoLayer
@@ -43,17 +43,17 @@ type Track struct {
func NewTrack(
downTrack *sfu.DownTrack,
source livekit.TrackSource,
isSimulcast bool,
isMultiLayered bool,
publisherID livekit.ParticipantID,
logger logger.Logger,
) *Track {
t := &Track{
downTrack: downTrack,
source: source,
isSimulcast: isSimulcast,
publisherID: publisherID,
logger: logger,
streamState: StreamStateInactive,
downTrack: downTrack,
source: source,
isMultiLayered: isMultiLayered,
publisherID: publisherID,
logger: logger,
streamState: StreamStateInactive,
}
t.SetPriority(0)
t.SetMaxLayer(downTrack.MaxLayer())
@@ -87,9 +87,9 @@ func (t *Track) SetPriority(priority uint8) bool {
if priority == 0 {
switch t.source {
case livekit.TrackSource_SCREEN_SHARE:
priority = PriorityDefaultScreenshare
priority = cPriorityDefaultScreenshare
default:
priority = PriorityDefaultVideo
priority = cPriorityDefaultVideo
}
}
@@ -110,7 +110,7 @@ func (t *Track) DownTrack() *sfu.DownTrack {
}
func (t *Track) IsManaged() bool {
return t.source != livekit.TrackSource_SCREEN_SHARE || t.isSimulcast
return t.source != livekit.TrackSource_SCREEN_SHARE || t.isMultiLayered
}
func (t *Track) ID() livekit.TrackID {
pkg/sfu/streamtracker/streamtracker.go
+16 -7
View File
@@ -69,8 +69,8 @@ type StreamTracker struct {
lastNotifiedStatus StreamStatus
lastBitrateReport time.Time
bytesForBitrate [4]int64
bitrate [4]int64
bytesForBitrate [buffer.DefaultMaxLayerTemporal + 1]int64
bitrate [buffer.DefaultMaxLayerTemporal + 1]int64
isStopped bool
}
@@ -158,10 +158,10 @@ func (s *StreamTracker) resetLocked() {
s.setStatusLocked(StreamStatusStopped)
for i := 0; i < len(s.bytesForBitrate); i++ {
for i := range len(s.bytesForBitrate) {
s.bytesForBitrate[i] = 0
}
for i := 0; i < len(s.bitrate); i++ {
for i := range len(s.bitrate) {
s.bitrate[i] = 0
}
@@ -207,8 +207,17 @@ func (s *StreamTracker) Observe(
go s.worker(s.generation.Load())
}
if temporalLayer >= 0 {
if temporalLayer >= 0 && int(temporalLayer) < len(s.bytesForBitrate) {
s.bytesForBitrate[temporalLayer] += int64(pktSize)
} else if int(temporalLayer) >= len(s.bytesForBitrate) {
s.params.Logger.Warnw(
"invalid temporal layer", nil,
"temporalLayer", temporalLayer,
"pktSize", pktSize,
"payloadSize", payloadSize,
"hasMarker", hasMarker,
"ts", ts,
)
}
s.lock.Unlock()
@@ -235,7 +244,7 @@ func (s *StreamTracker) BitrateTemporalCumulative() []int64 {
}
// clear higher layers
for i := 0; i < len(brs); i++ {
for i := range brs {
if brs[i] == 0 {
for j := i + 1; j < len(brs); j++ {
brs[j] = 0
@@ -291,7 +300,7 @@ func (s *StreamTracker) bitrateReport() {
s.lastBitrateReport = now
bitrateAvailabilityChanged := false
for i := 0; i < len(s.bytesForBitrate); i++ {
for i := range len(s.bytesForBitrate) {
bitrate := int64(float64(s.bytesForBitrate[i]*8) / diff.Seconds())
if (s.bitrate[i] == 0 && bitrate > 0) || (s.bitrate[i] > 0 && bitrate == 0) {
bitrateAvailabilityChanged = true
pkg/sfu/streamtracker/streamtracker_dd.go
+6 -6
View File
@@ -107,13 +107,13 @@ func (s *StreamTrackerDependencyDescriptor) resetLocked() {
// bump generation to trigger exit of current worker
s.generation.Inc()
for i := 0; i < len(s.bytesForBitrate); i++ {
for j := 0; j < len(s.bytesForBitrate[i]); j++ {
for i := range len(s.bytesForBitrate) {
for j := range len(s.bytesForBitrate[i]) {
s.bytesForBitrate[i][j] = 0
}
}
for i := 0; i < len(s.bitrate); i++ {
for j := 0; j < len(s.bitrate[i]); j++ {
for i := range len(s.bitrate) {
for j := range len(s.bitrate[i]) {
s.bitrate[i][j] = 0
}
}
@@ -248,11 +248,11 @@ func (s *StreamTrackerDependencyDescriptor) bitrateReport() {
s.lastBitrateReport = now
var availableChangedFns []func()
for spatial := 0; spatial < len(s.bytesForBitrate); spatial++ {
for spatial := range len(s.bytesForBitrate) {
bytesForBitrate := s.bytesForBitrate[spatial][:]
bitrateAvailabilityChanged := false
bitrates := s.bitrate[spatial][:]
for i := 0; i < len(bytesForBitrate); i++ {
for i := range bytesForBitrate {
bitrate := int64(float64(bytesForBitrate[i]*8) / diff.Seconds())
if (bitrates[i] == 0 && bitrate > 0) || (bitrates[i] > 0 && bitrate == 0) {
bitrateAvailabilityChanged = true
pkg/sfu/streamtrackermanager.go
+49 -59
View File
@@ -15,13 +15,13 @@
package sfu
import (
"sort"
"slices"
"sync"
"time"
"github.com/frostbyte73/core"
"go.uber.org/atomic"
"github.com/livekit/protocol/codecs/mime"
"github.com/livekit/protocol/livekit"
"github.com/livekit/protocol/logger"
"github.com/livekit/protocol/utils"
@@ -110,14 +110,15 @@ var (
// ---------------------------------------------------
type StreamTrackerManager struct {
logger logger.Logger
trackInfo atomic.Pointer[livekit.TrackInfo]
isSVC bool
clockRate uint32
logger logger.Logger
mimeType mime.MimeType
videoLayerMode livekit.VideoLayer_Mode
clockRate uint32
trackerConfig StreamTrackerConfig
lock sync.RWMutex
trackInfo *livekit.TrackInfo
maxPublishedLayer int32
maxTemporalLayerSeen int32
@@ -126,7 +127,6 @@ type StreamTrackerManager struct {
availableLayers []int32
maxExpectedLayer int32
paused bool
closed core.Fuse
@@ -136,18 +136,19 @@ type StreamTrackerManager struct {
func NewStreamTrackerManager(
logger logger.Logger,
trackInfo *livekit.TrackInfo,
isSVC bool,
mimeType mime.MimeType,
clockRate uint32,
config StreamTrackerManagerConfig,
) *StreamTrackerManager {
s := &StreamTrackerManager{
logger: logger,
isSVC: isSVC,
mimeType: mimeType,
videoLayerMode: buffer.GetVideoLayerModeForMimeType(mimeType, trackInfo),
trackInfo: utils.CloneProto(trackInfo),
maxPublishedLayer: buffer.InvalidLayerSpatial,
maxTemporalLayerSeen: buffer.InvalidLayerTemporal,
clockRate: clockRate,
}
s.trackInfo.Store(utils.CloneProto(trackInfo))
switch trackInfo.Source {
case livekit.TrackSource_SCREEN_SHARE:
@@ -158,7 +159,8 @@ func NewStreamTrackerManager(
s.trackerConfig = config.Video
}
s.maxExpectedLayerFromTrackInfo()
s.maxExpectedLayer = buffer.InvalidLayerSpatial
s.maxExpectedLayerFromTrackInfo(true)
if trackInfo.Type == livekit.TrackType_VIDEO {
go s.bitrateReporter()
@@ -290,7 +292,7 @@ func (s *StreamTrackerManager) AddTracker(layer int32) streamtracker.StreamTrack
})
s.lock.Lock()
paused := s.paused
paused := s.trackInfo.GetMuted()
s.trackers[layer] = tracker
notify := false
@@ -329,8 +331,10 @@ func (s *StreamTrackerManager) RemoveAllTrackers() {
s.trackers[layer] = nil
}
s.availableLayers = make([]int32, 0)
s.maxExpectedLayerFromTrackInfoLocked()
s.paused = false
s.maxExpectedLayer = buffer.InvalidLayerSpatial
s.maxExpectedLayerFromTrackInfoLocked(true)
ddTracker := s.ddTracker
s.ddTracker = nil
s.lock.Unlock()
@@ -356,9 +360,8 @@ func (s *StreamTrackerManager) GetTracker(layer int32) streamtracker.StreamTrack
return s.trackers[layer]
}
func (s *StreamTrackerManager) SetPaused(paused bool) {
func (s *StreamTrackerManager) setPaused(paused bool) {
s.lock.Lock()
s.paused = paused
trackers := s.trackers
s.lock.Unlock()
@@ -369,16 +372,15 @@ func (s *StreamTrackerManager) SetPaused(paused bool) {
}
}
func (s *StreamTrackerManager) IsPaused() bool {
s.lock.RLock()
defer s.lock.RUnlock()
return s.paused
}
func (s *StreamTrackerManager) UpdateTrackInfo(ti *livekit.TrackInfo) {
s.trackInfo.Store(utils.CloneProto(ti))
s.maxExpectedLayerFromTrackInfo()
s.lock.Lock()
s.trackInfo = utils.CloneProto(ti)
s.maxExpectedLayerFromTrackInfoLocked(false)
paused := s.trackInfo.GetMuted()
s.lock.Unlock()
s.setPaused(paused)
}
func (s *StreamTrackerManager) SetMaxExpectedSpatialLayer(layer int32) int32 {
@@ -417,7 +419,7 @@ func (s *StreamTrackerManager) DistanceToDesired() float64 {
s.lock.RLock()
defer s.lock.RUnlock()
if s.paused || s.maxExpectedLayer < 0 || s.maxTemporalLayerSeen < 0 {
if s.trackInfo.GetMuted() || s.maxExpectedLayer < 0 || s.maxTemporalLayerSeen < 0 {
return 0
}
@@ -480,18 +482,18 @@ func (s *StreamTrackerManager) getLayeredBitrateLocked() ([]int32, Bitrates) {
for i, tracker := range s.trackers {
if tracker != nil {
tls := make([]int64, buffer.DefaultMaxLayerTemporal+1)
if s.hasSpatialLayerLocked(int32(i)) {
if slices.Contains(s.availableLayers, int32(i)) {
tls = tracker.BitrateTemporalCumulative()
}
for j := 0; j < len(br[i]); j++ {
for j := range len(br[i]) {
br[i][j] = tls[j]
}
}
}
// accumulate bitrates for SVC streams without dependency descriptor
if s.isSVC && s.ddTracker == nil {
if s.videoLayerMode == livekit.VideoLayer_MULTIPLE_SPATIAL_LAYERS_PER_STREAM && s.ddTracker == nil {
for i := len(br) - 1; i >= 1; i-- {
for j := len(br[i]) - 1; j >= 0; j-- {
if br[i][j] != 0 {
@@ -509,32 +511,15 @@ func (s *StreamTrackerManager) getLayeredBitrateLocked() ([]int32, Bitrates) {
return availableLayers, br
}
func (s *StreamTrackerManager) hasSpatialLayerLocked(layer int32) bool {
for _, l := range s.availableLayers {
if l == layer {
return true
}
}
return false
}
func (s *StreamTrackerManager) addAvailableLayer(layer int32) {
s.lock.Lock()
hasLayer := false
for _, l := range s.availableLayers {
if l == layer {
hasLayer = true
break
}
}
if hasLayer {
if slices.Contains(s.availableLayers, layer) {
s.lock.Unlock()
return
}
s.availableLayers = append(s.availableLayers, layer)
sort.Slice(s.availableLayers, func(i, j int) bool { return s.availableLayers[i] < s.availableLayers[j] })
slices.Sort(s.availableLayers)
// check if new layer is the max layer
isMaxLayerChange := s.availableLayers[len(s.availableLayers)-1] == layer
@@ -568,7 +553,7 @@ func (s *StreamTrackerManager) removeAvailableLayer(layer int32) {
newLayers = append(newLayers, l)
}
}
sort.Slice(newLayers, func(i, j int) bool { return newLayers[i] < newLayers[j] })
slices.Sort(newLayers)
s.availableLayers = newLayers
s.logger.Debugw(
@@ -594,24 +579,29 @@ func (s *StreamTrackerManager) removeAvailableLayer(layer int32) {
}
}
func (s *StreamTrackerManager) maxExpectedLayerFromTrackInfo() {
func (s *StreamTrackerManager) maxExpectedLayerFromTrackInfo(force bool) {
s.lock.Lock()
defer s.lock.Unlock()
s.maxExpectedLayerFromTrackInfoLocked()
s.maxExpectedLayerFromTrackInfoLocked(force)
}
func (s *StreamTrackerManager) maxExpectedLayerFromTrackInfoLocked() {
s.maxExpectedLayer = buffer.InvalidLayerSpatial
ti := s.trackInfo.Load()
if ti != nil {
for _, layer := range ti.Layers {
spatialLayer := buffer.VideoQualityToSpatialLayer(layer.Quality, ti)
if spatialLayer > s.maxExpectedLayer {
s.maxExpectedLayer = spatialLayer
func (s *StreamTrackerManager) maxExpectedLayerFromTrackInfoLocked(force bool) {
maxExpectedLayer := buffer.InvalidLayerSpatial
if s.trackInfo != nil {
for _, layer := range buffer.GetVideoLayersForMimeType(s.mimeType, s.trackInfo) {
if layer.SpatialLayer > maxExpectedLayer {
maxExpectedLayer = layer.SpatialLayer
}
}
}
// when max expected is higher than current max, trackers are reset
// which allows a layer start to be detected on initial packets from that higher layer,
// so update max only on track info max being lower than current max
if force || maxExpectedLayer < s.maxExpectedLayer {
s.maxExpectedLayer = maxExpectedLayer
}
}
func (s *StreamTrackerManager) GetMaxTemporalLayerSeen() int32 {
pkg/sfu/testutils/data.go
+8 -5
View File
@@ -21,12 +21,13 @@ import (
"github.com/pion/webrtc/v4"
"github.com/livekit/livekit-server/pkg/sfu/buffer"
"github.com/livekit/mediatransportutil/pkg/codec"
)
// -----------------------------------------------------------
type TestExtPacketParams struct {
SetMarker bool
Marker bool
IsKeyFrame bool
PayloadType uint8
SequenceNumber uint16
@@ -38,6 +39,7 @@ type TestExtPacketParams struct {
PaddingSize byte
ArrivalTime time.Time
VideoLayer buffer.VideoLayer
IsOutOfOrder bool
}
// -----------------------------------------------------------
@@ -47,7 +49,7 @@ func GetTestExtPacket(params *TestExtPacketParams) (*buffer.ExtPacket, error) {
Header: rtp.Header{
Version: 2,
Padding: params.PaddingSize != 0,
Marker: params.SetMarker,
Marker: params.Marker,
PayloadType: params.PayloadType,
SequenceNumber: params.SequenceNumber,
Timestamp: params.Timestamp,
@@ -68,8 +70,9 @@ func GetTestExtPacket(params *TestExtPacketParams) (*buffer.ExtPacket, error) {
ExtTimestamp: uint64(params.TSCycles<<32) + uint64(params.Timestamp),
Arrival: params.ArrivalTime.UnixNano(),
Packet: &packet,
KeyFrame: params.IsKeyFrame,
IsKeyFrame: params.IsKeyFrame,
RawPacket: raw,
IsOutOfOrder: params.IsOutOfOrder,
}
return ep, nil
@@ -77,13 +80,13 @@ func GetTestExtPacket(params *TestExtPacketParams) (*buffer.ExtPacket, error) {
// --------------------------------------
func GetTestExtPacketVP8(params *TestExtPacketParams, vp8 *buffer.VP8) (*buffer.ExtPacket, error) {
func GetTestExtPacketVP8(params *TestExtPacketParams, vp8 *codec.VP8) (*buffer.ExtPacket, error) {
ep, err := GetTestExtPacket(params)
if err != nil {
return nil, err
}
ep.KeyFrame = vp8.IsKeyFrame
ep.IsKeyFrame = vp8.IsKeyFrame
ep.Payload = *vp8
if ep.DependencyDescriptor == nil {
ep.Temporal = int32(vp8.TID)
pkg/sfu/track_remote.go
+15
View File
@@ -1,3 +1,17 @@
// Copyright 2023 LiveKit, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package sfu
import "github.com/pion/webrtc/v4"
@@ -7,6 +21,7 @@ type TrackRemote interface {
RID() string
Msid() string
SSRC() webrtc.SSRC
RtxSSRC() webrtc.SSRC
StreamID() string
Kind() webrtc.RTPCodecType
Codec() webrtc.RTPCodecParameters
pkg/sfu/downtrackspreader.go → pkg/sfu/utils/downtrackspreader.go
+23 -20
View File
@@ -12,7 +12,7 @@
// See the License for the specific language governing permissions and
// limitations under the License.
package sfu
package utils
import (
"sync"
@@ -22,55 +22,59 @@ import (
"github.com/livekit/protocol/utils"
)
type sender interface {
SubscriberID() livekit.ParticipantID
}
type DownTrackSpreaderParams struct {
Threshold int
Logger logger.Logger
}
type DownTrackSpreader struct {
type DownTrackSpreader[T sender] struct {
params DownTrackSpreaderParams
downTrackMu sync.RWMutex
downTracks map[livekit.ParticipantID]TrackSender
downTracksShadow []TrackSender
downTracks map[livekit.ParticipantID]T
downTracksShadow []T
}
func NewDownTrackSpreader(params DownTrackSpreaderParams) *DownTrackSpreader {
d := &DownTrackSpreader{
func NewDownTrackSpreader[T sender](params DownTrackSpreaderParams) *DownTrackSpreader[T] {
d := &DownTrackSpreader[T]{
params: params,
downTracks: make(map[livekit.ParticipantID]TrackSender),
downTracks: make(map[livekit.ParticipantID]T),
}
return d
}
func (d *DownTrackSpreader) GetDownTracks() []TrackSender {
func (d *DownTrackSpreader[T]) GetDownTracks() []T {
d.downTrackMu.RLock()
defer d.downTrackMu.RUnlock()
return d.downTracksShadow
}
func (d *DownTrackSpreader) ResetAndGetDownTracks() []TrackSender {
func (d *DownTrackSpreader[T]) ResetAndGetDownTracks() []T {
d.downTrackMu.Lock()
defer d.downTrackMu.Unlock()
downTracks := d.downTracksShadow
d.downTracks = make(map[livekit.ParticipantID]TrackSender)
d.downTracks = make(map[livekit.ParticipantID]T)
d.downTracksShadow = nil
return downTracks
}
func (d *DownTrackSpreader) Store(ts TrackSender) {
func (d *DownTrackSpreader[T]) Store(sender T) {
d.downTrackMu.Lock()
defer d.downTrackMu.Unlock()
d.downTracks[ts.SubscriberID()] = ts
d.downTracks[sender.SubscriberID()] = sender
d.shadowDownTracks()
}
func (d *DownTrackSpreader) Free(subscriberID livekit.ParticipantID) {
func (d *DownTrackSpreader[T]) Free(subscriberID livekit.ParticipantID) {
d.downTrackMu.Lock()
defer d.downTrackMu.Unlock()
@@ -78,7 +82,7 @@ func (d *DownTrackSpreader) Free(subscriberID livekit.ParticipantID) {
d.shadowDownTracks()
}
func (d *DownTrackSpreader) HasDownTrack(subscriberID livekit.ParticipantID) bool {
func (d *DownTrackSpreader[T]) HasDownTrack(subscriberID livekit.ParticipantID) bool {
d.downTrackMu.RLock()
defer d.downTrackMu.RUnlock()
@@ -86,10 +90,10 @@ func (d *DownTrackSpreader) HasDownTrack(subscriberID livekit.ParticipantID) boo
return ok
}
func (d *DownTrackSpreader) Broadcast(writer func(TrackSender)) int {
func (d *DownTrackSpreader[T]) Broadcast(writer func(T)) {
downTracks := d.GetDownTracks()
if len(downTracks) == 0 {
return 0
return
}
threshold := uint64(d.params.Threshold)
@@ -101,17 +105,16 @@ func (d *DownTrackSpreader) Broadcast(writer func(TrackSender)) int {
// WriteRTP takes about 50µs on average, so we write to 2 down tracks per loop.
step := uint64(2)
utils.ParallelExec(downTracks, threshold, step, writer)
return len(downTracks)
}
func (d *DownTrackSpreader) DownTrackCount() int {
func (d *DownTrackSpreader[T]) DownTrackCount() int {
d.downTrackMu.RLock()
defer d.downTrackMu.RUnlock()
return len(d.downTracksShadow)
}
func (d *DownTrackSpreader) shadowDownTracks() {
d.downTracksShadow = make([]TrackSender, 0, len(d.downTracks))
func (d *DownTrackSpreader[T]) shadowDownTracks() {
d.downTracksShadow = make([]T, 0, len(d.downTracks))
for _, dt := range d.downTracks {
d.downTracksShadow = append(d.downTracksShadow, dt)
}
pkg/sfu/utils/helpers.go
+7 -1
View File
@@ -18,10 +18,12 @@ import (
"errors"
"fmt"
"github.com/livekit/livekit-server/pkg/sfu/mime"
"github.com/pion/interceptor"
"github.com/pion/rtp"
"github.com/pion/webrtc/v4"
"github.com/livekit/protocol/codecs/mime"
"github.com/livekit/protocol/livekit"
)
// Do a fuzzy find for a codec in the list of codecs
@@ -89,3 +91,7 @@ func ValidateRTPPacket(pkt *rtp.Packet, expectedPayloadType uint8, expectedSSRC
return nil
}
func IsSimulcastMode(m livekit.VideoLayer_Mode) bool {
return m == livekit.VideoLayer_ONE_SPATIAL_LAYER_PER_STREAM || m == livekit.VideoLayer_ONE_SPATIAL_LAYER_PER_STREAM_INCOMPLETE_RTCP_SR
}
pkg/sfu/utils/owd_estimator.go
-179
View File
@@ -1,179 +0,0 @@
// Copyright 2023 LiveKit, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package utils
import (
"time"
"go.uber.org/zap/zapcore"
)
type OWDEstimatorParams struct {
PropagationDelayFallFactor float64
PropagationDelayRiseFactor float64
PropagationDelaySpikeAdaptationFactor float64
PropagationDelayDeltaThresholdMin time.Duration
PropagationDelayDeltaThresholdMaxFactor int64
PropagationDelayDeltaHighResetNumReports int
PropagationDelayDeltaHighResetWait time.Duration
PropagationDelayDeltaLongTermAdaptationThreshold time.Duration
}
var OWDEstimatorParamsDefault = OWDEstimatorParams{
// OWD (One-Way-Delay) Estimator is used to estimate propagation delay between sender and receicer.
// As they operate on different clock domains, it is not possible to get exact propagation delay easily.
// So, this module is an estimator using a simple approach explained below. It should not be used for
// things that require high accuracy.
//
// One example is RTCP Sender Reports getting re-based to SFU time base so that all subscriber side
// can have the same time base (i. e. SFU time base). To convert publisher side
// RTCP Sender Reports to SFU timebase, a propagation delay is maintained.
// propagation_delay = time_of_report_reception - ntp_timestamp_in_report
//
// Propagation delay is adapted continuously. If it falls, adapt quickly to the
// lower value as that could be the real propagation delay. If it rises, adapt slowly
// as it might be a temporary change or slow drift. See below for handling of high deltas
// which could be a result of a path change.
PropagationDelayFallFactor: 0.9,
PropagationDelayRiseFactor: 0.1,
PropagationDelaySpikeAdaptationFactor: 0.5,
// To account for path changes mid-stream, if the delta of the propagation delay is consistently higher, reset.
// Reset at whichever of the below happens later.
// 1. 10 seconds of persistent high delta.
// 2. at least 2 consecutive reports with high delta.
//
// A long term estimate of delta of propagation delay is maintained and delta propagation delay exceeding
// a factor of the long term estimate is considered a sharp increase. That will trigger the start of the
// path change condition and if it persists, propagation delay will be reset.
PropagationDelayDeltaThresholdMin: 10 * time.Millisecond,
PropagationDelayDeltaThresholdMaxFactor: 2,
PropagationDelayDeltaHighResetNumReports: 2,
PropagationDelayDeltaHighResetWait: 10 * time.Second,
PropagationDelayDeltaLongTermAdaptationThreshold: 50 * time.Millisecond,
}
type OWDEstimator struct {
params OWDEstimatorParams
initialized bool
lastSenderClockTimeNs int64
lastPropagationDelayNs int64
lastDeltaPropagationDelayNs int64
estimatedPropagationDelayNs int64
longTermDeltaPropagationDelayNs int64
propagationDelayDeltaHighCount int
propagationDelayDeltaHighStartTime time.Time
propagationDelaySpikeNs int64
}
func NewOWDEstimator(params OWDEstimatorParams) *OWDEstimator {
return &OWDEstimator{
params: params,
}
}
func (o *OWDEstimator) MarshalLogObject(e zapcore.ObjectEncoder) error {
if o != nil {
e.AddTime("lastSenderClockTimeNs", time.Unix(0, o.lastSenderClockTimeNs))
e.AddDuration("lastPropagationDelayNs", time.Duration(o.lastPropagationDelayNs))
e.AddDuration("lastDeltaPropagationDelayNs", time.Duration(o.lastDeltaPropagationDelayNs))
e.AddDuration("estimatedPropagationDelayNs", time.Duration(o.estimatedPropagationDelayNs))
e.AddDuration("longTermDeltaPropagationDelayNs", time.Duration(o.longTermDeltaPropagationDelayNs))
e.AddInt("propagationDelayDeltaHighCount", o.propagationDelayDeltaHighCount)
e.AddTime("propagationDelayDeltaHighStartTime", o.propagationDelayDeltaHighStartTime)
e.AddDuration("propagationDelaySpikeNs", time.Duration(o.propagationDelaySpikeNs))
}
return nil
}
func (o *OWDEstimator) Update(senderClockTimeNs int64, receiverClockTimeNs int64) (int64, bool) {
resetDelta := func() {
o.propagationDelayDeltaHighCount = 0
o.propagationDelayDeltaHighStartTime = time.Time{}
o.propagationDelaySpikeNs = 0
}
initPropagationDelay := func(pd int64) {
o.estimatedPropagationDelayNs = pd
o.longTermDeltaPropagationDelayNs = 0
resetDelta()
}
o.lastPropagationDelayNs = receiverClockTimeNs - senderClockTimeNs
if !o.initialized {
o.initialized = true
o.lastSenderClockTimeNs = senderClockTimeNs
initPropagationDelay(o.lastPropagationDelayNs)
return o.estimatedPropagationDelayNs, true
}
stepChange := false
o.lastDeltaPropagationDelayNs = o.lastPropagationDelayNs - o.estimatedPropagationDelayNs
// check for path changes, i. e. a step jump increase in propagation delay observed over time
if o.lastDeltaPropagationDelayNs > o.params.PropagationDelayDeltaThresholdMin.Nanoseconds() { // ignore small changes for path change consideration
if o.longTermDeltaPropagationDelayNs != 0 &&
o.lastDeltaPropagationDelayNs > o.longTermDeltaPropagationDelayNs*o.params.PropagationDelayDeltaThresholdMaxFactor {
o.propagationDelayDeltaHighCount++
if o.propagationDelayDeltaHighStartTime.IsZero() {
o.propagationDelayDeltaHighStartTime = time.Now()
}
if o.propagationDelaySpikeNs == 0 {
o.propagationDelaySpikeNs = o.lastPropagationDelayNs
} else {
o.propagationDelaySpikeNs += int64(o.params.PropagationDelaySpikeAdaptationFactor * float64(o.lastPropagationDelayNs-o.propagationDelaySpikeNs))
}
if o.propagationDelayDeltaHighCount >= o.params.PropagationDelayDeltaHighResetNumReports && time.Since(o.propagationDelayDeltaHighStartTime) >= o.params.PropagationDelayDeltaHighResetWait {
stepChange = true
initPropagationDelay(o.propagationDelaySpikeNs)
}
} else {
resetDelta()
}
} else {
resetDelta()
factor := o.params.PropagationDelayFallFactor
if o.lastPropagationDelayNs > o.estimatedPropagationDelayNs {
factor = o.params.PropagationDelayRiseFactor
}
o.estimatedPropagationDelayNs += int64(factor * float64(o.lastPropagationDelayNs-o.estimatedPropagationDelayNs))
}
if o.lastDeltaPropagationDelayNs < o.params.PropagationDelayDeltaLongTermAdaptationThreshold.Nanoseconds() {
if o.longTermDeltaPropagationDelayNs == 0 {
o.longTermDeltaPropagationDelayNs = o.lastDeltaPropagationDelayNs
} else {
// do not adapt to large +ve spikes, can happen when channel is congested and reports are delivered very late
// if the spike is in fact a path change, it will persist and handled by path change detection above
sinceLast := senderClockTimeNs - o.lastSenderClockTimeNs
adaptationFactor := min(1.0, float64(sinceLast)/float64(o.params.PropagationDelayDeltaHighResetWait))
o.longTermDeltaPropagationDelayNs += int64(adaptationFactor * float64(o.lastDeltaPropagationDelayNs-o.longTermDeltaPropagationDelayNs))
}
}
if o.longTermDeltaPropagationDelayNs < 0 {
o.longTermDeltaPropagationDelayNs = 0
}
o.lastSenderClockTimeNs = senderClockTimeNs
return o.estimatedPropagationDelayNs, stepChange
}
func (o *OWDEstimator) EstimatedPropagationDelay() int64 {
return o.estimatedPropagationDelayNs
}
pkg/sfu/utils/rangemap.go
+1 -4
View File
@@ -80,10 +80,7 @@ func (r *RangeMap[RT, VT]) MarshalLogObject(e zapcore.ObjectEncoder) error {
e.AddInt("numRanges", len(r.ranges))
// just the last 10 ranges max
startIdx := len(r.ranges) - 10
if startIdx < 0 {
startIdx = 0
}
startIdx := max(len(r.ranges)-10, 0)
for i := startIdx; i < len(r.ranges); i++ {
e.AddObject(fmt.Sprintf("range[%d]", i), r.ranges[i])
}
pkg/sfu/utils/rangemap_test.go
+10 -10
View File
@@ -66,7 +66,7 @@ func TestRangeMapUint32(t *testing.T) {
require.Equal(t, uint32(1), value)
// excluded range should return error
value, err = r.GetValue(10)
_, err = r.GetValue(10)
require.ErrorIs(t, err, errKeyExcluded)
// out-of-order exclusion should return error
@@ -98,11 +98,11 @@ func TestRangeMapUint32(t *testing.T) {
require.Equal(t, expectedRanges, r.ranges)
// excluded range should return error, now is excluded because exclusion range could be extended
value, err = r.GetValue(11)
_, err = r.GetValue(11)
require.ErrorIs(t, err, errKeyExcluded)
// getting value in old range should return 0
value, err = r.GetValue(6)
_, err = r.GetValue(6)
require.NoError(t, err)
// newer should return 2
@@ -129,7 +129,7 @@ func TestRangeMapUint32(t *testing.T) {
require.Equal(t, expectedRanges, r.ranges)
// excluded range should return error, now is excluded because exclusion range could be extended
value, err = r.GetValue(15)
_, err = r.GetValue(15)
require.ErrorIs(t, err, errKeyExcluded)
// newer should return 12
@@ -190,15 +190,15 @@ func TestRangeMapUint32(t *testing.T) {
require.Equal(t, expectedRanges, r.ranges)
// excluded range should return error
value, err = r.GetValue(50)
_, err = r.GetValue(50)
require.ErrorIs(t, err, errKeyExcluded)
value, err = r.GetValue(28)
_, err = r.GetValue(28)
require.ErrorIs(t, err, errKeyExcluded)
value, err = r.GetValue(17)
_, err = r.GetValue(17)
require.ErrorIs(t, err, errKeyTooOld)
// previously valid, but aged out key should return error
value, err = r.GetValue(5)
_, err = r.GetValue(5)
require.ErrorIs(t, err, errKeyTooOld)
// valid range access should return values
@@ -279,7 +279,7 @@ func TestRangeMapUint32(t *testing.T) {
require.Equal(t, expectedRanges, r.ranges)
// before first range access
value, err = r.GetValue(5)
_, err = r.GetValue(5)
require.ErrorIs(t, err, errKeyTooOld)
// first range access
@@ -319,7 +319,7 @@ func TestRangeMapUint32(t *testing.T) {
require.Equal(t, expectedRanges, r.ranges)
// aged out range access
value, err = r.GetValue(25)
_, err = r.GetValue(25)
require.ErrorIs(t, err, errKeyTooOld)
// access closed range before decrementing value
pkg/sfu/utils/wraparound.go
-224
View File
@@ -1,224 +0,0 @@
// Copyright 2023 LiveKit, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package utils
import (
"unsafe"
"go.uber.org/zap/zapcore"
)
type number interface {
uint16 | uint32
}
type extendedNumber interface {
uint32 | uint64
}
type WrapAroundParams struct {
IsRestartAllowed bool
}
type WrapAround[T number, ET extendedNumber] struct {
params WrapAroundParams
fullRange ET
initialized bool
start T
highest T
cycles ET
extendedHighest ET
}
func NewWrapAround[T number, ET extendedNumber](params WrapAroundParams) *WrapAround[T, ET] {
var t T
return &WrapAround[T, ET]{
params: params,
fullRange: 1 << (unsafe.Sizeof(t) * 8),
}
}
func (w *WrapAround[T, ET]) Seed(from *WrapAround[T, ET]) {
w.initialized = from.initialized
w.start = from.start
w.highest = from.highest
w.cycles = from.cycles
w.updateExtendedHighest()
}
type WrapAroundUpdateResult[ET extendedNumber] struct {
IsUnhandled bool // when set, other fields are invalid
IsRestart bool
PreExtendedStart ET // valid only if IsRestart = true
PreExtendedHighest ET
ExtendedVal ET
}
func (w *WrapAroundUpdateResult[ET]) MarshalLogObject(e zapcore.ObjectEncoder) error {
if w == nil {
return nil
}
e.AddBool("IsUnhandled", w.IsUnhandled)
e.AddBool("IsRestart", w.IsRestart)
e.AddUint64("PreExtendedStart", uint64(w.PreExtendedStart))
e.AddUint64("PreExtendedHighest", uint64(w.PreExtendedHighest))
e.AddUint64("ExtendedVal", uint64(w.ExtendedVal))
return nil
}
func (w *WrapAround[T, ET]) Update(val T) (result WrapAroundUpdateResult[ET]) {
if !w.initialized {
result.PreExtendedHighest = ET(val) - 1
result.ExtendedVal = ET(val)
w.start = val
w.highest = val
w.updateExtendedHighest()
w.initialized = true
return
}
gap := val - w.highest
if gap > T(w.fullRange>>1) {
// out-of-order
return w.maybeAdjustStart(val)
}
// in-order
result.PreExtendedHighest = w.extendedHighest
if val < w.highest {
w.cycles += w.fullRange
}
w.highest = val
w.updateExtendedHighest()
result.ExtendedVal = w.extendedHighest
return
}
func (w *WrapAround[T, ET]) UndoUpdate(result WrapAroundUpdateResult[ET]) {
if !w.initialized || result.PreExtendedHighest >= result.ExtendedVal {
return
}
w.ResetHighest(result.PreExtendedHighest)
}
func (w *WrapAround[T, ET]) Rollover(val T, numCycles int) (result WrapAroundUpdateResult[ET]) {
if numCycles < 0 || !w.initialized {
return w.Update(val)
}
result.PreExtendedHighest = w.extendedHighest
w.cycles += ET(numCycles) * w.fullRange
w.highest = val
w.updateExtendedHighest()
result.ExtendedVal = w.extendedHighest
return
}
func (w *WrapAround[T, ET]) RollbackRestart(ev ET) {
if w.isWrapBack(w.start, T(ev)) {
w.cycles -= w.fullRange
w.updateExtendedHighest()
}
w.start = T(ev)
}
func (w *WrapAround[T, ET]) ResetHighest(ev ET) {
w.highest = T(ev)
w.cycles = ev & ^(w.fullRange - 1)
w.updateExtendedHighest()
}
func (w *WrapAround[T, ET]) GetStart() T {
return w.start
}
func (w *WrapAround[T, ET]) GetExtendedStart() ET {
return ET(w.start)
}
func (w *WrapAround[T, ET]) GetHighest() T {
return w.highest
}
func (w *WrapAround[T, ET]) GetExtendedHighest() ET {
return w.extendedHighest
}
func (w *WrapAround[T, ET]) updateExtendedHighest() {
w.extendedHighest = getExtended(w.cycles, w.highest)
}
func (w *WrapAround[T, ET]) maybeAdjustStart(val T) (result WrapAroundUpdateResult[ET]) {
// re-adjust start if necessary. The conditions are
// 1. Not seen more than half the range yet
// 1. wrap back compared to start and not completed a half cycle, sequences like (10, 65530) in uint16 space
// 2. no wrap around, but out-of-order compared to start and not completed a half cycle , sequences like (10, 9), (65530, 65528) in uint16 space
cycles := w.cycles
totalNum := w.GetExtendedHighest() - w.GetExtendedStart() + 1
if totalNum > (w.fullRange >> 1) {
if w.isWrapBack(val, w.highest) {
cycles -= w.fullRange
}
result.PreExtendedHighest = w.extendedHighest
result.ExtendedVal = getExtended(cycles, val)
return
}
if val-w.start > T(w.fullRange>>1) {
if w.params.IsRestartAllowed {
// out-of-order with existing start => a new start
result.IsRestart = true
if val > w.start {
result.PreExtendedStart = w.fullRange + ET(w.start)
} else {
result.PreExtendedStart = ET(w.start)
}
if w.isWrapBack(val, w.highest) {
w.cycles = w.fullRange
w.updateExtendedHighest()
cycles = 0
}
w.start = val
} else {
result.IsUnhandled = true
}
} else {
if w.isWrapBack(val, w.highest) {
cycles -= w.fullRange
}
}
result.PreExtendedHighest = w.extendedHighest
result.ExtendedVal = getExtended(cycles, val)
return
}
func (w *WrapAround[T, ET]) isWrapBack(earlier T, later T) bool {
return ET(later) < (w.fullRange>>1) && ET(earlier) >= (w.fullRange>>1)
}
// ------------------------------------
func getExtended[T number, ET extendedNumber](cycles ET, val T) ET {
return cycles + ET(val)
}
pkg/sfu/utils/wraparound_test.go
-679
View File
@@ -1,679 +0,0 @@
// Copyright 2023 LiveKit, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package utils
import (
"testing"
"github.com/stretchr/testify/require"
)
func TestWrapAroundUint16(t *testing.T) {
w := NewWrapAround[uint16, uint32](WrapAroundParams{IsRestartAllowed: true})
testCases := []struct {
name string
input uint16
updated WrapAroundUpdateResult[uint32]
start uint16
extendedStart uint32
highest uint16
extendedHighest uint32
}{
// initialize
{
name: "initialize",
input: 10,
updated: WrapAroundUpdateResult[uint32]{
IsRestart: false,
PreExtendedStart: 0,
PreExtendedHighest: 9,
ExtendedVal: 10,
},
start: 10,
extendedStart: 10,
highest: 10,
extendedHighest: 10,
},
// an older number without wrap around should reset start point
{
name: "reset start no wrap around",
input: 8,
updated: WrapAroundUpdateResult[uint32]{
IsRestart: true,
PreExtendedStart: 10,
PreExtendedHighest: 10,
ExtendedVal: 8,
},
start: 8,
extendedStart: 8,
highest: 10,
extendedHighest: 10,
},
// an older number with wrap around should reset start point
{
name: "reset start wrap around",
input: (1 << 16) - 6,
updated: WrapAroundUpdateResult[uint32]{
IsRestart: true,
PreExtendedStart: (1 << 16) + 8,
PreExtendedHighest: (1 << 16) + 10,
ExtendedVal: (1 << 16) - 6,
},
start: (1 << 16) - 6,
extendedStart: (1 << 16) - 6,
highest: 10,
extendedHighest: (1 << 16) + 10,
},
// an older number with wrap around should reset start point again
{
name: "reset start again",
input: (1 << 16) - 12,
updated: WrapAroundUpdateResult[uint32]{
IsRestart: true,
PreExtendedStart: (1 << 16) - 6,
PreExtendedHighest: (1 << 16) + 10,
ExtendedVal: (1 << 16) - 12,
},
start: (1 << 16) - 12,
extendedStart: (1 << 16) - 12,
highest: 10,
extendedHighest: (1 << 16) + 10,
},
// out of order with highest, wrap back, but no restart
{
name: "out of order - no restart",
input: (1 << 16) - 3,
updated: WrapAroundUpdateResult[uint32]{
IsRestart: false,
PreExtendedStart: 0,
PreExtendedHighest: (1 << 16) + 10,
ExtendedVal: (1 << 16) - 3,
},
start: (1 << 16) - 12,
extendedStart: (1 << 16) - 12,
highest: 10,
extendedHighest: (1 << 16) + 10,
},
// duplicate should return same as highest
{
name: "duplicate",
input: 10,
updated: WrapAroundUpdateResult[uint32]{
IsRestart: false,
PreExtendedStart: 0,
PreExtendedHighest: (1 << 16) + 10,
ExtendedVal: (1 << 16) + 10,
},
start: (1 << 16) - 12,
extendedStart: (1 << 16) - 12,
highest: 10,
extendedHighest: (1 << 16) + 10,
},
// a significant jump in order should not reset start
{
name: "big in-order jump",
input: (1 << 15) - 10,
updated: WrapAroundUpdateResult[uint32]{
IsRestart: false,
PreExtendedStart: 0,
PreExtendedHighest: (1 << 16) + 10,
ExtendedVal: (1 << 16) + (1 << 15) - 10,
},
start: (1 << 16) - 12,
extendedStart: (1 << 16) - 12,
highest: (1 << 15) - 10,
extendedHighest: (1 << 16) + (1 << 15) - 10,
},
// now out-of-order should not reset start as half the range has been seen
{
name: "out-of-order after half range",
input: (1 << 15) - 11,
updated: WrapAroundUpdateResult[uint32]{
IsRestart: false,
PreExtendedStart: 0,
PreExtendedHighest: (1 << 16) + (1 << 15) - 10,
ExtendedVal: (1 << 16) + (1 << 15) - 11,
},
start: (1 << 16) - 12,
extendedStart: (1 << 16) - 12,
highest: (1 << 15) - 10,
extendedHighest: (1 << 16) + (1 << 15) - 10,
},
// wrap back out-of-order
{
name: "wrap back out-of-order after half range",
input: (1 << 16) - 1,
updated: WrapAroundUpdateResult[uint32]{
IsRestart: false,
PreExtendedStart: 0,
PreExtendedHighest: (1 << 16) + (1 << 15) - 10,
ExtendedVal: (1 << 16) - 1,
},
start: (1 << 16) - 12,
extendedStart: (1 << 16) - 12,
highest: (1 << 15) - 10,
extendedHighest: (1 << 16) + (1 << 15) - 10,
},
// in-order, should update highest
{
name: "in-order",
input: (1 << 15) + 3,
updated: WrapAroundUpdateResult[uint32]{
IsRestart: false,
PreExtendedStart: 0,
PreExtendedHighest: (1 << 16) + (1 << 15) - 10,
ExtendedVal: (1 << 16) + (1 << 15) + 3,
},
start: (1 << 16) - 12,
extendedStart: (1 << 16) - 12,
highest: (1 << 15) + 3,
extendedHighest: (1 << 16) + (1 << 15) + 3,
},
}
for _, tc := range testCases {
t.Run(tc.name, func(t *testing.T) {
require.Equal(t, tc.updated, w.Update(tc.input))
require.Equal(t, tc.start, w.GetStart())
require.Equal(t, tc.extendedStart, w.GetExtendedStart())
require.Equal(t, tc.highest, w.GetHighest())
require.Equal(t, tc.extendedHighest, w.GetExtendedHighest())
})
}
}
func TestWrapAroundUint16NoRestart(t *testing.T) {
w := NewWrapAround[uint16, uint32](WrapAroundParams{IsRestartAllowed: false})
testCases := []struct {
name string
input uint16
updated WrapAroundUpdateResult[uint32]
start uint16
extendedStart uint32
highest uint16
extendedHighest uint32
}{
// initialize
{
name: "initialize",
input: 10,
updated: WrapAroundUpdateResult[uint32]{
IsRestart: false,
PreExtendedStart: 0,
PreExtendedHighest: 9,
ExtendedVal: 10,
},
start: 10,
extendedStart: 10,
highest: 10,
extendedHighest: 10,
},
// an older number without wrap around should not reset start point
{
name: "no reset start no wrap around",
input: 8,
updated: WrapAroundUpdateResult[uint32]{
IsUnhandled: true,
// the following fields are not valid when `IsUnhandled = true`, but code fills it in
// and they are filled in here for testing purposes
PreExtendedHighest: 10,
ExtendedVal: 8,
},
start: 10,
extendedStart: 10,
highest: 10,
extendedHighest: 10,
},
// an older number with wrap around should not reset start point
{
name: "no reset start wrap around",
input: (1 << 16) - 6,
updated: WrapAroundUpdateResult[uint32]{
IsUnhandled: true,
PreExtendedHighest: 10,
ExtendedVal: (1 << 16) - 6,
},
start: 10,
extendedStart: 10,
highest: 10,
extendedHighest: 10,
},
// yet another older number with wrap around should not reset start point
{
name: "no reset start again",
input: (1 << 16) - 12,
updated: WrapAroundUpdateResult[uint32]{
IsUnhandled: true,
PreExtendedHighest: 10,
ExtendedVal: (1 << 16) - 12,
},
start: 10,
extendedStart: 10,
highest: 10,
extendedHighest: 10,
},
// duplicate should return same as highest
{
name: "duplicate",
input: 10,
updated: WrapAroundUpdateResult[uint32]{
PreExtendedHighest: 10,
ExtendedVal: 10,
},
start: 10,
extendedStart: 10,
highest: 10,
extendedHighest: 10,
},
// a significant jump in order should move highest to that
{
name: "big in-order jump",
input: (1 << 15) - 10,
updated: WrapAroundUpdateResult[uint32]{
PreExtendedHighest: 10,
ExtendedVal: (1 << 15) - 10,
},
start: 10,
extendedStart: 10,
highest: (1 << 15) - 10,
extendedHighest: (1 << 15) - 10,
},
// in-order, should update highest
{
name: "in-order",
input: (1 << 15) + 13,
updated: WrapAroundUpdateResult[uint32]{
PreExtendedHighest: (1 << 15) - 10,
ExtendedVal: (1 << 15) + 13,
},
start: 10,
extendedStart: 10,
highest: (1 << 15) + 13,
extendedHighest: (1 << 15) + 13,
},
// now out-of-order should not reset start as half the range has been seen
{
name: "out-of-order after half range",
input: (1 << 15) - 11,
updated: WrapAroundUpdateResult[uint32]{
PreExtendedHighest: (1 << 15) + 13,
ExtendedVal: (1 << 15) - 11,
},
start: 10,
extendedStart: 10,
highest: (1 << 15) + 13,
extendedHighest: (1 << 15) + 13,
},
}
for _, tc := range testCases {
t.Run(tc.name, func(t *testing.T) {
require.Equal(t, tc.updated, w.Update(tc.input))
require.Equal(t, tc.start, w.GetStart())
require.Equal(t, tc.extendedStart, w.GetExtendedStart())
require.Equal(t, tc.highest, w.GetHighest())
require.Equal(t, tc.extendedHighest, w.GetExtendedHighest())
})
}
}
func TestWrapAroundUint16RollbackRestartAndResetHighest(t *testing.T) {
w := NewWrapAround[uint16, uint64](WrapAroundParams{IsRestartAllowed: true})
// initialize
w.Update(23)
require.Equal(t, uint16(23), w.GetStart())
require.Equal(t, uint64(23), w.GetExtendedStart())
require.Equal(t, uint16(23), w.GetHighest())
require.Equal(t, uint64(23), w.GetExtendedHighest())
// an in-order update
w.Update(25)
require.Equal(t, uint16(23), w.GetStart())
require.Equal(t, uint64(23), w.GetExtendedStart())
require.Equal(t, uint16(25), w.GetHighest())
require.Equal(t, uint64(25), w.GetExtendedHighest())
// force restart without wrap
res := w.Update(12)
expectedResult := WrapAroundUpdateResult[uint64]{
IsRestart: true,
PreExtendedStart: 23,
PreExtendedHighest: 25,
ExtendedVal: 12,
}
require.Equal(t, expectedResult, res)
require.Equal(t, uint16(12), w.GetStart())
require.Equal(t, uint64(12), w.GetExtendedStart())
require.Equal(t, uint16(25), w.GetHighest())
require.Equal(t, uint64(25), w.GetExtendedHighest())
// roll back restart
w.RollbackRestart(res.PreExtendedStart)
require.Equal(t, uint16(23), w.GetStart())
require.Equal(t, uint64(23), w.GetExtendedStart())
require.Equal(t, uint16(25), w.GetHighest())
require.Equal(t, uint64(25), w.GetExtendedHighest())
// force restart with wrap
res = w.Update(65533)
expectedResult = WrapAroundUpdateResult[uint64]{
IsRestart: true,
PreExtendedStart: (1 << 16) + 23,
PreExtendedHighest: (1 << 16) + 25,
ExtendedVal: 65533,
}
require.Equal(t, expectedResult, res)
require.Equal(t, uint16(65533), w.GetStart())
require.Equal(t, uint64(65533), w.GetExtendedStart())
require.Equal(t, uint16(25), w.GetHighest())
require.Equal(t, uint64(65536+25), w.GetExtendedHighest())
// roll back restart
w.RollbackRestart(res.PreExtendedStart)
require.Equal(t, uint16(23), w.GetStart())
require.Equal(t, uint64(23), w.GetExtendedStart())
require.Equal(t, uint16(25), w.GetHighest())
require.Equal(t, uint64(25), w.GetExtendedHighest())
// reset highest
w.ResetHighest(0x1234)
require.Equal(t, uint16(23), w.GetStart())
require.Equal(t, uint64(23), w.GetExtendedStart())
require.Equal(t, uint16(0x1234), w.GetHighest())
require.Equal(t, uint64(0x1234), w.GetExtendedHighest())
w.ResetHighest(0x7f1234)
require.Equal(t, uint16(23), w.GetStart())
require.Equal(t, uint64(23), w.GetExtendedStart())
require.Equal(t, uint16(0x1234), w.GetHighest())
require.Equal(t, uint64(0x7f1234), w.GetExtendedHighest())
}
func TestWrapAroundUint16WrapAroundRestartDuplicate(t *testing.T) {
w := NewWrapAround[uint16, uint64](WrapAroundParams{IsRestartAllowed: true})
// initialize
w.Update(65534)
require.Equal(t, uint16(65534), w.GetStart())
require.Equal(t, uint64(65534), w.GetExtendedStart())
require.Equal(t, uint16(65534), w.GetHighest())
require.Equal(t, uint64(65534), w.GetExtendedHighest())
// an in-order update with a roll over
w.Update(32)
require.Equal(t, uint16(65534), w.GetStart())
require.Equal(t, uint64(65534), w.GetExtendedStart())
require.Equal(t, uint16(32), w.GetHighest())
require.Equal(t, uint64(65568), w.GetExtendedHighest())
// duplicate of start
res := w.Update(65534)
expectedResult := WrapAroundUpdateResult[uint64]{
IsRestart: false,
PreExtendedStart: 0,
PreExtendedHighest: 65568,
ExtendedVal: 65534,
}
require.Equal(t, expectedResult, res)
require.Equal(t, uint16(65534), w.GetStart())
require.Equal(t, uint64(65534), w.GetExtendedStart())
require.Equal(t, uint16(32), w.GetHighest())
require.Equal(t, uint64(65568), w.GetExtendedHighest())
// duplicate of start - again
res = w.Update(65534)
expectedResult = WrapAroundUpdateResult[uint64]{
IsRestart: false,
PreExtendedStart: 0,
PreExtendedHighest: 65568,
ExtendedVal: 65534,
}
require.Equal(t, expectedResult, res)
require.Equal(t, uint16(65534), w.GetStart())
require.Equal(t, uint64(65534), w.GetExtendedStart())
require.Equal(t, uint16(32), w.GetHighest())
require.Equal(t, uint64(65568), w.GetExtendedHighest())
}
func TestWrapAroundUint16Rollover(t *testing.T) {
w := NewWrapAround[uint16, uint32](WrapAroundParams{IsRestartAllowed: false})
testCases := []struct {
name string
input uint16
numCycles int
updated WrapAroundUpdateResult[uint32]
start uint16
extendedStart uint32
highest uint16
extendedHighest uint32
}{
// initialize - should initialize irrespective of numCycles
{
name: "initialize",
input: 10,
numCycles: 10,
updated: WrapAroundUpdateResult[uint32]{
IsRestart: false,
PreExtendedStart: 0,
PreExtendedHighest: 9,
ExtendedVal: 10,
},
start: 10,
extendedStart: 10,
highest: 10,
extendedHighest: 10,
},
// negative cycles - should just do an update
{
name: "zero",
input: 8,
numCycles: -1,
updated: WrapAroundUpdateResult[uint32]{
IsUnhandled: true,
// the following fields are not valid when `IsUnhandled = true`, but code fills it in
// and they are filled in here for testing purposes
PreExtendedHighest: 10,
ExtendedVal: 8,
},
start: 10,
extendedStart: 10,
highest: 10,
extendedHighest: 10,
},
// one cycle
{
name: "one cycle",
input: (1 << 16) - 6,
numCycles: 1,
updated: WrapAroundUpdateResult[uint32]{
PreExtendedHighest: 10,
ExtendedVal: (1 << 16) - 6 + (1 << 16),
},
start: 10,
extendedStart: 10,
highest: (1 << 16) - 6,
extendedHighest: (1 << 16) - 6 + (1 << 16),
},
// two cycles
{
name: "two cycles",
input: (1 << 16) - 7,
numCycles: 2,
updated: WrapAroundUpdateResult[uint32]{
PreExtendedHighest: (1 << 16) - 6 + (1 << 16),
ExtendedVal: (1 << 16) - 7 + 3*(1<<16),
},
start: 10,
extendedStart: 10,
highest: (1 << 16) - 7,
extendedHighest: (1 << 16) - 7 + 3*(1<<16),
},
}
for _, tc := range testCases {
t.Run(tc.name, func(t *testing.T) {
require.Equal(t, tc.updated, w.Rollover(tc.input, tc.numCycles))
require.Equal(t, tc.start, w.GetStart())
require.Equal(t, tc.extendedStart, w.GetExtendedStart())
require.Equal(t, tc.highest, w.GetHighest())
require.Equal(t, tc.extendedHighest, w.GetExtendedHighest())
})
}
}
func TestWrapAroundUint32(t *testing.T) {
w := NewWrapAround[uint32, uint64](WrapAroundParams{IsRestartAllowed: true})
testCases := []struct {
name string
input uint32
updated WrapAroundUpdateResult[uint64]
start uint32
extendedStart uint64
highest uint32
extendedHighest uint64
}{
// initialize
{
name: "initialize",
input: 10,
updated: WrapAroundUpdateResult[uint64]{
IsRestart: false,
PreExtendedStart: 0,
PreExtendedHighest: 9,
ExtendedVal: 10,
},
start: 10,
extendedStart: 10,
highest: 10,
extendedHighest: 10,
},
// an older number without wrap around should reset start point
{
name: "reset start no wrap around",
input: 8,
updated: WrapAroundUpdateResult[uint64]{
IsRestart: true,
PreExtendedStart: 10,
PreExtendedHighest: 10,
ExtendedVal: 8,
},
start: 8,
extendedStart: 8,
highest: 10,
extendedHighest: 10,
},
// an older number with wrap around should reset start point
{
name: "reset start wrap around",
input: (1 << 32) - 6,
updated: WrapAroundUpdateResult[uint64]{
IsRestart: true,
PreExtendedStart: (1 << 32) + 8,
PreExtendedHighest: (1 << 32) + 10,
ExtendedVal: (1 << 32) - 6,
},
start: (1 << 32) - 6,
extendedStart: (1 << 32) - 6,
highest: 10,
extendedHighest: (1 << 32) + 10,
},
// an older number with wrap around should reset start point again
{
name: "reset start again",
input: (1 << 32) - 12,
updated: WrapAroundUpdateResult[uint64]{
IsRestart: true,
PreExtendedStart: (1 << 32) - 6,
PreExtendedHighest: (1 << 32) + 10,
ExtendedVal: (1 << 32) - 12,
},
start: (1 << 32) - 12,
extendedStart: (1 << 32) - 12,
highest: 10,
extendedHighest: (1 << 32) + 10,
},
// duplicate should return same as highest
{
name: "duplicate",
input: 10,
updated: WrapAroundUpdateResult[uint64]{
IsRestart: false,
PreExtendedStart: 0,
PreExtendedHighest: (1 << 32) + 10,
ExtendedVal: (1 << 32) + 10,
},
start: (1 << 32) - 12,
extendedStart: (1 << 32) - 12,
highest: 10,
extendedHighest: (1 << 32) + 10,
},
// a significant jump in order should not reset start
{
name: "big in-order jump",
input: 1 << 31,
updated: WrapAroundUpdateResult[uint64]{
IsRestart: false,
PreExtendedStart: 0,
PreExtendedHighest: (1 << 32) + 10,
ExtendedVal: (1 << 32) + (1 << 31),
},
start: (1 << 32) - 12,
extendedStart: (1 << 32) - 12,
highest: 1 << 31,
extendedHighest: (1 << 32) + (1 << 31),
},
// now out-of-order should not reset start as half the range has been seen
{
name: "out-of-order after half range",
input: (1 << 31) - 1,
updated: WrapAroundUpdateResult[uint64]{
IsRestart: false,
PreExtendedStart: 0,
PreExtendedHighest: (1 << 32) + (1 << 31),
ExtendedVal: (1 << 32) + (1 << 31) - 1,
},
start: (1 << 32) - 12,
extendedStart: (1 << 32) - 12,
highest: 1 << 31,
extendedHighest: (1 << 32) + (1 << 31),
},
// in-order, should update highest
{
name: "in-order",
input: (1 << 31) + 3,
updated: WrapAroundUpdateResult[uint64]{
IsRestart: false,
PreExtendedStart: 0,
PreExtendedHighest: (1 << 32) + (1 << 31),
ExtendedVal: (1 << 32) + (1 << 31) + 3,
},
start: (1 << 32) - 12,
extendedStart: (1 << 32) - 12,
highest: (1 << 31) + 3,
extendedHighest: (1 << 32) + (1 << 31) + 3,
},
}
for _, tc := range testCases {
t.Run(tc.name, func(t *testing.T) {
require.Equal(t, tc.updated, w.Update(tc.input))
require.Equal(t, tc.start, w.GetStart())
require.Equal(t, tc.extendedStart, w.GetExtendedStart())
require.Equal(t, tc.highest, w.GetHighest())
require.Equal(t, tc.extendedHighest, w.GetExtendedHighest())
})
}
}
pkg/sfu/videolayerselector/dependencydescriptor.go
+31 -3
View File
@@ -24,6 +24,11 @@ import (
"github.com/livekit/protocol/logger"
)
const (
decisionCacheMaxElements = 256
decisionCacheNackEntries = 80
)
type DependencyDescriptor struct {
*Base
@@ -40,12 +45,14 @@ type DependencyDescriptor struct {
decodeTargetsLock sync.RWMutex
decodeTargets []*DecodeTarget
fnWrapper FrameNumberWrapper
restartGeneration int
}
func NewDependencyDescriptor(logger logger.Logger) *DependencyDescriptor {
return &DependencyDescriptor{
Base: NewBase(logger),
decisions: NewSelectorDecisionCache(256, 80),
decisions: NewSelectorDecisionCache(decisionCacheMaxElements, decisionCacheNackEntries),
fnWrapper: FrameNumberWrapper{logger: logger},
}
}
@@ -75,6 +82,21 @@ func (d *DependencyDescriptor) Select(extPkt *buffer.ExtPacket, _layer int32) (r
return
}
if ddwdt.RestartGeneration > d.restartGeneration {
d.logger.Debugw(
"stream restarted",
"packet", ddwdt.RestartGeneration,
"current", d.restartGeneration,
"structureKeyFrame", d.extKeyFrameNum,
"efn", ddwdt.ExtFrameNum,
"lastEfn", d.fnWrapper.LastOrigin(),
)
d.restart(ddwdt.RestartGeneration)
} else if ddwdt.RestartGeneration < d.restartGeneration {
// must not happen
d.logger.Warnw("packet from old generation", nil, "packet", ddwdt.RestartGeneration, "current", d.restartGeneration)
}
dd := ddwdt.Descriptor
extFrameNum := ddwdt.ExtFrameNum
@@ -269,7 +291,7 @@ func (d *DependencyDescriptor) Select(extPkt *buffer.ExtPacket, _layer int32) (r
"fn", dd.FrameNumber,
"efn", extFrameNum,
"sn", extPkt.Packet.SequenceNumber,
"isKeyFrame", extPkt.KeyFrame,
"isKeyFrame", extPkt.IsKeyFrame,
)
}
@@ -286,7 +308,7 @@ func (d *DependencyDescriptor) Select(extPkt *buffer.ExtPacket, _layer int32) (r
"fn", dd.FrameNumber,
"efn", extFrameNum,
"sn", extPkt.Packet.SequenceNumber,
"isKeyFrame", extPkt.KeyFrame,
"isKeyFrame", extPkt.IsKeyFrame,
)
result.IsRelevant = true
@@ -434,3 +456,9 @@ func (d *DependencyDescriptor) CheckSync() (locked bool, layer int32) {
return false, layer
}
func (d *DependencyDescriptor) restart(generation int) {
d.restartGeneration = generation
d.invalidateKeyFrame()
d.decisions = NewSelectorDecisionCache(decisionCacheMaxElements, decisionCacheNackEntries)
}
pkg/sfu/videolayerselector/dependencydescriptor_test.go
+11 -5
View File
@@ -15,7 +15,7 @@
package videolayerselector
import (
"sort"
"slices"
"testing"
"github.com/pion/rtp"
@@ -143,7 +143,7 @@ func TestDependencyDescriptor(t *testing.T) {
// non key frame, dropped
ret = ddSelector.Select(&buffer.ExtPacket{
KeyFrame: false,
IsKeyFrame: false,
DependencyDescriptor: &buffer.ExtDependencyDescriptor{
Descriptor: &dd.DependencyDescriptor{
FrameNumber: 1,
@@ -287,8 +287,14 @@ func createDDFrames(maxLayer buffer.VideoLayer, startFrameNumber uint16) []*buff
activeBitMask |= 1 << uint(i*int(maxLayer.Temporal+1)+j)
}
}
sort.Slice(decodeTargets, func(i, j int) bool {
return decodeTargets[i].Layer.GreaterThan(decodeTargets[j].Layer)
slices.SortFunc(decodeTargets, func(a, b buffer.DependencyDescriptorDecodeTarget) int {
if a.Layer.GreaterThan(b.Layer) {
return -1
}
if b.Layer.GreaterThan(a.Layer) {
return 1
}
return 0
})
chainDiffs := make([]int, int(maxLayer.Spatial)+1)
@@ -308,7 +314,7 @@ func createDDFrames(maxLayer buffer.VideoLayer, startFrameNumber uint16) []*buff
}
}
keyFrame := &buffer.ExtPacket{
KeyFrame: true,
IsKeyFrame: true,
DependencyDescriptor: &buffer.ExtDependencyDescriptor{
Descriptor: &dd.DependencyDescriptor{
FrameNumber: startFrameNumber,
pkg/sfu/videolayerselector/framechain.go
+3 -3
View File
@@ -71,11 +71,11 @@ func (fc *FrameChain) OnFrame(extFrameNum uint64, fd *dd.FrameDependencyTemplate
}
var intact bool
switch {
case sd == selectorDecisionForwarded:
switch sd {
case selectorDecisionForwarded:
intact = true
case sd == selectorDecisionUnknown:
case selectorDecisionUnknown:
// If the previous frame is unknown, means it has not arrived but could be recovered by NACK / out-of-order arrival,
// set up a expected callback here to determine if the chain is broken or intact
if fc.decisions.ExpectDecision(prevFrameInChain, fc.OnExpectFrameChanged) {
pkg/sfu/videolayerselector/framenumberwrapper.go
+4
View File
@@ -52,3 +52,7 @@ func (f *FrameNumberWrapper) UpdateAndGet(new uint64, updateOffset bool) uint64
f.last = new
return new + f.offset
}
func (f *FrameNumberWrapper) LastOrigin() uint64 {
return f.last
}
pkg/sfu/videolayerselector/framenumberwrapper_test.go
+2 -2
View File
@@ -21,7 +21,7 @@ import (
"github.com/stretchr/testify/require"
"github.com/livekit/livekit-server/pkg/sfu/utils"
"github.com/livekit/mediatransportutil/pkg/utils"
"github.com/livekit/protocol/logger"
)
@@ -65,7 +65,7 @@ func TestFrameNumberWrapper(t *testing.T) {
// frame out of order
firstF = secondF
secondF = getFrame(firstF, false)
// it is possile that an out of order non-keyframe has been converted to in order frame number if the diff is 32768
// it is possible that an out of order non-keyframe has been converted to in order frame number if the diff is 32768
// that is ok because the client can't decode in such case and always need to wait for the key frame.
// so it is just a failure of test case and increase the frame number here.
if secondF-firstF == 0x8000 {
pkg/sfu/videolayerselector/simulcast.go
+2 -2
View File
@@ -93,7 +93,7 @@ func (s *Simulcast) Select(extPkt *buffer.ExtPacket, layer int32) (result VideoL
isActive := s.currentLayer.IsValid()
found := false
reason := ""
if extPkt.KeyFrame {
if extPkt.IsKeyFrame {
if layer > s.currentLayer.Spatial && layer <= s.targetLayer.Spatial {
reason = "upgrading layer"
found = true
@@ -124,7 +124,7 @@ func (s *Simulcast) Select(extPkt *buffer.ExtPacket, layer int32) (result VideoL
}
// if locked to higher than max layer due to overshoot, check if it can be dialed back
if s.currentLayer.Spatial > s.maxLayer.Spatial && layer <= s.maxLayer.Spatial && extPkt.KeyFrame {
if s.currentLayer.Spatial > s.maxLayer.Spatial && layer <= s.maxLayer.Spatial && extPkt.IsKeyFrame {
s.previousLayer = s.currentLayer
s.currentLayer.Spatial = layer
pkg/sfu/videolayerselector/temporallayerselector/vp8.go
+2 -1
View File
@@ -16,6 +16,7 @@ package temporallayerselector
import (
"github.com/livekit/livekit-server/pkg/sfu/buffer"
"github.com/livekit/mediatransportutil/pkg/codec"
"github.com/livekit/protocol/logger"
)
@@ -36,7 +37,7 @@ func (v *VP8) Select(extPkt *buffer.ExtPacket, current int32, target int32) (thi
return
}
vp8, ok := extPkt.Payload.(buffer.VP8)
vp8, ok := extPkt.Payload.(codec.VP8)
if !ok {
return
}
pkg/sfu/videolayerselector/vp9.go
+1 -1
View File
@@ -50,7 +50,7 @@ func (v *VP9) Select(extPkt *buffer.ExtPacket, _layer int32) (result VideoLayerS
updatedLayer := v.currentLayer
if !v.currentLayer.IsValid() {
if !extPkt.KeyFrame {
if !extPkt.IsKeyFrame {
return
}