Squashed 'livekit-server/' changes from 154b4d26..46c43095
46c43095 fix goreleaser workflow, version 1.13.1 (#4577) e0815be2 chore: improve docker test shutdown reliability (#4576) bfd9deff expose TCPFallbackRTTThreshold and AllowUDPUnstableFallback via config (#4556) b93c1e16 Release v1.13.0. (#4573) fd452212 Update mediatransportutil to get ICE candidate timeout config (#4572) 8be8c74a Update github workflows (#4463) c4e41872 Update go deps to v1.17.2 (#4462) dc8e0310 Update go deps to v4 (#4482) 20fd1ad2 turn: allow for providing secret via file (#4564) 6590570d Pin pion/dtls to v3.1.2 (#4570) cdbbee1f deps: bump protocol + psrpc to latest tips (#4565) d290de81 Correct config comment (#4563) 77ecf920 rtc: report participant session end time on room move (#4561) 63be96f6 Prevent panic from nil(illegal) syncState.Subscriptions message (#4560) 835ef1b3 Metrics for participant active, i. e. fully established. (#4557) 5bd42534 Document of advertise_internal_ip and external_ip_only (#4554) 356ae211 Config documentation for advertise_internal_ip and skip_external_ip_validation (#4552) 7c319a67 rtc: prevent duration reporting for inactive participants (#4550) 2dd5e632 telemetry: split webhook-processed hook out of NewTelemetryService (#4548) 222177a9 service: prevent nil deref in validate with wrapped join request (#4547) dd7580b4 Protect against nil clientInfo (#4546) 145689e6 Start tracking Twirp method request latency in prometheus too, not just in logs (#4545) cde89627 rtc: emit per-data-track bytes via BytesTrackStats (#4540) 2e22911d Remove backwards compatibility support for TURN auth. (#4539) 062d1219 Use NACKQuueInterface type. (#4538) 7f08b04c Add IsIntentionalDisconnect helper (#4537) 1ab2bf04 Clean up packet size logging (#4536) 8ab92a80 Don't require media sections when joining (#4535) 019a6640 rtc: report participant kind code and details (#4534) 77595d38 TEL-336: fix sip error categorization (#4528) f303f499 Always enable rtx codec (#4533) e4a8a55c Check Less and LessEq in version compare. (#4532) 37eb7a32 Release v1.12.0 (#4529) 4a7b1e85 Create NACK tracker only once. (#4527) 89faaeba Apply ttl check only when authenticate allocation creating (#4526) b32933b0 Log details of RTCP packets. (#4525) 8b79ec9e Support SIP auth realm for inbound. (#4522) 4b8db3cf Add integration test for TURN auth failures (#4524) ef2e5efe Log large packets receive/send. (#4521) d1236750 feat: auto create rooms for tokens with the RoomCreate grant (#4320) 7a3e595b apply room tags from JWT grant room configuration (#4518) ab7fdeab add AssignmentHook to AssignJob; propagate websocket write errors (#4516) cf20c9cd Add expiry to TURN password. (#4515) 20d4a3a1 Populate data track loggers with context (#4514) 12fff29a allow setting agent job assignment url (#4512) ba366fc7 Fix SIP media config upgrade. (#4511) 8fbc5adf update protocol for protojson (#4510) 3de6f517 Add TURN permission handler. (#4505) 8ffcef93 Update protocol to support SIP media config. (#4509) 1ab1e072 test: verify upstream and downstream connection stats end-to-end (#4508) c4fd71a5 Fix sense check in DeltaInfo gathering (#4507) 803999ef rename agent environment to deployment (#4506) bacc21e6 add helper to check for agent worker endpoint (#4503) 253f977d add duration seconds reporting (#4500) ffab3bd3 add agent environment (#4498) ccdf23c8 Use mediatransportutil/codec package, no functional change (#4497) 680703f2 Include reception reoprts in receiver report callback. (#4496) f51798bc Fix publish-only limitations being incorrectly applied to receivers (#4495) a002337d Legacy TrackInfo.Simulcast flag. (#4493) af1dcc88 Add CloseWithReason to agent SignalConn interface (#4492) d7c2daf1 report all simulcast layers (#4491) c1ad2b22 Misc optimisations. (#4490) 19b9e8c0 Additional data tracks logging (#4489) 743d9c8b add support for client capabilities (#4461) fc47e478 Close peer connection unconditionally to unblock set local/remote (#4485) 639406eb Update module github.com/pion/ice/v4 to v4.2.3 (#4481) dc6b7505 reduce some heap use in packet path (#4478) f3b80b28 fix: wrap IPv6 addresses in brackets in UDP TURN URLs (RFC 3986) (#4476) 3a7f2628 Turn off transceiver re-use on Safari. (#4474) d84f3d7a add more types to signum (#4473) 701a37c2 Convert sort.Slice -> slices.SortFunc (#4472) 85be9d70 Avoid stream allocator event data cast to interface and back. (#4471) b43685e8 Keep a shadow copy of tracks for use by different stream allocator state (#4470) 27c2b149 Consolidate RTCP packets and do RTCP callback outside lock. (#4469) 31083307 do not log data track stats if not started (#4468) 9ee06635 feat(pion/ice): replace deprecated NAT1To1 with SetAddressRewriteRules (#4466) 8ccad68d Release v1.11.0 (#4459) dbf5cf61 Store concrete ICE candidate for remote candidates. (#4458) 2a04bc3c fix publisher frame count reporting for simulcast streams (#4457) 1d804737 fix: limit join request and WHIP request body to http.DefaultMaxHeaderBytes (#4450) 3cfb71e7 Use Muted in TrackInfo to propagated published track muted. (#4453) 69aa9479 Some drive-by clean up (#4452) 6c81f678 Add subscriber stream start event notification (#4449) ce1bf47b Revert "fix: ensure num_participants is accurate in webhook events (#4265) (#…" (#4448) cdb0769c fix: ensure num_participants is accurate in webhook events (#4265) (#4422) c91e79af Switch to stdlib maps, slices (#4445) ea7b9c6f Update module github.com/livekit/protocol to v1.45.3 (#4435) 97378368 Update go deps (major) (#3179) d6aef547 Update go deps (#3862) afc9feae Update github workflows (#4331) 4b385612 chore: pin GH commits and switch to golangci-lint (#4444) 2974ba87 Unsubscribe from data track on close (#4443) 5dc4e90d Apply IPFilter when get local ip (#4440) 88c77dc6 compute agent dispatch affinity from target load (#4442) 8fe99377 Log join duration. (#4433) 0a503a57 Add `Close` method for UpDataTrackManager and call it on participant (#4432) 55912dff Add some simple data track stats (#4431) 050909e6 Enable data tracks by default. (#4429) 72c7e65c chore: log API key during worker registration (#4428) 8a67dd1b Do not close publisher peer connection to aid migration. (#4427) 91e90c10 Add some more logging around migration. (#4426) c6ddc879 isExpectedToResume is based on whether flushing or not. (#4425) 7d06cfca Keep subscription synchronous when publisher is expected to resume. (#4424) 934f8598 Clean up data track observers on unsubscribe. (#4421) 9674ac48 Cleaning up some logs and standardising log frequency. (#4420) 7b925304 Drop time inverted packets in RED -> Opus conversion. (#4418) 4d8d232a ensure participant init is correctly serialized for logging (#4417) 4fe80877 Log time inversion between incoming packets (#4415) 248d7394 Guard against timestamp inversion in RED -> Opus conversion. (#4414) 9ab8c1d5 clear track notifier observers on subscription teardown (#4413) 397cd09a Embedded turn test (#4412) 56326654 Prepare release v1.10.1 (#4408) e9b113c8 Make the TURN bind address configurable and allow for multiple addresses. (#4315) 4bc5e6bb Address malformed H264/H265 parsing issues. (#4407) 77a0a4fc AV1 parser overflow fix. (#4405) ff7fd7ed feat(agent-dispatch): add job restart policy (#4401) 34bd1e08 do not log roll over for padding only packets (#4396) 13d02ee9 add deadline to dtls connect context (#4395) 9055a349 Path check helpers (#4392) 1f1eeb68 Fallback to servicestore if rpc is unavailable (#4391) 59e9bb41 Fix TURN server URL (#4389) 9e0a7e54 Close both peer connections to aid migration. (#4382) 9474c807 route participant reads through PSRPC instead of Redis (#4387) a5333a86 add packet trailer stripping support (#4361) 0e3d765d Release v1.10.0 (#4380) bc3aeaf3 Update grpc to address CVE-2026-33186 (#4381) 8cdd6f4c Replace deprecated io/ioutil with io in whipservice (#4375) 89410df7 handle AGENT_ERROR disconnect reason (#4339) 8f984c77 Fix repair stream ID reporting for RTX pairing. (#4369) cdfaacfc Restart nacker on OOB sequence number restart. (#4368) 4a9e0045 Update protocol. (#4367) 750d5904 Add API to restart lite stats. (#4366) c8bb2578 Rename log field "pID" to "participantID" for consistency (#4365) 77fc74a7 Do not block all ext ID determination on stream allocator listener (#4364) 90a46fab Do not kick off migration of closed participant (#4363) 5dc2e7b1 Switch data track extension to 1-byte ID/length. (#4362) 7323ad02 Sample data send error logging. (#4358) 0d34e455 Add option to not re-use transceiver in e2ee. (#4356) 95225ff2 don't require media section for dual peerconnection mode (#4354) e9639538 Refine ipv6 support (#4352) b34b0472 Add StopEgress function to the EgressLauncher interface (#4353) 69235ed2 update readme (#4340) db1a8046 defensive check for peer connection instance (#4350) cb7dc2d0 TEL-405: support originating calls from custom domains (#4349) 7eaaaada Mark last run of grow bucket outside goroutine. (#4348) caa47522 Add option to require media sections when participant joining (#4347) 087050d1 Release v1.9.12 (#4346) 493e87df Fix SIP client timeout. (#4345) 52c28a93 Log a bit more details of out-of-order TWCC feedback report. (#4343) 516aeabf Use ParticipantTelemetryListener of LocalParticipant. (#4342) b3510565 Exclude ice restart case from offer answer id mismatch warning (#4341) 303657bc feat: make INSTALL_PATH overridable in install script (#3954) 9d418689 Send participant left event after track unpublished for moved (#4334) bab91868 do not discount packets lost on duplicate packets (#4333) 939794cf mark + restart (#4329) 75f9c462 Add debug for receiver restart. (#4328) 74891f30 Protect against incorrect temporal layer. (#4327) f51b2732 Update pion/webrtc and deps to update dtls (#4326) b81bac0e Key telemetry stats worker using combination of roomID, participantID (#4323) 2d40449f Update self-hosting deployment documentation link (#4312) 03e90dd7 Fix for some CodeQL reported issues (#4314) 77c858f0 Log stats worker when it not closable. (#4313) a6035212 ESP32 client info (#4267) 478e486a Copilot suggested improvement to Github Actions permissions (#4310) cbd2f82d Copilot suggested improvement to Github Actions permissions (#4311) a9b8d40d Publish is always on publisher peer connection. (#4307) 8ae56406 generate & log egressID for start egress request (#4303) 35d7ef88 Avoid alloc in RTPStatsReceiver.Update (#4302) bb744916 More optimisation in RTPStats module. (#4298) cefd5da9 Optimise some bits in rtpstats_receiver (#4297) 52a4b848 Read client protocol from query param (#4294) 195b17f6 Populate client_protocol field in ParticipantInfo (#4293) 370e0a4d Set up audio config in audio level module when config is updated. (#4290) f3e9b688 Do not increase max expected layer on track info update. (#4285) a9849340 Avoid logger data race. (#4284) 97016627 Do not hold lock when creating buffer (#4283) 6b68e3d5 Create buffer if needed when a PLI is requested. (#4282) 3cca7180 use separate allocation for signal stats telemetry guard (#4281) d2bae34d refresh telemetry guard on participant move (#4280) 333f0349 clear reference guard when resetting signal stats (#4279) d1bab17b Add session duration and participant kind to closing log. (#4277) 1e689e1a Reducing some info level logs. (#4274) 88facc02 adds a test to ensure agent worker errors cause disconnection (#4273) 76a41a7a Generate config flags (#4268) 700e1788 require participant broadcast when metadata/attributes are set in token (#4266) b61799ec Ignore parse addr error when add remote candidate (#4264) 01bd966f Add silent frame for pcmu/a (#4258) 0c33b8c6 chore: move codecs/mime stuff to protocol (#4255) 165c1735 Update livekit protocol to v1.44.0 (#4254) 343f12b8 Wrapping SIP errors for invalid argument and not found (#4253) aea044c5 Defer setting clock rate in RTPStats module till codec is bound. (#4250) d9f716c1 FIx receiver restart race (#4248) 0508ee9f remove copy/paste left over line (#4246) 40408407 Release v1.9.11 (#4245) f8675507 Do not remove participant from cache on disconnect. (#4241) a35a6ae7 Add participant option for data track auto-subscribe. (#4240) 07572511 Wrapping the invalid request errors for CreateSipParticipant (#4239) 843d8c3e Update Pion transport package. (#4237) a05690d2 Changing field naming of data track packet (#4235) c4015008 Clear participant version cache on disconnect (#4234) 18db4ec1 Log modified timeout of API context. (#4232) ac20ccda Log timeout in API. (#4231) 541a7d6c Change some logs to debugw (#4229) 52ab3374 Return on SDP fragment read error. (#4228) 7fae5ac9 Do not restart receiver on codec change mid-session. (#4225) dafdc36e Add option to force simuclast codec. (#4226) 0a7dd40b Use only layer 0 for SVC codecs. (#4224) 4ec0f8f4 Support OpenTelemetry tracing. Add Jaeger support. (#4222) 80ba93fa Do NACK updates as soon as flow state is processed. (#4221) 4405afe2 Use atomic pointer and return interface from RED transformer constructors (#4220) b649c2fe Remove method not needed from REDTransformer. (#4219) f0080f35 Remove enable arrival time forwarding method. (#4217) 335f4c33 Swap result sink atomically rather than closing and setting. (#4216) 46651c19 Release v1.9.10 (#4214) 08ac4ecd Support preserving external supplied time. (#4212) f6efccce report video size from media data for whip (#4211) d92f6a79 return iceservers for whip (#4210) 1a4758ed Skip restart callback if external. (#4208) dde4fb49 configurable dependency descriptor restart (#4207) 08793bea Use active at time to check for track not bound timeout. (#4206) 3606ce54 Do not warn about track not bound if participant is not ready. (#4205) b8ddd0f9 Taking interface{} -> any modernize bits (#4204) b91cd2e4 Rework receiver restart. (#4202) bb00c864 Restart API on receiver. (#4200) 25ece1e9 Minor refactor in buffer base and audio level (#4198) 599002f8 ignore PLI requests for non-video (#4196) 2510b946 Taking a bunch of go modernize suggestions. (#4194) ed8e6afc Handle repair SSRC of simulcast tracks during migration. (#4193) c6bf7a27 Fix logging key and other clean up around stream restart. (#4192) 3cb9abb6 Update pion/webrtc to v4.2.1 (#4191) 8b0efb8c Resolve RTX pair via OnTrack also. (#4190) 381bce03 Return extended sequence number only and not packet. (#4189) 6bcbf54e Always instantiate nacker when using out-of-band sequence numbers. (#4187) e71184de Store buffer after creating it. (#4186) 7c8ea115 Refactor receiver and buffer into Base and higher layer. (#4185) 4104b827 update protocol (#4183) cd99fec2 Make new path for signalling v1.5 support. (#4180) 32cd0370 Flush the ext packets on restart/close and release packets. (#4179) 1df1316b Move OnDataTrackMessage to participant listener to support replay. (#4178) e7601251 Make data message naming a bit more consistent. (#4177) a04e566d Use published track for model access in data down track. (#4176) 47c86be1 Add support for TURN static auth secret credentials (#3796) 24559a28 chore(deps): update github workflows to v6 (#3866) 0a824386 add explicit room exists servicestore op (#4175) 39bd077d Release v1.9.9 (#4174) cbb2c617 Publish/Unpublish counter match. (#4173) fb849edc Minor clean up (#4172) c28e5e45 fix(kindToProto): update protocol (#4171) 47324abd Drop run away receiver reports. (#4170) 462ec324 prevent uint overflow setting packet not found count (#4169) 5c841b8e Some logging changes. (#4168) 2f2d0a57 skip lost sequence number ranges in getIntervalStats (#4166) 898ebe05 clean up manual roomservice log redaction (#4165) 3e417253 move delete to oss service store (#4164) 5964efbb Ensure subscribe data track handles are unique (#4162) a26c4830 Add support for RTP stream restart. (#4161) 386f0b38 fix typo in clearing index when removing track from room track manager (#4158) 0abfb251 deregister observability function when participant is closed (#4157) 97aba5e7 Consistently undo update to sequence number and timestamp when the (#4156) 2317c295 Fix panic while removing track from room track manager. (#4153) a0a28ac5 Avoid duplicate track add to room track manager. (#4152) f01008f8 Revert telemetry stats worker wait configuration. (#4151) ca4b56d2 Handle case of sequence number jump just after start. (#4150) 97099cae Configurable telemetry stats worker clean up wait. (#4148) d7db7cb3 chore: fix a large number of spelling issues (#4147) cadf2ad9 Release v1.9.8 (#4145) 498304cd defensive nil check (#4144) 20f6a497 Store ddParser in atomic.Pointer (#4143) 037cb906 release ext packet if patching fails (#4142) dd598ef2 Release ExtPacket if dependency descriptor or other parsing fails (#4141) 35b0e2bc update webrtc to 4.1.8 to pick up DTLS fingerprint check during handshake (#4140) 1c1a836c Mark RTCP buffer Write as noinline. (#4138) ea9b2177 protocol deps to get inactive file adjusted memory usage. (#4137) 64f3d1e9 switch participant callbacks to room to listener interface (#4136) c6e6c021 add debug metric for tracking references (#4134) a30c79fa Use isEnding to indicate if down track could be resumed. (#4132) 8c241ecf Fix RTCP reader leak in DownTrack. (#4131) 3eef869a Do not pause rid in SDP (#4129) 8e01e595 Release 1.9.7 (#4128) 7c1a0fab Fix concurrent map access. (#4127) 14446b1c Let participant close remove the published tracks. (#4125) fa0633aa move utils.WrapAround to mediatransportutil (#4124) f8706cd4 Update pion/ice to stop gather first on close (#4123) 7954748d Data tracks (#4089) 7158d983 log bucket growth (#4122) 04b35eb6 Release v1.9.6 (#4121) ebdcead5 Update mediatransportutil to get bucket packet size limit. (#4120) 411b09f6 Release v1.9.5 (#4119) 8dcf235a Update pion/ice - attempt to address tcp packet conn close hang (#4116) 64c65143 Update mediatransportutil (#4115) 0a2943bb Clean up bits added to debug peer connection close hang. (#4114) 9c483a69 Use released version v1.8.41 of pion/sctp (#4113) 35c79a57 Update SCTP hash, had the wrong one in previous PR (#4111) e0fbbef1 Update pion/sctp with RFC9260 revert (#4110) f3c80917 Try SCTP with read deadline to unblock abort. (#4109) bd5382da Splitting transport close timeout logs. (#4108) 6d4154b8 Update pion/ice. (#4107) a6418ae2 Log more peer conenction state on close timeout. (#4105) 06d99974 Check for cancel on unsubscription/source track going away. (#4104) 7f10e18b Record join/publish/subscribe cancellations. (#4102) 40293632 Clear stereo=1 if stereo is not enabled. (#4101) 70f6def3 Add checks for participant and sub-components close. (#4100) ffbabcc7 Switch forwarding latency log to Debugw (#4098) 27d82a72 Fix "address" typo in transport logs (addddress → address) (#4097) 37a06821 logger proto redaction. (#4090) 54cf7d46 Control latency of lossy data channel (#4088) 5175c1af Lock x/tools at 0.37.0 (#4085) d510fff1 Downgrade x/tools to be able to make a release (#4084) c3ea5890 Prepare release v1.9.4. (#4083) 3a128e61 protocol bump for SIP error mapping and validation (#4081) c3964ba2 Use sync.Pool for objects in packet path. (#4066) f8b994d4 Forwarding latency measurement tweaks. (#4080) f4929f09 Revert "Revert pion/transpor to v3.0.8 (#4073)" (#4074) a04d9c48 Revert pion/transpor to v3.0.8 (#4073) 2d5054ad kind details for connector (#4072) a272e28a Log raeson for subscriber not being to determine codec. (#4071) b9b4eec9 Update pion/transport to v3.1.1 (#4070) b23d093c update protocol (#4069) 4ce07bed Higher resolution forwarding latency histogram. (#4067) 858db7ab fix(deps): update module github.com/livekit/protocol to v1.43.0 (#4015) 1dc9b8fc Use buffered indicator to exclude from forwarding latency. (#4062) f117ee51 Track start up delay. (#4061) 4872f205 Return write count from WriteRTP. (#4059) d0ba46b4 Log write count atomic. (#4057) ae5fb7e8 Add packet to forwarding stats only if packet is forwarded. (#4056) f6909192 Update PsRPC to get redis pipeliner implementation. (#4055) ca3c507b Prevent invalid track access while peer connection is shutting down. (#4054) 9ca6ee00 Use replace so that x/tools does not get overridden (#4048) b9323eab chore(deps): downgrade x/tools for counterfeiter (#4047) 2f1e6c36 Prep release v1.9.3 (#4046) 9d5c351d Fix prom units for forwarding latency/jitter. (#4045) e183657c Add prom histogram for forwarding latency and jitter. (#4044) 1eefeb30 Enable AbsCaptureTimeURI in RTC configuration (#4043) 075a7576 Use simulcast codec as default policy for audio track (#4040) c264b504 Don't warn 0 payload type for PCMU (#4039) 32fc3525 Broadcast cond var on RTX write. (#4038) 061eb8b4 AddDownTrack to regressed codec after restarting forwarder. (#4037) c87eb8ed fix: add missing Unlock() in AddReceiver (#4036) 70444924 if RingingTimeout is provided, deadline should be set to that timeout. (#4018) ab906d71 Prevent leakage of previous codec after codec regression. (#4035) 79b03f97 Log queueing latency when encountering high forwarding latency (#4034) 29117b14 set max layer in allocation (#4033) 15b19ccd Remove ~ from rid which indicates disabled layer to get the actual rid (#4032) 34e16a87 Check more conditions for opportunistic alloc. (#4031) 81fbd355 Use the optimal allocation function for opportunistic allocation. (#4030) a2ce73e0 Do not bind buffer if codec is invalid. (#4028) cef6fdb7 Correct direction for request/response for prom counters. (#4027) 5042c06c Use rtp converter from protocol/utils/rtputil (#4020) 5a426d15 Use rtp converter from protocol/utils (#4019) 35fb8877 feat: use env var for GOARCH (#4012) c0397696 Issue #1 only: Fix spatial layer initialization in Forwarder (#4003) 2afbf0e8 Some golang modernisation bits. (#4016) 484f784a Prepare release v1.9.2 (#4011) ad074ed2 counterfeiter needs an older version of x/tools (#4009) e63e8b6f Include mid -> trackID in both SDP offer and answer. (#4007) 781dfede Do not call receiver methods under settings lock. (#4006) 69ff25a0 Use answer with mid -> trackID mapping when in single peer connection (#4005) fe912acf Update pion/webrtc to prevent GetStats panic. (#4004) 7930dcde Do not try to read stats from peer connection after close. (#4002) ca0d5ee9 Count request/response packets on both client and server side. (#4001) dd62eb00 Resort to full search for requested quality is not available. (#4000) f6ca82d1 Revert to using silence packets for audio dummy start. (#3999) 0e2c59c8 Sort codec layers when adding track (#3998) 100bb46a Adding ProviderInfo to GetSIPTrunkAuthenticationResponse (#3993) a8d4df66 "Power of Two Random Choices" option for node selection (#3785) a20bbe34 Log RPC details. (#3991) 158496bc Increment RTP timestamp on padding when using dummy start. (#3989) 4f6ed65d Limit check to red + opus when looking for primary codec match. (#3988) a87f6c4b Allow passing inline trunk for outbound calls. (#3987) bf06596f Support Opus mixed with RED when encrypted. (#3986) ea208a1c Add encryption datapacket type (#3869) 2a6adbe8 Use padding only packets for dummy start of audio. (#3984) be018f97 Provide the InputVideo/AudioState to Ingress in WHIPRTCConnectionNotify (#3982) 146bd969 Do not panic of redis is not configured (#3981) 01337ba7 Do not start forawarding on out-of-order packet. (#3985) c7f625d6 Do not force codec regression between opus and red. (#3980) 3bd20ddb Revert unintentional change to not handle transport fallback on (#3970) 89a2f46c Update deps to fix redis issue when 1 cluster address is provided (#3969) 060719d1 add config for user data recording (#3966) b3ee219c fix stats worker closed condition (#3965) 3d737031 add idempotent reference count to telemetry stats worker (#3964) 735c663a Update protocol for EventKey helper. (#3963) 646b9de8 Add node_ip to config-sample.yaml (#3960) 0bf7b178 avoid logging on small values (#3958) 00ff2ab9 Adjust for hold time when fowarding RTCP report. (#3956) e180be06 short circuit participant broadcast filter in livestream mode (#3955) bfba6fee Adjust stream allocator ping interval based on state. (#3951) 3837006b Revert "Switch ops queue a singly linked list. (#3949)" (#3950) 990c5faf feat: server rpc apis (#3904) 80b11662 Switch ops queue a singly linked list. (#3949) 56ee2328 handle terminated job requests (#3948) 49f9b9c8 Flush stats when there are no packets. (#3947) e6a3df1e ForwarStats.GetStats needs to be public (#3946) 824d116b Tweaks tresholds for logging high forwarding latency/jitter. (#3945) 408492e0 Log some information around high forwarding latency. (#3944) 6a41fae5 Use microseconds for forwarding stats. (#3943) 856e0871 mediatransportutil to log local address when validating external IP (#3942) 40101cf7 Update protocol for SipCreateParticipant (#3939) b07e7a38 Use difference in key frame counter to stop seeder. (#3936) d7f92878 Avoid matching on empty track id. (#3937) 56fb2885 Do DD restart only if DD structure is present. (#3935) 86facce9 More debugging of DD jump (#3934) 6058a3f6 Add debugging from DD frame number wrap around. (#3933) dc6825c0 mediatransportutil crash fix for logging local address (#3930) d6f0588f Update mediatransportutil to log external IP found via STUN. (#3929) 2c30a064 Fix dynacast subscriber node clearing on move participant. (#3926) 6489237e Simulcast audio fixes (#3925) 9f0ab870 Wait for `SetRemoteDescription` before configuring senders. (#3924) df6c26db Subscrbed audio codecs - update from remote nodes. (#3921) 798fa761 Support simulcasting of audio (#3920) f4a06cf0 Clean code as there is no oss sweeper for ingress (#3918) 5f561b4f Include agent_name as a participant attribute (#3914) 782a35e8 update protocol for psrpc (#3915) eee2001a Set publisher codec preferences after setting remote description (#3913) fc995533 add incoming request id to request response message (#3912) 76645fad Rpcs for ingress proxy WHIP (#3911) 991a4a4f Refactor subscribedTrack + mediaTrackSubscriptions. (#3908) e16b3ba9 Use gzip reader pool (#3903) 17c34921 update protocol for sip api change (#3902) 2f43a575 Release candidate for v1.9.1 (#3899) 07c40cf3 Use `RequestResponse` to report protocol handling errors (#3895) 98352fd0 Prevent race in determining BWE type. (#3891) f7291fda Do not send both asb-send-time and twcc. (#3890) 21b42fa6 Do not advertise NACK for RED. (#3889) 6633bf93 Use departure timeout from room preset. (#3888) 38f7906e Handle migration better in single peer connection case. (#3886) 5026de2b handle frame number wrap back in svc (#3885) 091e3c13 Revert to using answer for migration case. (#3884) 2aeadf14 init ua parser once (#3883) 998a9f94 Switch known rids from 012 -> 210, used by OBS. (#3882) 890fd942 Single peer connection mode (#3873) bfe98eaa fix: ensure the participant kind is set on refresh tokens (#3881) 8d270e2a chunk room updates (#3880) b4e146c5 update mediatransport util for ice port 3478 (#3877) dc3a7753 Fix timeout handing in StopEgress (#3876) d62336e1 Remove unnecessary check (#3870) c58e5d23 Update golang Docker tag to v1.25 (#3864) 98d577ee Update module github.com/livekit/protocol to v1.40.0 (#3865) afbf541e Update pion deps (#3863) b660c3b5 Extract video size from media stream (#3856) 456b8709 Fix missed unlock (#3861) d500806e Handle no codecs in track info. (#3859) 11b240d6 Log track settings more. (#3853) 1aa0f963 Log signal messages on media node. (#3852) b182d07b Log signal messages as debug. (#3851) a370bb20 Support G.711 A-law and U-law (#3849) fa5f4ef3 Populate SDP cid in track info when available. (#3845) eed27885 Send `participant_connection_aborted` when participant session is closed (#3848) 61e59346 Update go deps (#3439) 1b228913 Support video layer mode from client and make most of the code mime aware (#3843) f2da4444 Support per simulcast codec layers. (#3840) f275f592 handle SyncState in join request (#3839) 5d44cf6d Use wrapped join request to be able to support compressed and uncompressed. (#3838) 5ca16264 Support join request as proto + base64 encoded query param (#3836) 7dea1012 Clean up missed v2 pieces (#3837) 34a49130 Delete v2 signalling (#3835) 1fe33716 Fix: RingingTimeout was being skipped for transferParticipant (#3831) 5751692a deps (#3829) db4bc127 Get to the point of connecting publisher PC and using it for async signalling (#3822) 5e483e75 update readme (#3809) e3155b14 Get to the point of establishing subscriber peer connection. (#3821) a7ce1382 HTTP DELETE of participant session (#3819) 01de0e36 Do not send leave if nil (to older clients) (#3817) 10103449 Add country label to edge prom stats. (#3816) 68387b41 Minor tweak to keep RPC type at service level. (#3815) a75295fc More v2 signalling changes (#3814) b20db94d Validation end point for v2 signalling. (#3811) f2f595f4 update readme (#3808) fffc2ac0 Use signalling utils from protocol (#3807) f5fc82d3 Filling out messages unlikely to change in v2. (#3806) 1c99b9ad Split signal segmenter and reassembler. (#3805) 0a1bfd30 Signal handling interfaces and participant specific HTTP PATCH. (#3804) 7837c8e5 starting signaller interface (#3802) 18ce5244 Handle Metadata field from RoomConfig (#3798) 2a6a9b8a Grouping all signal messages into participant_signal. (#3801) 078c01fa Signal v2: envelope and fragments as wire message format. (#3800) b9a44c3f Signalling V2 protocol implementation start (#3794) ba702a53 forward agent id to job state (#3786) 1f31d430 Map ErrNoResponse to ErrRequestTimedOut in StopEgress to avoid returning 503 (#3788) 51bbe8c5 Set participant active when peerconnection connected (#3790) 40028dc3 Normalize known rids. (#3779) ddd92329 Return default layer for invalid rid + track info combination. (#3778) 8c033ce9 Enable H265 by default (#3773) 7678e087 Set rids for all codecs. (#3772) 5d636acf Limit taking rids from SDP only in WHIP path. (#3771) 4d09e5b5 Log SDP rids to understand the mapping better. (#3770) c69f1aae Revert "Temporary change: use pre-defined rids" (#3769) 8197438e bounds check layer index (#3768) d11da5f5 Temporary change: use pre-defined rids (#3767) cb4da533 fix(deps): update module github.com/livekit/protocol to v1.39.3 (#3733) d6d2b6d8 feat(cli-flags): add option for cpu profiling (#3765) 9fc4ddbe ClearAllReceivers interface is used to pause relay tracks. (#3761) 1216113b Do not need to just clean up receivers. Remove that interface. (#3760) ef6c38ce Log previous allocation to see changes. (#3759) 01bf9685 SVC with RID -> spatial layer mapping (#3754) c481396f offer could be nil when migrating. (#3752) 8c2fc0bc Fix svc encoding for chrome mobile on iOS (#3751) e467daa0 move egress roomID load to launcher (#3748) 3783ebb3 feat(cli): update to urfave/cli/v3 (#3745) 03d3fcab Fix data packet ParticipantIdentity override logic in participant.go (#3735) 068b4366 reuse compiled client config scripts (#3743) e754a860 return error when moving egree/agent participant (#3741) 7542cf07 remove unused code (#3740) 9d569e2f Take ClientInfo from request. (#3738) 80774263 chore: set workerid on job creation (#3737) 5549ab55 Revert clearing RIDs. (#3732) ae967313 Clear rids if not present in SDP. (#3731) 0e033907 Return highest available layer if requested quality is higher than (#3729) 9ce737db Add log for dropping out of order reliable message (#3728) 1b95e818 Don't check bindState on downtrack.Bind (#3726) 670f927f Set and use rid/spatial layer in TrackInfo. (#3724) a9e29116 Add Id to SDP signalling messages. (#3722) 4ec828ce Fix bug with SDP rid, clear only overflow. (#3723) 8f6c3a9b Clear rids from default for layers not published. (#3721) ce07740e Add simulcast support for WHIP. (#3719) e98fb94f Create client config manager in room manager constructor. (#3718) fdf9b852 e2e reliability for data channel (#3716) 35dda8ea swap pub/sub track metrics (#3717) 1d9a4366 Do not require create permission for WHIP participant. (#3715) e0aea17a Flush stats on close (#3713) 630aa7d9 implement observability for room metrics (#3712) e7f0294e remove unused ws signal read loop (#3709) 77f70b18 for real, pick up protocol change for webhooks queue length bucnkets (#3707) 7b180646 protocol dep for webhook stats buckets (#3706) b0ab95ba warn about credentials when used in tokens (#3705) a72ce30f Small changes to add/use helper functions for length checks. (#3704) 425f6bb3 Allow passing extra attributes to RTC endpoint. (#3693) 758e1762 Add a trend check before declaring joint queuing region. (#3701) fe81e411 Adds Devin to readme so it auto updates DeepWiki weekly (#3699) 09dede35 version bump v1.9.0 (#3698) fc867c5b Webhook prom stats (#3697) 0e17916f Do not use Redis pipeline for SIP delete. Fixes Redis clustering support. (#3694) 1b760393 WHIP support. (#3692) e4f7d81b add client ip to agent worker registration (#3675) 6b849a87 update mediatransportutil for sctp congestion control (#3673) 83b189b0 Add ServerInfo to ReconnectResponse (#3671) 5f87a35b Prevent operating on swapped out map. (#3670) 13b55a80 move agent token (#3669) c9385edd handle agent worker jwt (#3668) 3b359d8b Use logger resolver reset to reset contexts. (#3665) dbb70e0f Fix dynacast quality for moving out tracks (#3664) 0a5f3c2a resolve new room name logger earlier when moving participant (#3662) 2df05517 Revert unbound transceiver stop. (#3661) 5172af15 ~Send initial participant update only after a participant becomes active.~ - General clean up (#3655) 5d4d86f8 protocol to pick resolver values replace (#3659) 7f8e6323 Send self participant update immediately. (#3656) 11630878 Use unordered for lossy data channel. (#3653) aee34ffe log request for agent dispatch api (#3650) 793b383a Add Moving participant to another room (#3648) 2fff36cb Stub MoveParticipant so that cloud can include the latest protocol. (#3646) d4ab1142 Redact address (#3643) b83190a3 protocol update to fix memory stats path (#3642) a1f4e88e Update protocol to latest, got bit by tag (#3641) 8d3902af Protocol to pick up cgroups v2 memory path fix (#3640) 2f002388 Use participant close reason in remove. (#3639) 58822c26 Include clientInfo in connectivity logs. (#3638) 6d6393a6 Take AudioFeatures from AddTrack. (#3635) 08670412 Limit buffer queue before Bind. (#3634) 9f5bc9b9 Avoid synthesising duplicate feature. (#3632) 847239c3 Disable vp9 for safari 18.4 (#3631) f69ab680 Populate the sender identity when translating to user packet. (#3628) e1490558 Forward data between WHIP client and non-WHIP client (#3627) 6739e7bc Broadcast inside lock (#3626) f24152b4 Call Broadcast in lock scope. (#3625) b760918a Use logger from request context. (#3623) 34a2e2c1 Check for multiple layers for managed track. (#3622) 4955ebe4 Forward transfer headers to internal request (#3615) d9ee9214 Set up RTX for WHIP publish (#3619) d8cf5439 Determine TURN connection type and no fallback for TURN/TLS. (#3612) d0d212fd Fix WHIP ICE restart. (#3616) 5e7f8a12 Update mediatransportutil for max sctp message (65K) (#3611) 28dfac14 Use exported GetEgressNotifyOptions (#3604) 75236bef protocol update to fix IPv6 SDP fragment parsing (#3603) 2130980d Add basic video support to WHIP. (#3602) e5cbb227 Allow specifying extra webhooks with egress requests (#3597) 7e16106a Add OnSubscirberReady callback on LocalParticipant. (#3600) 5c2d96b9 Check DestinationRoom of VideoGrant for participant forwarding (#3599) 2e236a19 Revert participant state ACTIVE change. (#3598) 35ac5f56 Add support for WHIP ICE Trickle/Restart. (#3596) ec2dff96 Fix SIP updates when replacing slices. (#3592) e24fe77b map PEER_CONNECTION_DISCONNECTED -> CONNECTION_TIMEOUT (#3591) 6ee6eb43 Do not drop audio codecs (#3590) 68357ba6 List audio codecs after video codecs. (#3589) 05a891ff Fix rule (had an extra bracket) (#3588) d7c41091 Exclude RED from enabled codecs for Flutter + 2.4.2 + Android. (#3587) ee08aede skip out of order participant state updates (#3583) 15a8d9a2 Break track published fuse when there are no tracks (#3581) 35f83c51 Replace Promise with Fuse. (#3580) 07fe9b72 Prevent migration race. (#3579) ac8082ff Use older SDP module to accommodate bad SDP. (#3578) 1c8307c7 Use cgroup for memstats. (#3573) e9be0fca log SDP offer on error (#3577) 3238ab8d Calculate rates for memory used and total. (#3570) d08487bf Unlabeled (pass through) data channels. (#3567) 52ce18d5 fix: revert recent changes to determine simulcast from sdp (#3565) cdfbb106 Audio uses signal SignalCid and SdpCid. (#3564) ed5e2f16 Keep simulcast information tied to receiver. (#3563) ad010cfc chore(logs): log VLS type for VP9/AV1 (#3561) 8cc17f8f Rework node stats a bit. (#3555) 15f56551 fix(video): determine svc/simulcast from SDP for advanced codecs (#3549) 2b6a46f4 Handle `prefer_regression` for backup codec (#3554) b0abb0ae Add option to use different pacer with send side bwe. (#3552) 26822b6b ParseUsername utility for TURN user name. (#3547) 55909ed7 log the initial join response (#3546) 97fcb82a Fix: Return NotFoundErr instead of Unavailable when the participant does not exist in UpdateParticipant. (#3543) 75d0e18e Implement SIP update API. (#3141) e118aff1 Fire track subscribed when the subscriber connected (#3540) 13417c01 Send mute event only on change (#3537) 7f4c4597 Stubs for SIP update API. (#3533) fe673bb2 Send regressed codec upstream stats to analytics. (#3532) 8eb81388 Use a generation to counter to stop key frame seeder on codec change (#3531) 188470a2 Do not accept unsupported track type in AddTrack (#3530) 507fc9cf Do not instantiate 0 sized sequencer. (#3529) 20bddfea Clean up published track on participant removal. (#3527) 65d8aa28 Handle subscribe race with track close better. (#3526) a6cb00b3 Reduce seeder duration to 30s and also do not force send PLI. (#3525) c8233205 Add a key frame seeder in up track. (#3524) 0f61ff3a Remove redundant log (#3523) 7685cd25 Log ParticipantInit on signal start to get a picture of join params (#3522) ac9e62ef add server agent load threshold config (#3520) cd5d32f0 Add pID and connID to log context to make it easier to search using pID. (#3518) 2d9aa6dd Update api call info method (#3515) b3779a90 WebHookConfig (#3517) 6121b9af Check ForwardParticipant room name (#3514) 9a7c9442 mediatransportutil update (#3511) 50ab47c1 Log packet drops/forward. (#3510) 139d1b13 Add ForwardParticipant method to room service (#3507) 6c04909f Use atomic to store codec. (#3505) 7f6afe05 Prevent bind lock deadlock on muted. (#3504) 48063df5 load mime type before calling writeBlankFrameRTP (#3502) d2e6cd15 Do not bind lock across flush which could take time (#3501) 47896f50 Update protocol and IO service. (#3499) 1dc42eef Bump github.com/go-jose/go-jose/v3 from 3.0.3 to 3.0.4 (#3497) 3a35cbc4 Log migration complete only when coming from sync (#3496) c2f17a10 refactor: using slices.Contains to simplify the code (#3495) 01e51dbd fix: fix the wrong error return value (#3493) ff9115b2 Disable dd parser for vp8 if extension is not found (#3492) c3e06f05 Do not attempt to create objects for URL ingresses as the ingress service will do so (#3491) f0edfbba Fix receiver rtt/jitter. (#3487) 05dfd30d Take RTT and jitter from receiver view while reporting track stats for (#3483) 04ed5683 Don't issue TrackPublished/Unpublished event on migrated track (#3482) 1cffe30c Use a RED transformer to consolidate both RED -> Opus OR Opus -> RED (#3481) 591888f7 Fix missing RTCP sender report when forwarding RED as Opus. (#3480) git-subtree-dir: livekit-server git-subtree-split: 46c4309554d37d23ee8da88a8a7e02a68fba09c1
This commit is contained in:
@@ -25,13 +25,12 @@ import (
|
||||
"github.com/livekit/protocol/livekit"
|
||||
"github.com/livekit/protocol/utils"
|
||||
"github.com/livekit/protocol/utils/mono"
|
||||
"github.com/livekit/protocol/utils/rtputil"
|
||||
)
|
||||
|
||||
const (
|
||||
cFirstPacketTimeAdjustWindow = 2 * time.Minute
|
||||
cFirstPacketTimeAdjustThreshold = 15 * 1e9
|
||||
|
||||
cSequenceNumberLargeJumpThreshold = 100
|
||||
)
|
||||
|
||||
// -------------------------------------------------------
|
||||
@@ -192,6 +191,8 @@ func (w WrappedRTCPSenderReportStateLogger) MarshalLogObject(e zapcore.ObjectEnc
|
||||
return nil
|
||||
}
|
||||
|
||||
// ------------------------------------------------------------------
|
||||
|
||||
func RTCPSenderReportPropagationDelay(rsrs *livekit.RTCPSenderReportState, passThrough bool) time.Duration {
|
||||
if passThrough {
|
||||
return 0
|
||||
@@ -205,6 +206,8 @@ func RTCPSenderReportPropagationDelay(rsrs *livekit.RTCPSenderReportState, passT
|
||||
type rtpStatsBase struct {
|
||||
*rtpStatsBaseLite
|
||||
|
||||
rtpConverter *rtputil.RTPConverter
|
||||
|
||||
firstTime int64
|
||||
firstTimeAdjustment time.Duration
|
||||
highestTime int64
|
||||
@@ -251,6 +254,18 @@ func newRTPStatsBase(params RTPStatsParams) *rtpStatsBase {
|
||||
}
|
||||
}
|
||||
|
||||
func (r *rtpStatsBase) SetClockRate(clockRate uint32) {
|
||||
r.lock.Lock()
|
||||
defer r.lock.Unlock()
|
||||
|
||||
r.setClockRateLocked(clockRate)
|
||||
}
|
||||
|
||||
func (r *rtpStatsBase) setClockRateLocked(clockRate uint32) {
|
||||
r.rtpConverter = rtputil.NewRTPConverter(int64(clockRate))
|
||||
r.rtpStatsBaseLite.setClockRateLocked(clockRate)
|
||||
}
|
||||
|
||||
func (r *rtpStatsBase) seed(from *rtpStatsBase) bool {
|
||||
if !r.rtpStatsBaseLite.seed(from.rtpStatsBaseLite) {
|
||||
return false
|
||||
@@ -346,6 +361,13 @@ func (r *rtpStatsBase) UpdateKeyFrame(kfCount uint32) {
|
||||
r.lastKeyFrame = time.Now()
|
||||
}
|
||||
|
||||
func (r *rtpStatsBase) KeyFrame() (uint32, time.Time) {
|
||||
r.lock.RLock()
|
||||
defer r.lock.RUnlock()
|
||||
|
||||
return r.keyFrames, r.lastKeyFrame
|
||||
}
|
||||
|
||||
func (r *rtpStatsBase) UpdateRtt(rtt uint32) {
|
||||
r.lock.Lock()
|
||||
defer r.lock.Unlock()
|
||||
@@ -374,7 +396,11 @@ func (r *rtpStatsBase) GetRtt() uint32 {
|
||||
return r.rtt
|
||||
}
|
||||
|
||||
func (r *rtpStatsBase) maybeAdjustFirstPacketTime(srData *livekit.RTCPSenderReportState, tsOffset uint64, extStartTS uint64) (err error, loggingFields []interface{}) {
|
||||
func (r *rtpStatsBase) maybeAdjustFirstPacketTime(
|
||||
srData *livekit.RTCPSenderReportState,
|
||||
tsOffset uint64,
|
||||
extStartTS uint64,
|
||||
) (adjustment int64, loggingFields []any, err error) {
|
||||
nowNano := mono.UnixNano()
|
||||
if time.Duration(nowNano-r.startTime) > cFirstPacketTimeAdjustWindow {
|
||||
return
|
||||
@@ -387,49 +413,52 @@ func (r *rtpStatsBase) maybeAdjustFirstPacketTime(srData *livekit.RTCPSenderRepo
|
||||
// in some network element along the way), push back first time
|
||||
// to an earlier instance.
|
||||
timeSinceReceive := time.Duration(nowNano - srData.AtAdjusted)
|
||||
extNowTS := srData.RtpTimestampExt - tsOffset + uint64(timeSinceReceive.Nanoseconds()*int64(r.params.ClockRate)/1e9)
|
||||
extNowTS := srData.RtpTimestampExt - tsOffset + r.rtpConverter.ToRTPExt(timeSinceReceive)
|
||||
samplesDiff := int64(extNowTS - extStartTS)
|
||||
if samplesDiff < 0 {
|
||||
// out-of-order, skip
|
||||
return
|
||||
}
|
||||
|
||||
samplesDuration := time.Duration(float64(samplesDiff) / float64(r.params.ClockRate) * float64(time.Second))
|
||||
samplesDuration := r.rtpConverter.ToDurationExt(uint64(samplesDiff))
|
||||
timeSinceFirst := time.Duration(nowNano - r.firstTime)
|
||||
now := r.firstTime + timeSinceFirst.Nanoseconds()
|
||||
firstTime := now - samplesDuration.Nanoseconds()
|
||||
|
||||
getFields := func() []interface{} {
|
||||
return []interface{}{
|
||||
"startTime", time.Unix(0, r.startTime),
|
||||
"nowTime", time.Unix(0, now),
|
||||
"before", time.Unix(0, r.firstTime),
|
||||
"after", time.Unix(0, firstTime),
|
||||
"adjustment", time.Duration(r.firstTime - firstTime),
|
||||
"extNowTS", extNowTS,
|
||||
"extStartTS", extStartTS,
|
||||
"srData", WrappedRTCPSenderReportStateLogger{srData},
|
||||
"tsOffset", tsOffset,
|
||||
"timeSinceReceive", timeSinceReceive,
|
||||
"timeSinceFirst", timeSinceFirst,
|
||||
"samplesDiff", samplesDiff,
|
||||
"samplesDuration", samplesDuration,
|
||||
}
|
||||
}
|
||||
adjustment = r.firstTime - firstTime
|
||||
|
||||
if firstTime < r.firstTime {
|
||||
if r.firstTime-firstTime > cFirstPacketTimeAdjustThreshold {
|
||||
if adjustment > cFirstPacketTimeAdjustThreshold {
|
||||
err = errors.New("adjusting first packet time, too big, ignoring")
|
||||
loggingFields = getFields()
|
||||
loggingFields = r.firstPacketTimeAdjustLogFields(now, firstTime, adjustment, extNowTS, extStartTS, srData, tsOffset, timeSinceReceive, timeSinceFirst, samplesDuration, samplesDiff)
|
||||
} else {
|
||||
r.logger.Debugw("adjusting first packet time", getFields()...)
|
||||
r.firstTimeAdjustment += time.Duration(r.firstTime - firstTime)
|
||||
r.firstTimeAdjustment += time.Duration(adjustment)
|
||||
r.logger.Debugw("adjusting first packet time", r.firstPacketTimeAdjustLogFields(now, firstTime, adjustment, extNowTS, extStartTS, srData, tsOffset, timeSinceReceive, timeSinceFirst, samplesDuration, samplesDiff)...)
|
||||
r.firstTime = firstTime
|
||||
}
|
||||
}
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
func (r *rtpStatsBase) firstPacketTimeAdjustLogFields(now, firstTime int64, adjustment int64, extNowTS, extStartTS uint64, srData *livekit.RTCPSenderReportState, tsOffset uint64, timeSinceReceive, timeSinceFirst, samplesDuration time.Duration, samplesDiff int64) []any {
|
||||
return []any{
|
||||
"startTime", time.Unix(0, r.startTime),
|
||||
"nowTime", time.Unix(0, now),
|
||||
"before", time.Unix(0, r.firstTime),
|
||||
"after", time.Unix(0, firstTime),
|
||||
"adjustment", time.Duration(adjustment),
|
||||
"firstTimeAdjustment", r.firstTimeAdjustment,
|
||||
"extNowTS", extNowTS,
|
||||
"extStartTS", extStartTS,
|
||||
"srData", WrappedRTCPSenderReportStateLogger{srData},
|
||||
"tsOffset", tsOffset,
|
||||
"timeSinceReceive", timeSinceReceive,
|
||||
"timeSinceFirst", timeSinceFirst,
|
||||
"samplesDiff", samplesDiff,
|
||||
"samplesDuration", samplesDuration,
|
||||
}
|
||||
}
|
||||
|
||||
func (r *rtpStatsBase) getPacketsSeenMinusPadding(extStartSN, extHighestSN uint64) uint64 {
|
||||
packetsSeen := r.getPacketsSeen(extStartSN, extHighestSN)
|
||||
if r.packetsPadding > packetsSeen {
|
||||
@@ -447,9 +476,13 @@ func (r *rtpStatsBase) deltaInfo(
|
||||
snapshotID uint32,
|
||||
extStartSN uint64,
|
||||
extHighestSN uint64,
|
||||
) (deltaInfo *RTPDeltaInfo, err error, loggingFields []interface{}) {
|
||||
then, now := r.getAndResetSnapshot(snapshotID, extStartSN, extHighestSN)
|
||||
if now == nil || then == nil {
|
||||
) (deltaInfo *RTPDeltaInfo, loggingFields []any, err error) {
|
||||
if r.clockRate == 0 {
|
||||
return
|
||||
}
|
||||
|
||||
then, now, ok := r.getAndResetSnapshot(snapshotID, extStartSN, extHighestSN)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
|
||||
@@ -461,7 +494,7 @@ func (r *rtpStatsBase) deltaInfo(
|
||||
packetsExpected = 0
|
||||
}
|
||||
if packetsExpected > cNumSequenceNumbers {
|
||||
loggingFields = []interface{}{
|
||||
loggingFields = []any{
|
||||
"snapshotID", snapshotID,
|
||||
"snapshotNow", now,
|
||||
"snapshotThen", then,
|
||||
@@ -487,7 +520,7 @@ func (r *rtpStatsBase) deltaInfo(
|
||||
// padding packets delta could be higher than expected due to out-of-order padding packets
|
||||
packetsPadding := now.packetsPadding - then.packetsPadding
|
||||
if packetsExpected < packetsPadding {
|
||||
loggingFields = []interface{}{
|
||||
loggingFields = []any{
|
||||
"snapshotID", snapshotID,
|
||||
"snapshotNow", now,
|
||||
"snapshotThen", then,
|
||||
@@ -518,7 +551,7 @@ func (r *rtpStatsBase) deltaInfo(
|
||||
PacketsOutOfOrder: uint32(now.packetsOutOfOrder - then.packetsOutOfOrder),
|
||||
Frames: now.frames - then.frames,
|
||||
RttMax: then.maxRtt,
|
||||
JitterMax: then.maxJitter / float64(r.params.ClockRate) * 1e6,
|
||||
JitterMax: then.maxJitter / float64(r.clockRate) * 1e6,
|
||||
Nacks: now.nacks - then.nacks,
|
||||
Plis: now.plis - then.plis,
|
||||
Firs: now.firs - then.firs,
|
||||
@@ -614,8 +647,8 @@ func (r *rtpStatsBase) toProto(
|
||||
p.KeyFrames = r.keyFrames
|
||||
p.LastKeyFrame = timestamppb.New(r.lastKeyFrame)
|
||||
|
||||
p.JitterCurrent = jitter / float64(r.params.ClockRate) * 1e6
|
||||
p.JitterMax = maxJitter / float64(r.params.ClockRate) * 1e6
|
||||
p.JitterCurrent = jitter / float64(r.clockRate) * 1e6
|
||||
p.JitterMax = maxJitter / float64(r.clockRate) * 1e6
|
||||
|
||||
p.Firs = r.firs
|
||||
p.LastFir = timestamppb.New(r.lastFir)
|
||||
@@ -635,9 +668,9 @@ func (r *rtpStatsBase) updateJitter(ets uint64, packetTime int64) float64 {
|
||||
// p1f1 -> p1f2 -> p2f1
|
||||
// In this case, p2f1 (packet 2, frame 1) will still be used in jitter calculation
|
||||
// although it is the second packet of a frame because of out-of-order receival.
|
||||
if r.lastJitterExtTimestamp != ets {
|
||||
if r.lastJitterExtTimestamp != ets && r.rtpConverter != nil {
|
||||
timeSinceFirst := packetTime - r.firstTime
|
||||
packetTimeRTP := uint64(timeSinceFirst * int64(r.params.ClockRate) / 1e9)
|
||||
packetTimeRTP := r.rtpConverter.ToRTPExt(time.Duration(timeSinceFirst))
|
||||
transit := packetTimeRTP - ets
|
||||
|
||||
if r.lastTransit != 0 {
|
||||
@@ -661,9 +694,9 @@ func (r *rtpStatsBase) updateJitter(ets uint64, packetTime int64) float64 {
|
||||
return r.jitter
|
||||
}
|
||||
|
||||
func (r *rtpStatsBase) getAndResetSnapshot(snapshotID uint32, extStartSN uint64, extHighestSN uint64) (*snapshot, *snapshot) {
|
||||
if !r.initialized {
|
||||
return nil, nil
|
||||
func (r *rtpStatsBase) getAndResetSnapshot(snapshotID uint32, extStartSN uint64, extHighestSN uint64) (snapshot, snapshot, bool) {
|
||||
if !r.initialized || snapshotID < cFirstSnapshotID {
|
||||
return snapshot{}, snapshot{}, false
|
||||
}
|
||||
|
||||
idx := snapshotID - cFirstSnapshotID
|
||||
@@ -676,7 +709,7 @@ func (r *rtpStatsBase) getAndResetSnapshot(snapshotID uint32, extStartSN uint64,
|
||||
// snapshot now
|
||||
now := r.getSnapshot(mono.UnixNano(), extHighestSN+1)
|
||||
r.snapshots[idx] = now
|
||||
return &then, &now
|
||||
return then, now, true
|
||||
}
|
||||
|
||||
func (r *rtpStatsBase) getDrift(extStartTS, extHighestTS uint64) (
|
||||
@@ -686,21 +719,20 @@ func (r *rtpStatsBase) getDrift(extStartTS, extHighestTS uint64) (
|
||||
rebasedReportDrift *livekit.RTPDrift,
|
||||
) {
|
||||
if r.firstTime != 0 {
|
||||
elapsed := r.highestTime - r.firstTime
|
||||
elapsed := time.Duration(r.highestTime - r.firstTime)
|
||||
rtpClockTicks := extHighestTS - extStartTS
|
||||
driftSamples := int64(rtpClockTicks - uint64(elapsed*int64(r.params.ClockRate)/1e9))
|
||||
driftSamples := int64(rtpClockTicks - r.rtpConverter.ToRTPExt(elapsed))
|
||||
if elapsed > 0 {
|
||||
elapsedSeconds := time.Duration(elapsed).Seconds()
|
||||
packetDrift = &livekit.RTPDrift{
|
||||
StartTime: timestamppb.New(time.Unix(0, r.firstTime)),
|
||||
EndTime: timestamppb.New(time.Unix(0, r.highestTime)),
|
||||
Duration: elapsedSeconds,
|
||||
Duration: elapsed.Seconds(),
|
||||
StartTimestamp: extStartTS,
|
||||
EndTimestamp: extHighestTS,
|
||||
RtpClockTicks: rtpClockTicks,
|
||||
DriftSamples: driftSamples,
|
||||
DriftMs: (float64(driftSamples) * 1000) / float64(r.params.ClockRate),
|
||||
ClockRate: float64(rtpClockTicks) / elapsedSeconds,
|
||||
DriftMs: (float64(driftSamples) * 1000) / float64(r.clockRate),
|
||||
ClockRate: float64(rtpClockTicks) / elapsed.Seconds(),
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -708,25 +740,27 @@ func (r *rtpStatsBase) getDrift(extStartTS, extHighestTS uint64) (
|
||||
if r.srFirst != nil && r.srNewest != nil && r.srFirst.RtpTimestamp != r.srNewest.RtpTimestamp {
|
||||
rtpClockTicks := r.srNewest.RtpTimestampExt - r.srFirst.RtpTimestampExt
|
||||
|
||||
elapsed := mediatransportutil.NtpTime(r.srNewest.NtpTimestamp).Time().Sub(mediatransportutil.NtpTime(r.srFirst.NtpTimestamp).Time())
|
||||
srFirstTime := mediatransportutil.NtpTime(r.srFirst.NtpTimestamp).Time()
|
||||
srNewestTime := mediatransportutil.NtpTime(r.srNewest.NtpTimestamp).Time()
|
||||
elapsed := srNewestTime.Sub(srFirstTime)
|
||||
if elapsed.Seconds() > 0.0 {
|
||||
driftSamples := int64(rtpClockTicks - uint64(elapsed.Nanoseconds()*int64(r.params.ClockRate)/1e9))
|
||||
driftSamples := int64(rtpClockTicks - r.rtpConverter.ToRTPExt(elapsed))
|
||||
ntpReportDrift = &livekit.RTPDrift{
|
||||
StartTime: timestamppb.New(mediatransportutil.NtpTime(r.srFirst.NtpTimestamp).Time()),
|
||||
EndTime: timestamppb.New(mediatransportutil.NtpTime(r.srNewest.NtpTimestamp).Time()),
|
||||
StartTime: timestamppb.New(srFirstTime),
|
||||
EndTime: timestamppb.New(srNewestTime),
|
||||
Duration: elapsed.Seconds(),
|
||||
StartTimestamp: r.srFirst.RtpTimestampExt,
|
||||
EndTimestamp: r.srNewest.RtpTimestampExt,
|
||||
RtpClockTicks: rtpClockTicks,
|
||||
DriftSamples: driftSamples,
|
||||
DriftMs: (float64(driftSamples) * 1000) / float64(r.params.ClockRate),
|
||||
DriftMs: (float64(driftSamples) * 1000) / float64(r.clockRate),
|
||||
ClockRate: float64(rtpClockTicks) / elapsed.Seconds(),
|
||||
}
|
||||
}
|
||||
|
||||
elapsed = time.Duration(r.srNewest.At - r.srFirst.At)
|
||||
if elapsed.Seconds() > 0.0 {
|
||||
driftSamples := int64(rtpClockTicks - uint64(elapsed.Nanoseconds()*int64(r.params.ClockRate)/1e9))
|
||||
driftSamples := int64(rtpClockTicks - r.rtpConverter.ToRTPExt(elapsed))
|
||||
receivedReportDrift = &livekit.RTPDrift{
|
||||
StartTime: timestamppb.New(time.Unix(0, r.srFirst.At)),
|
||||
EndTime: timestamppb.New(time.Unix(0, r.srNewest.At)),
|
||||
@@ -735,14 +769,14 @@ func (r *rtpStatsBase) getDrift(extStartTS, extHighestTS uint64) (
|
||||
EndTimestamp: r.srNewest.RtpTimestampExt,
|
||||
RtpClockTicks: rtpClockTicks,
|
||||
DriftSamples: driftSamples,
|
||||
DriftMs: (float64(driftSamples) * 1000) / float64(r.params.ClockRate),
|
||||
DriftMs: (float64(driftSamples) * 1000) / float64(r.clockRate),
|
||||
ClockRate: float64(rtpClockTicks) / elapsed.Seconds(),
|
||||
}
|
||||
}
|
||||
|
||||
elapsed = time.Duration(r.srNewest.AtAdjusted - r.srFirst.AtAdjusted)
|
||||
if elapsed.Seconds() > 0.0 {
|
||||
driftSamples := int64(rtpClockTicks - uint64(elapsed.Nanoseconds()*int64(r.params.ClockRate)/1e9))
|
||||
driftSamples := int64(rtpClockTicks - r.rtpConverter.ToRTPExt(elapsed))
|
||||
rebasedReportDrift = &livekit.RTPDrift{
|
||||
StartTime: timestamppb.New(time.Unix(0, r.srFirst.AtAdjusted)),
|
||||
EndTime: timestamppb.New(time.Unix(0, r.srNewest.AtAdjusted)),
|
||||
@@ -751,7 +785,7 @@ func (r *rtpStatsBase) getDrift(extStartTS, extHighestTS uint64) (
|
||||
EndTimestamp: r.srNewest.RtpTimestampExt,
|
||||
RtpClockTicks: rtpClockTicks,
|
||||
DriftSamples: driftSamples,
|
||||
DriftMs: (float64(driftSamples) * 1000) / float64(r.params.ClockRate),
|
||||
DriftMs: (float64(driftSamples) * 1000) / float64(r.clockRate),
|
||||
ClockRate: float64(rtpClockTicks) / elapsed.Seconds(),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -17,6 +17,7 @@ package rtpstats
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
@@ -95,14 +96,13 @@ func (s *snapshotLite) MarshalLogObject(e zapcore.ObjectEncoder) error {
|
||||
// ------------------------------------------------------------------
|
||||
|
||||
type RTPStatsParams struct {
|
||||
ClockRate uint32
|
||||
IsRTX bool
|
||||
Logger logger.Logger
|
||||
IsRTX bool
|
||||
}
|
||||
|
||||
type rtpStatsBaseLite struct {
|
||||
params RTPStatsParams
|
||||
logger logger.Logger
|
||||
params RTPStatsParams
|
||||
clockRate uint32
|
||||
logger logger.Logger
|
||||
|
||||
lock sync.RWMutex
|
||||
|
||||
@@ -134,7 +134,7 @@ type rtpStatsBaseLite struct {
|
||||
func newRTPStatsBaseLite(params RTPStatsParams) *rtpStatsBaseLite {
|
||||
return &rtpStatsBaseLite{
|
||||
params: params,
|
||||
logger: params.Logger,
|
||||
logger: logger.GetLogger(),
|
||||
nextSnapshotLiteID: cFirstSnapshotID,
|
||||
snapshotLites: make([]snapshotLite, 2),
|
||||
}
|
||||
@@ -172,7 +172,21 @@ func (r *rtpStatsBaseLite) seed(from *rtpStatsBaseLite) bool {
|
||||
return true
|
||||
}
|
||||
|
||||
func (r *rtpStatsBaseLite) SetClockRate(clockRate uint32) {
|
||||
r.lock.Lock()
|
||||
defer r.lock.Unlock()
|
||||
|
||||
r.setClockRateLocked(clockRate)
|
||||
}
|
||||
|
||||
func (r *rtpStatsBaseLite) setClockRateLocked(clockRate uint32) {
|
||||
r.clockRate = clockRate
|
||||
}
|
||||
|
||||
func (r *rtpStatsBaseLite) SetLogger(logger logger.Logger) {
|
||||
r.lock.Lock()
|
||||
defer r.lock.Unlock()
|
||||
|
||||
r.logger = logger
|
||||
}
|
||||
|
||||
@@ -305,9 +319,9 @@ func (r *rtpStatsBaseLite) deltaInfoLite(
|
||||
snapshotLiteID uint32,
|
||||
extStartSN uint64,
|
||||
extHighestSN uint64,
|
||||
) (deltaInfoLite *RTPDeltaInfoLite, err error, loggingFields []interface{}) {
|
||||
then, now := r.getAndResetSnapshotLite(snapshotLiteID, extStartSN, extHighestSN)
|
||||
if now == nil || then == nil {
|
||||
) (deltaInfoLite *RTPDeltaInfoLite, loggingFields []any, err error) {
|
||||
then, now, ok := r.getAndResetSnapshotLite(snapshotLiteID, extStartSN, extHighestSN)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
|
||||
@@ -319,7 +333,7 @@ func (r *rtpStatsBaseLite) deltaInfoLite(
|
||||
packetsExpected = 0
|
||||
}
|
||||
if packetsExpected > cNumSequenceNumbers {
|
||||
loggingFields = []interface{}{
|
||||
loggingFields = []any{
|
||||
"snapshotLiteID", snapshotLiteID,
|
||||
"snapshotLiteNow", now,
|
||||
"snapshotLiteThen", then,
|
||||
@@ -342,7 +356,7 @@ func (r *rtpStatsBaseLite) deltaInfoLite(
|
||||
packetsLost = 0
|
||||
}
|
||||
if packetsLost > packetsExpected {
|
||||
loggingFields = []interface{}{
|
||||
loggingFields = []any{
|
||||
"snapshotLiteID", snapshotLiteID,
|
||||
"snapshotLiteNow", now,
|
||||
"snapshotLiteThen", then,
|
||||
@@ -366,7 +380,7 @@ func (r *rtpStatsBaseLite) deltaInfoLite(
|
||||
}
|
||||
|
||||
func (r *rtpStatsBaseLite) marshalLogObject(e zapcore.ObjectEncoder, packetsExpected, packetsSeenMinusPadding uint64) (float64, error) {
|
||||
if r == nil || !r.initialized {
|
||||
if r == nil || !r.initialized || r.clockRate == 0 {
|
||||
return 0, errors.New("not initialized")
|
||||
}
|
||||
|
||||
@@ -399,25 +413,18 @@ func (r *rtpStatsBaseLite) marshalLogObject(e zapcore.ObjectEncoder, packetsExpe
|
||||
e.AddFloat32("packetLostPercentage", float32(r.packetsLost)/float32(packetsExpected)*100.0)
|
||||
}
|
||||
|
||||
hasLoss := false
|
||||
first := true
|
||||
str := "["
|
||||
var sb strings.Builder
|
||||
for burst, count := range r.gapHistogram {
|
||||
if count == 0 {
|
||||
continue
|
||||
}
|
||||
|
||||
hasLoss = true
|
||||
|
||||
if !first {
|
||||
str += ", "
|
||||
if sb.Len() > 0 {
|
||||
sb.WriteString(", ")
|
||||
}
|
||||
first = false
|
||||
str += fmt.Sprintf("%d:%d", burst+1, count)
|
||||
fmt.Fprintf(&sb, "%d:%d", burst+1, count)
|
||||
}
|
||||
str += "]"
|
||||
if hasLoss {
|
||||
e.AddString("gapHistogram", str)
|
||||
if sb.Len() > 0 {
|
||||
e.AddString("gapHistogram", "["+sb.String()+"]")
|
||||
}
|
||||
|
||||
e.AddUint32("nacks", r.nacks)
|
||||
@@ -431,7 +438,7 @@ func (r *rtpStatsBaseLite) marshalLogObject(e zapcore.ObjectEncoder, packetsExpe
|
||||
}
|
||||
|
||||
func (r *rtpStatsBaseLite) toProto(packetsExpected, packetsSeenMinusPadding, packetsLost uint64) *livekit.RTPStats {
|
||||
if r.startTime == 0 {
|
||||
if r.startTime == 0 || r.clockRate == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -473,33 +480,22 @@ func (r *rtpStatsBaseLite) toProto(packetsExpected, packetsSeenMinusPadding, pac
|
||||
LastPli: timestamppb.New(time.Unix(0, r.lastPli)),
|
||||
}
|
||||
|
||||
gapsPresent := false
|
||||
for i := 0; i < len(r.gapHistogram); i++ {
|
||||
for i := range len(r.gapHistogram) {
|
||||
if r.gapHistogram[i] == 0 {
|
||||
continue
|
||||
}
|
||||
|
||||
gapsPresent = true
|
||||
break
|
||||
}
|
||||
|
||||
if gapsPresent {
|
||||
p.GapHistogram = make(map[int32]uint32, len(r.gapHistogram))
|
||||
for i := 0; i < len(r.gapHistogram); i++ {
|
||||
if r.gapHistogram[i] == 0 {
|
||||
continue
|
||||
}
|
||||
|
||||
p.GapHistogram[int32(i+1)] = r.gapHistogram[i]
|
||||
if p.GapHistogram == nil {
|
||||
p.GapHistogram = make(map[int32]uint32, len(r.gapHistogram))
|
||||
}
|
||||
p.GapHistogram[int32(i+1)] = r.gapHistogram[i]
|
||||
}
|
||||
|
||||
return p
|
||||
}
|
||||
|
||||
func (r *rtpStatsBaseLite) getAndResetSnapshotLite(snapshotLiteID uint32, extStartSN uint64, extHighestSN uint64) (*snapshotLite, *snapshotLite) {
|
||||
func (r *rtpStatsBaseLite) getAndResetSnapshotLite(snapshotLiteID uint32, extStartSN uint64, extHighestSN uint64) (snapshotLite, snapshotLite, bool) {
|
||||
if !r.initialized {
|
||||
return nil, nil
|
||||
return snapshotLite{}, snapshotLite{}, false
|
||||
}
|
||||
|
||||
idx := snapshotLiteID - cFirstSnapshotID
|
||||
@@ -512,7 +508,7 @@ func (r *rtpStatsBaseLite) getAndResetSnapshotLite(snapshotLiteID uint32, extSta
|
||||
// snapshot now
|
||||
now := r.getSnapshotLite(mono.UnixNano(), extHighestSN+1)
|
||||
r.snapshotLites[idx] = now
|
||||
return &then, &now
|
||||
return then, now, true
|
||||
}
|
||||
|
||||
func (r *rtpStatsBaseLite) updateGapHistogram(gap int) {
|
||||
@@ -554,4 +550,8 @@ func getPacketsExpected(extStartSN, extHighestSN uint64) uint64 {
|
||||
return extHighestSN - extStartSN + 1
|
||||
}
|
||||
|
||||
func shouldLog(count int) bool {
|
||||
return count < 20 || count%200 == 0
|
||||
}
|
||||
|
||||
// ----------------------------------
|
||||
|
||||
@@ -20,10 +20,12 @@ import (
|
||||
"time"
|
||||
|
||||
"github.com/pion/rtcp"
|
||||
"go.uber.org/zap"
|
||||
"go.uber.org/zap/zapcore"
|
||||
|
||||
"github.com/livekit/livekit-server/pkg/sfu/utils"
|
||||
"github.com/livekit/mediatransportutil"
|
||||
"github.com/livekit/mediatransportutil/pkg/latency"
|
||||
"github.com/livekit/mediatransportutil/pkg/utils"
|
||||
"github.com/livekit/protocol/livekit"
|
||||
"github.com/livekit/protocol/logger"
|
||||
protoutils "github.com/livekit/protocol/utils"
|
||||
@@ -37,12 +39,53 @@ const (
|
||||
cReportSlack = float64(60.0)
|
||||
|
||||
cTSJumpTooHighFactor = float64(1.5)
|
||||
|
||||
restartThreshold = 5
|
||||
)
|
||||
|
||||
// ---------------------------------------------------------------------
|
||||
|
||||
type RTPFlowUnhandledReason int
|
||||
|
||||
const (
|
||||
RTPFlowUnhandledReasonNone RTPFlowUnhandledReason = iota
|
||||
RTPFlowUnhandledReasonEnded
|
||||
RTPFlowUnhandledReasonUnconfigured
|
||||
RTPFlowUnhandledReasonPaddingOnly
|
||||
RTPFlowUnhandledReasonPreStartTimestamp
|
||||
RTPFlowUnhandledReasonOldTimestamp
|
||||
RTPFlowUnhandledReasonPreStartSequenceNumber
|
||||
RTPFlowUnhandledReasonOldSequenceNumber
|
||||
RTPFlowUnhandledReasonRestart
|
||||
)
|
||||
|
||||
func (r RTPFlowUnhandledReason) String() string {
|
||||
switch r {
|
||||
case RTPFlowUnhandledReasonNone:
|
||||
return "NONE"
|
||||
case RTPFlowUnhandledReasonEnded:
|
||||
return "ENDED"
|
||||
case RTPFlowUnhandledReasonUnconfigured:
|
||||
return "UNCONFIGURED"
|
||||
case RTPFlowUnhandledReasonPaddingOnly:
|
||||
return "PADDING_ONLY"
|
||||
case RTPFlowUnhandledReasonPreStartTimestamp:
|
||||
return "PRE_START_TIMESTAMP"
|
||||
case RTPFlowUnhandledReasonOldTimestamp:
|
||||
return "OLD_TIMESTAMP"
|
||||
case RTPFlowUnhandledReasonPreStartSequenceNumber:
|
||||
return "PRE_START_SEQUENCE_NUMBER"
|
||||
case RTPFlowUnhandledReasonOldSequenceNumber:
|
||||
return "OLD_SEQUENCE_NUMBER"
|
||||
case RTPFlowUnhandledReasonRestart:
|
||||
return "RESTART"
|
||||
default:
|
||||
return fmt.Sprintf("UNKNOWN: %d", int(r))
|
||||
}
|
||||
}
|
||||
|
||||
type RTPFlowState struct {
|
||||
IsNotHandled bool
|
||||
UnhandledReason RTPFlowUnhandledReason
|
||||
|
||||
LossStartInclusive uint64
|
||||
LossEndExclusive uint64
|
||||
@@ -59,7 +102,7 @@ func (r *RTPFlowState) MarshalLogObject(e zapcore.ObjectEncoder) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
e.AddBool("IsNotHandled", r.IsNotHandled)
|
||||
e.AddString("UnhandledReason", r.UnhandledReason.String())
|
||||
e.AddUint64("LossStartInclusive", r.LossStartInclusive)
|
||||
e.AddUint64("LossEndExclusive", r.LossEndExclusive)
|
||||
e.AddBool("IsDuplicate", r.IsDuplicate)
|
||||
@@ -71,6 +114,62 @@ func (r *RTPFlowState) MarshalLogObject(e zapcore.ObjectEncoder) error {
|
||||
|
||||
// ---------------------------------------------------------------------
|
||||
|
||||
type packet struct {
|
||||
sequenceNumber uint16
|
||||
timestamp uint32
|
||||
}
|
||||
|
||||
func (p packet) MarshalLogObject(e zapcore.ObjectEncoder) error {
|
||||
e.AddUint16("sequenceNumber", p.sequenceNumber)
|
||||
e.AddUint32("timestamp", p.timestamp)
|
||||
return nil
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------
|
||||
|
||||
type receiverUpdateLoggingFields struct {
|
||||
packetTime int64
|
||||
sequenceNumber uint16
|
||||
timestamp uint32
|
||||
marker bool
|
||||
hdrSize int
|
||||
payloadSize int
|
||||
paddingSize int
|
||||
resSN utils.WrapAroundUpdateResult[uint64]
|
||||
gapSN int64
|
||||
resTS utils.WrapAroundUpdateResult[uint64]
|
||||
gapTS int64
|
||||
snRolloverCount int
|
||||
expectedTSJump int64
|
||||
tsRolloverCount int
|
||||
timeSinceHighest int64
|
||||
rtpStats *RTPStatsReceiver
|
||||
}
|
||||
|
||||
func (rulf *receiverUpdateLoggingFields) MarshalLogObject(e zapcore.ObjectEncoder) error {
|
||||
if rulf != nil {
|
||||
e.AddObject("resSN", &rulf.resSN)
|
||||
e.AddInt64("gapSN", rulf.gapSN)
|
||||
e.AddObject("resTS", &rulf.resTS)
|
||||
e.AddInt64("gapTS", rulf.gapTS)
|
||||
e.AddInt("snRolloverCount", rulf.snRolloverCount)
|
||||
e.AddInt64("expectedTSJump", rulf.expectedTSJump)
|
||||
e.AddInt("tsRolloverCount", rulf.tsRolloverCount)
|
||||
e.AddTime("packetTime", time.Unix(0, rulf.packetTime))
|
||||
e.AddDuration("timeSinceHighest", time.Duration(rulf.timeSinceHighest))
|
||||
e.AddUint16("sequenceNumber", rulf.sequenceNumber)
|
||||
e.AddUint32("timestamp", rulf.timestamp)
|
||||
e.AddBool("marker", rulf.marker)
|
||||
e.AddInt("hdrSize", rulf.hdrSize)
|
||||
e.AddInt("payloadSize", rulf.payloadSize)
|
||||
e.AddInt("paddingSize", rulf.paddingSize)
|
||||
e.AddObject("rtpStats", lockedRTPStatsReceiverLogEncoder{rulf.rtpStats})
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------
|
||||
|
||||
type RTPStatsReceiver struct {
|
||||
*rtpStatsBase
|
||||
|
||||
@@ -81,27 +180,40 @@ type RTPStatsReceiver struct {
|
||||
|
||||
history *protoutils.Bitmap[uint64]
|
||||
|
||||
propagationDelayEstimator *utils.OWDEstimator
|
||||
propagationDelayEstimator *latency.OWDEstimator
|
||||
|
||||
clockSkewCount int
|
||||
clockSkewMediaPathCount int
|
||||
outOfOrderSenderReportCount int
|
||||
largeJumpCount int
|
||||
largeJumpNegativeCount int
|
||||
timeReversedCount int
|
||||
|
||||
packetsDroppedPreStartTimestamp int
|
||||
packetsDroppedOldTimestamp int
|
||||
packetsDroppedPreStartSequenceNumber int
|
||||
packetsDroppedOldSequenceNumber int
|
||||
|
||||
restartPacketsBuf [restartThreshold]packet
|
||||
restartPacketsN int
|
||||
}
|
||||
|
||||
func NewRTPStatsReceiver(params RTPStatsParams) *RTPStatsReceiver {
|
||||
return &RTPStatsReceiver{
|
||||
rtpStatsBase: newRTPStatsBase(params),
|
||||
sequenceNumber: utils.NewWrapAround[uint16, uint64](utils.WrapAroundParams{IsRestartAllowed: false}),
|
||||
tsRolloverThreshold: (1 << 31) * 1e9 / int64(params.ClockRate),
|
||||
timestamp: utils.NewWrapAround[uint32, uint64](utils.WrapAroundParams{IsRestartAllowed: false}),
|
||||
history: protoutils.NewBitmap[uint64](cHistorySize),
|
||||
propagationDelayEstimator: utils.NewOWDEstimator(utils.OWDEstimatorParamsDefault),
|
||||
propagationDelayEstimator: latency.NewOWDEstimator(latency.OWDEstimatorParamsDefault),
|
||||
}
|
||||
}
|
||||
|
||||
func (r *RTPStatsReceiver) SetClockRate(clockRate uint32) {
|
||||
r.lock.Lock()
|
||||
defer r.lock.Unlock()
|
||||
|
||||
r.tsRolloverThreshold = (1 << 31) * 1e9 / int64(clockRate)
|
||||
r.rtpStatsBase.setClockRateLocked(clockRate)
|
||||
}
|
||||
|
||||
func (r *RTPStatsReceiver) NewSnapshotId() uint32 {
|
||||
r.lock.Lock()
|
||||
defer r.lock.Unlock()
|
||||
@@ -115,17 +227,19 @@ func (r *RTPStatsReceiver) getTSRolloverCount(diffNano int64, ts uint32) int {
|
||||
return -1
|
||||
}
|
||||
|
||||
excess := (diffNano - r.tsRolloverThreshold*2) * int64(r.params.ClockRate) / 1e9
|
||||
roc := excess / (1 << 32)
|
||||
if roc < 0 {
|
||||
roc = 0
|
||||
}
|
||||
excess := (diffNano - r.tsRolloverThreshold*2) * int64(r.clockRate) / 1e9
|
||||
roc := max(excess/(1<<32), 0)
|
||||
if r.timestamp.GetHighest() > ts {
|
||||
roc++
|
||||
}
|
||||
return int(roc)
|
||||
}
|
||||
|
||||
func (r *RTPStatsReceiver) undoUpdatesLocked(resSN utils.WrapAroundUpdateResult[uint64], resTS utils.WrapAroundUpdateResult[uint64]) {
|
||||
r.sequenceNumber.UndoUpdate(resSN)
|
||||
r.timestamp.UndoUpdate(resTS)
|
||||
}
|
||||
|
||||
func (r *RTPStatsReceiver) Update(
|
||||
packetTime int64,
|
||||
sequenceNumber uint16,
|
||||
@@ -139,7 +253,11 @@ func (r *RTPStatsReceiver) Update(
|
||||
defer r.lock.Unlock()
|
||||
|
||||
if r.endTime != 0 {
|
||||
flowState.IsNotHandled = true
|
||||
flowState.UnhandledReason = RTPFlowUnhandledReasonEnded
|
||||
return
|
||||
}
|
||||
if r.clockRate == 0 {
|
||||
flowState.UnhandledReason = RTPFlowUnhandledReasonUnconfigured
|
||||
return
|
||||
}
|
||||
|
||||
@@ -152,31 +270,10 @@ func (r *RTPStatsReceiver) Update(
|
||||
var tsRolloverCount int
|
||||
var snRolloverCount int
|
||||
|
||||
logger := func() logger.UnlikelyLogger {
|
||||
return r.logger.WithUnlikelyValues(
|
||||
"resSN", resSN,
|
||||
"gapSN", gapSN,
|
||||
"resTS", resTS,
|
||||
"gapTS", gapTS,
|
||||
"snRolloverCount", snRolloverCount,
|
||||
"expectedTSJump", expectedTSJump,
|
||||
"tsRolloverCount", tsRolloverCount,
|
||||
"packetTime", time.Unix(0, packetTime),
|
||||
"timeSinceHighest", time.Duration(timeSinceHighest),
|
||||
"sequenceNumber", sequenceNumber,
|
||||
"timestamp", timestamp,
|
||||
"marker", marker,
|
||||
"hdrSize", hdrSize,
|
||||
"payloadSize", payloadSize,
|
||||
"paddingSize", paddingSize,
|
||||
"rtpStats", lockedRTPStatsReceiverLogEncoder{r},
|
||||
)
|
||||
}
|
||||
|
||||
if !r.initialized {
|
||||
if payloadSize == 0 {
|
||||
// do not start on a padding only packet
|
||||
flowState.IsNotHandled = true
|
||||
flowState.UnhandledReason = RTPFlowUnhandledReasonPaddingOnly
|
||||
return
|
||||
}
|
||||
|
||||
@@ -201,57 +298,164 @@ func (r *RTPStatsReceiver) Update(
|
||||
)
|
||||
} else {
|
||||
resSN = r.sequenceNumber.Update(sequenceNumber)
|
||||
if resSN.IsUnhandled {
|
||||
flowState.IsNotHandled = true
|
||||
return
|
||||
}
|
||||
gapSN = int64(resSN.ExtendedVal - resSN.PreExtendedHighest)
|
||||
|
||||
timeSinceHighest = packetTime - r.highestTime
|
||||
expectedTSJump = int64(r.rtpConverter.ToRTPExt(time.Duration(timeSinceHighest)))
|
||||
tsRolloverCount = r.getTSRolloverCount(timeSinceHighest, timestamp)
|
||||
if tsRolloverCount >= 0 {
|
||||
logger().Warnw("potential time stamp roll over", nil)
|
||||
}
|
||||
resTS = r.timestamp.Rollover(timestamp, tsRolloverCount)
|
||||
gapTS = int64(resTS.ExtendedVal - resTS.PreExtendedHighest)
|
||||
if resTS.IsUnhandled {
|
||||
flowState.IsNotHandled = true
|
||||
r.undoUpdatesLocked(resSN, resTS)
|
||||
|
||||
r.packetsDroppedPreStartTimestamp++
|
||||
rulf := &receiverUpdateLoggingFields{
|
||||
packetTime: packetTime,
|
||||
sequenceNumber: sequenceNumber,
|
||||
timestamp: timestamp,
|
||||
marker: marker,
|
||||
hdrSize: hdrSize,
|
||||
payloadSize: payloadSize,
|
||||
paddingSize: paddingSize,
|
||||
resSN: resSN,
|
||||
gapSN: gapSN,
|
||||
resTS: resTS,
|
||||
gapTS: gapTS,
|
||||
snRolloverCount: snRolloverCount,
|
||||
expectedTSJump: expectedTSJump,
|
||||
tsRolloverCount: tsRolloverCount,
|
||||
timeSinceHighest: timeSinceHighest,
|
||||
rtpStats: r,
|
||||
}
|
||||
if shouldLog(r.packetsDroppedPreStartTimestamp) {
|
||||
r.logger.Warnw("dropping packet, pre-start timestamp", nil, zap.Inline(rulf))
|
||||
}
|
||||
|
||||
if r.maybeRestart(sequenceNumber, timestamp, payloadSize) {
|
||||
r.logger.Infow("potential restart", zap.Inline(rulf))
|
||||
r.resetRestart()
|
||||
flowState.UnhandledReason = RTPFlowUnhandledReasonRestart
|
||||
} else {
|
||||
flowState.UnhandledReason = RTPFlowUnhandledReasonPreStartTimestamp
|
||||
}
|
||||
return
|
||||
}
|
||||
gapTS = int64(resTS.ExtendedVal - resTS.PreExtendedHighest)
|
||||
|
||||
// it is possible to receive old packets in two different scenarios
|
||||
// as it is not possible to detect how far to roll back, ignore old packets
|
||||
//
|
||||
// Case 1:
|
||||
// Very old time stamp, happens under the following conditions
|
||||
// - resume after long mute, big time stamp jump
|
||||
// - an out of order packet from before the mute arrives (unsure what causes this
|
||||
// very old packet to be trasmitted from remote), causing time stamp to jump back
|
||||
// to before mute, but it appears like it has rolled over.
|
||||
// Use a threshold against expected to ignore these.
|
||||
if gapSN < 0 && gapTS > 0 {
|
||||
expectedTSJump = timeSinceHighest * int64(r.params.ClockRate) / 1e9
|
||||
if gapTS > int64(float64(expectedTSJump)*cTSJumpTooHighFactor) {
|
||||
r.sequenceNumber.UndoUpdate(resSN)
|
||||
r.timestamp.UndoUpdate(resTS)
|
||||
logger().Warnw("dropping old packet, timestamp", nil)
|
||||
flowState.IsNotHandled = true
|
||||
if tsRolloverCount >= 0 && payloadSize > 0 {
|
||||
rulf := &receiverUpdateLoggingFields{
|
||||
packetTime: packetTime,
|
||||
sequenceNumber: sequenceNumber,
|
||||
timestamp: timestamp,
|
||||
marker: marker,
|
||||
hdrSize: hdrSize,
|
||||
payloadSize: payloadSize,
|
||||
paddingSize: paddingSize,
|
||||
resSN: resSN,
|
||||
gapSN: gapSN,
|
||||
resTS: resTS,
|
||||
gapTS: gapTS,
|
||||
snRolloverCount: snRolloverCount,
|
||||
expectedTSJump: expectedTSJump,
|
||||
tsRolloverCount: tsRolloverCount,
|
||||
timeSinceHighest: timeSinceHighest,
|
||||
rtpStats: r,
|
||||
}
|
||||
r.logger.Warnw("potential time stamp roll over", nil, zap.Inline(rulf))
|
||||
}
|
||||
|
||||
if !resSN.IsUnhandled {
|
||||
// it is possible to receive old packets in two different scenarios
|
||||
// as it is not possible to detect how far to roll back, ignore old packets
|
||||
//
|
||||
// Case 1:
|
||||
// Very old time stamp, happens under the following conditions
|
||||
// - resume after long mute, this casues big time stamp jump ahead for the packets
|
||||
// after unmute
|
||||
// - an out of order packet from before the mute arrives (unsure what causes this
|
||||
// very old packet to be transmitted from remote), causing time stamp to jump back
|
||||
// to before mute, but it appears like it has rolled over.
|
||||
// Use a threshold against expected to ignore these.
|
||||
if gapSN < 0 && gapTS > 0 {
|
||||
if gapTS > int64(float64(expectedTSJump)*cTSJumpTooHighFactor) {
|
||||
r.undoUpdatesLocked(resSN, resTS)
|
||||
|
||||
r.packetsDroppedOldTimestamp++
|
||||
rulf := &receiverUpdateLoggingFields{
|
||||
packetTime: packetTime,
|
||||
sequenceNumber: sequenceNumber,
|
||||
timestamp: timestamp,
|
||||
marker: marker,
|
||||
hdrSize: hdrSize,
|
||||
payloadSize: payloadSize,
|
||||
paddingSize: paddingSize,
|
||||
resSN: resSN,
|
||||
gapSN: gapSN,
|
||||
resTS: resTS,
|
||||
gapTS: gapTS,
|
||||
snRolloverCount: snRolloverCount,
|
||||
expectedTSJump: expectedTSJump,
|
||||
tsRolloverCount: tsRolloverCount,
|
||||
timeSinceHighest: timeSinceHighest,
|
||||
rtpStats: r,
|
||||
}
|
||||
if shouldLog(r.packetsDroppedOldTimestamp) {
|
||||
r.logger.Warnw("dropping packet, old timestamp", nil, zap.Inline(rulf))
|
||||
}
|
||||
|
||||
if r.maybeRestart(sequenceNumber, timestamp, payloadSize) {
|
||||
r.logger.Infow("potential restart", zap.Inline(rulf))
|
||||
r.resetRestart()
|
||||
flowState.UnhandledReason = RTPFlowUnhandledReasonRestart
|
||||
} else {
|
||||
flowState.UnhandledReason = RTPFlowUnhandledReasonOldTimestamp
|
||||
}
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
// Case 2:
|
||||
// Sequence number looks like it is moving forward, but it is actually a very old packet.
|
||||
if gapTS < 0 && gapSN > 0 {
|
||||
r.undoUpdatesLocked(resSN, resTS)
|
||||
|
||||
expectedTSJump = int64(r.rtpConverter.ToRTPExt(time.Duration(timeSinceHighest)))
|
||||
|
||||
r.packetsDroppedOldSequenceNumber++
|
||||
rulf := &receiverUpdateLoggingFields{
|
||||
packetTime: packetTime,
|
||||
sequenceNumber: sequenceNumber,
|
||||
timestamp: timestamp,
|
||||
marker: marker,
|
||||
hdrSize: hdrSize,
|
||||
payloadSize: payloadSize,
|
||||
paddingSize: paddingSize,
|
||||
resSN: resSN,
|
||||
gapSN: gapSN,
|
||||
resTS: resTS,
|
||||
gapTS: gapTS,
|
||||
snRolloverCount: snRolloverCount,
|
||||
expectedTSJump: expectedTSJump,
|
||||
tsRolloverCount: tsRolloverCount,
|
||||
timeSinceHighest: timeSinceHighest,
|
||||
rtpStats: r,
|
||||
}
|
||||
if shouldLog(r.packetsDroppedOldSequenceNumber) {
|
||||
r.logger.Warnw("dropping packet, old sequence number", nil, zap.Inline(rulf))
|
||||
}
|
||||
|
||||
if r.maybeRestart(sequenceNumber, timestamp, payloadSize) {
|
||||
r.logger.Infow("potential restart", zap.Inline(rulf))
|
||||
r.resetRestart()
|
||||
flowState.UnhandledReason = RTPFlowUnhandledReasonRestart
|
||||
} else {
|
||||
flowState.UnhandledReason = RTPFlowUnhandledReasonOldSequenceNumber
|
||||
}
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
// Case 2:
|
||||
// Sequence number looks like it is moving forward, but it is actually a very old packet.
|
||||
if gapTS < 0 && gapSN > 0 {
|
||||
r.sequenceNumber.UndoUpdate(resSN)
|
||||
r.timestamp.UndoUpdate(resTS)
|
||||
logger().Warnw("dropping old packet, sequence number", nil)
|
||||
flowState.IsNotHandled = true
|
||||
return
|
||||
}
|
||||
|
||||
// it is possible that sequence number has rolled over too
|
||||
if gapSN < 0 && gapTS > 0 && payloadSize > 0 {
|
||||
if (gapSN < 0 || gapSN > (1<<15)) && gapTS > 0 && payloadSize > 0 {
|
||||
// not possible to know how many cycles of sequence number roll over could have happened,
|
||||
// ensure that it at least does not go backwards
|
||||
snRolloverCount = 0
|
||||
@@ -259,12 +463,63 @@ func (r *RTPStatsReceiver) Update(
|
||||
snRolloverCount = 1
|
||||
}
|
||||
resSN = r.sequenceNumber.Rollover(sequenceNumber, snRolloverCount)
|
||||
if resSN.IsUnhandled {
|
||||
flowState.IsNotHandled = true
|
||||
return
|
||||
if !resSN.IsUnhandled {
|
||||
rulf := &receiverUpdateLoggingFields{
|
||||
packetTime: packetTime,
|
||||
sequenceNumber: sequenceNumber,
|
||||
timestamp: timestamp,
|
||||
marker: marker,
|
||||
hdrSize: hdrSize,
|
||||
payloadSize: payloadSize,
|
||||
paddingSize: paddingSize,
|
||||
resSN: resSN,
|
||||
gapSN: gapSN,
|
||||
resTS: resTS,
|
||||
gapTS: gapTS,
|
||||
snRolloverCount: snRolloverCount,
|
||||
expectedTSJump: expectedTSJump,
|
||||
tsRolloverCount: tsRolloverCount,
|
||||
timeSinceHighest: timeSinceHighest,
|
||||
rtpStats: r,
|
||||
}
|
||||
r.logger.Warnw("forcing sequence number rollover", nil, zap.Inline(rulf))
|
||||
}
|
||||
}
|
||||
|
||||
if resSN.IsUnhandled {
|
||||
r.undoUpdatesLocked(resSN, resTS)
|
||||
|
||||
r.packetsDroppedPreStartSequenceNumber++
|
||||
rulf := &receiverUpdateLoggingFields{
|
||||
packetTime: packetTime,
|
||||
sequenceNumber: sequenceNumber,
|
||||
timestamp: timestamp,
|
||||
marker: marker,
|
||||
hdrSize: hdrSize,
|
||||
payloadSize: payloadSize,
|
||||
paddingSize: paddingSize,
|
||||
resSN: resSN,
|
||||
gapSN: gapSN,
|
||||
resTS: resTS,
|
||||
gapTS: gapTS,
|
||||
snRolloverCount: snRolloverCount,
|
||||
expectedTSJump: expectedTSJump,
|
||||
tsRolloverCount: tsRolloverCount,
|
||||
timeSinceHighest: timeSinceHighest,
|
||||
rtpStats: r,
|
||||
}
|
||||
if shouldLog(r.packetsDroppedPreStartSequenceNumber) {
|
||||
r.logger.Warnw("dropping packet, pre-start sequence number", nil, zap.Inline(rulf))
|
||||
}
|
||||
|
||||
logger().Warnw("forcing sequence number rollover", nil)
|
||||
if r.maybeRestart(sequenceNumber, timestamp, payloadSize) {
|
||||
r.logger.Infow("potential restart", zap.Inline(rulf))
|
||||
r.resetRestart()
|
||||
flowState.UnhandledReason = RTPFlowUnhandledReasonRestart
|
||||
} else {
|
||||
flowState.UnhandledReason = RTPFlowUnhandledReasonPreStartSequenceNumber
|
||||
}
|
||||
return
|
||||
}
|
||||
}
|
||||
gapSN = int64(resSN.ExtendedVal - resSN.PreExtendedHighest)
|
||||
@@ -287,32 +542,31 @@ func (r *RTPStatsReceiver) Update(
|
||||
}
|
||||
|
||||
flowState.IsOutOfOrder = true
|
||||
|
||||
if !flowState.IsDuplicate && -gapSN >= cSequenceNumberLargeJumpThreshold {
|
||||
r.largeJumpNegativeCount++
|
||||
if (r.largeJumpNegativeCount-1)%100 == 0 {
|
||||
logger().Warnw(
|
||||
"large sequence number gap negative", nil,
|
||||
"count", r.largeJumpNegativeCount,
|
||||
)
|
||||
}
|
||||
}
|
||||
} else { // in-order
|
||||
if gapSN >= cSequenceNumberLargeJumpThreshold {
|
||||
r.largeJumpCount++
|
||||
if (r.largeJumpCount-1)%100 == 0 {
|
||||
logger().Warnw(
|
||||
"large sequence number gap", nil,
|
||||
"count", r.largeJumpCount,
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
if resTS.ExtendedVal < resTS.PreExtendedHighest {
|
||||
if resTS.ExtendedVal < resTS.PreExtendedHighest && r.bytes > 0 {
|
||||
r.timeReversedCount++
|
||||
if (r.timeReversedCount-1)%100 == 0 {
|
||||
logger().Warnw(
|
||||
if shouldLog(r.timeReversedCount) {
|
||||
rulf := &receiverUpdateLoggingFields{
|
||||
packetTime: packetTime,
|
||||
sequenceNumber: sequenceNumber,
|
||||
timestamp: timestamp,
|
||||
marker: marker,
|
||||
hdrSize: hdrSize,
|
||||
payloadSize: payloadSize,
|
||||
paddingSize: paddingSize,
|
||||
resSN: resSN,
|
||||
gapSN: gapSN,
|
||||
resTS: resTS,
|
||||
gapTS: gapTS,
|
||||
snRolloverCount: snRolloverCount,
|
||||
expectedTSJump: expectedTSJump,
|
||||
tsRolloverCount: tsRolloverCount,
|
||||
timeSinceHighest: timeSinceHighest,
|
||||
rtpStats: r,
|
||||
}
|
||||
r.logger.Warnw(
|
||||
"time reversed", nil,
|
||||
zap.Inline(rulf),
|
||||
"count", r.timeReversedCount,
|
||||
)
|
||||
}
|
||||
@@ -355,6 +609,7 @@ func (r *RTPStatsReceiver) Update(
|
||||
r.updateJitter(resTS.ExtendedVal, packetTime)
|
||||
}
|
||||
}
|
||||
r.resetRestart()
|
||||
return
|
||||
}
|
||||
|
||||
@@ -363,10 +618,12 @@ func (r *RTPStatsReceiver) getExtendedSenderReport(srData *livekit.RTCPSenderRep
|
||||
if r.srNewest != nil {
|
||||
// use time since last sender report to ensure long gaps where the time stamp might
|
||||
// jump more than half the range
|
||||
timeSinceLastReport := mediatransportutil.NtpTime(srData.NtpTimestamp).Time().Sub(mediatransportutil.NtpTime(r.srNewest.NtpTimestamp).Time())
|
||||
expectedRTPTimestampExt := r.srNewest.RtpTimestampExt + uint64(timeSinceLastReport.Nanoseconds()*int64(r.params.ClockRate)/1e9)
|
||||
lbound := expectedRTPTimestampExt - uint64(cReportSlack*float64(r.params.ClockRate))
|
||||
ubound := expectedRTPTimestampExt + uint64(cReportSlack*float64(r.params.ClockRate))
|
||||
srTime := mediatransportutil.NtpTime(srData.NtpTimestamp).Time()
|
||||
srNewestTime := mediatransportutil.NtpTime(r.srNewest.NtpTimestamp).Time()
|
||||
timeSinceLastReport := srTime.Sub(srNewestTime)
|
||||
expectedRTPTimestampExt := r.srNewest.RtpTimestampExt + r.rtpConverter.ToRTPExt(timeSinceLastReport)
|
||||
lbound := expectedRTPTimestampExt - uint64(cReportSlack*float64(r.clockRate))
|
||||
ubound := expectedRTPTimestampExt + uint64(cReportSlack*float64(r.clockRate))
|
||||
isInRange := (srData.RtpTimestamp-uint32(lbound) < (1 << 31)) && (uint32(ubound)-srData.RtpTimestamp < (1 << 31))
|
||||
if isInRange {
|
||||
lbTSCycles := lbound & 0xFFFF_FFFF_0000_0000
|
||||
@@ -409,7 +666,7 @@ func (r *RTPStatsReceiver) checkOutOfOrderSenderReport(srData *livekit.RTCPSende
|
||||
// Or it could be due bad report generation.
|
||||
// In any case, ignore out-of-order reports.
|
||||
r.outOfOrderSenderReportCount++
|
||||
if (r.outOfOrderSenderReportCount-1)%10 == 0 {
|
||||
if shouldLog(r.outOfOrderSenderReportCount) {
|
||||
r.logger.Infow(
|
||||
"received sender report, out-of-order, skipping",
|
||||
"current", WrappedRTCPSenderReportStateLogger{srData},
|
||||
@@ -428,18 +685,22 @@ func (r *RTPStatsReceiver) checkRTPClockSkewForSenderReport(srData *livekit.RTCP
|
||||
return
|
||||
}
|
||||
|
||||
timeSinceLast := mediatransportutil.NtpTime(srData.NtpTimestamp).Time().Sub(mediatransportutil.NtpTime(r.srNewest.NtpTimestamp).Time()).Seconds()
|
||||
srTime := mediatransportutil.NtpTime(srData.NtpTimestamp).Time()
|
||||
srNewestTime := mediatransportutil.NtpTime(r.srNewest.NtpTimestamp).Time()
|
||||
srFirstTime := mediatransportutil.NtpTime(r.srFirst.NtpTimestamp).Time()
|
||||
|
||||
timeSinceLast := srTime.Sub(srNewestTime).Seconds()
|
||||
rtpDiffSinceLast := srData.RtpTimestampExt - r.srNewest.RtpTimestampExt
|
||||
calculatedClockRateFromLast := float64(rtpDiffSinceLast) / timeSinceLast
|
||||
|
||||
timeSinceFirst := mediatransportutil.NtpTime(srData.NtpTimestamp).Time().Sub(mediatransportutil.NtpTime(r.srFirst.NtpTimestamp).Time()).Seconds()
|
||||
timeSinceFirst := srTime.Sub(srFirstTime).Seconds()
|
||||
rtpDiffSinceFirst := srData.RtpTimestampExt - r.srFirst.RtpTimestampExt
|
||||
calculatedClockRateFromFirst := float64(rtpDiffSinceFirst) / timeSinceFirst
|
||||
|
||||
if (timeSinceLast > 0.2 && math.Abs(float64(r.params.ClockRate)-calculatedClockRateFromLast) > 0.2*float64(r.params.ClockRate)) ||
|
||||
(timeSinceFirst > 0.2 && math.Abs(float64(r.params.ClockRate)-calculatedClockRateFromFirst) > 0.2*float64(r.params.ClockRate)) {
|
||||
if (timeSinceLast > 0.2 && math.Abs(float64(r.clockRate)-calculatedClockRateFromLast) > 0.2*float64(r.clockRate)) ||
|
||||
(timeSinceFirst > 0.2 && math.Abs(float64(r.clockRate)-calculatedClockRateFromFirst) > 0.2*float64(r.clockRate)) {
|
||||
r.clockSkewCount++
|
||||
if (r.clockSkewCount-1)%100 == 0 {
|
||||
if shouldLog(r.clockSkewCount) {
|
||||
r.logger.Infow(
|
||||
"received sender report, clock skew",
|
||||
"current", WrappedRTCPSenderReportStateLogger{srData},
|
||||
@@ -463,20 +724,20 @@ func (r *RTPStatsReceiver) checkRTPClockSkewAgainstMediaPathForSenderReport(srDa
|
||||
|
||||
nowNano := mono.UnixNano()
|
||||
timeSinceSR := time.Duration(nowNano - srData.AtAdjusted)
|
||||
extNowTSSR := srData.RtpTimestampExt + uint64(timeSinceSR.Nanoseconds()*int64(r.params.ClockRate)/1e9)
|
||||
extNowTSSR := srData.RtpTimestampExt + r.rtpConverter.ToRTPExt(timeSinceSR)
|
||||
|
||||
timeSinceHighest := time.Duration(nowNano - r.highestTime)
|
||||
extNowTSHighest := r.timestamp.GetExtendedHighest() + uint64(timeSinceHighest.Nanoseconds()*int64(r.params.ClockRate)/1e9)
|
||||
extNowTSHighest := r.timestamp.GetExtendedHighest() + r.rtpConverter.ToRTPExt(timeSinceHighest)
|
||||
diffHighest := extNowTSSR - extNowTSHighest
|
||||
|
||||
timeSinceFirst := time.Duration(nowNano - r.firstTime)
|
||||
extNowTSFirst := r.timestamp.GetExtendedStart() + uint64(timeSinceFirst.Nanoseconds()*int64(r.params.ClockRate)/1e9)
|
||||
extNowTSFirst := r.timestamp.GetExtendedStart() + r.rtpConverter.ToRTPExt(timeSinceFirst)
|
||||
diffFirst := extNowTSSR - extNowTSFirst
|
||||
|
||||
// is it more than 5 seconds off?
|
||||
if uint32(math.Abs(float64(int64(diffHighest)))) > 5*r.params.ClockRate || uint32(math.Abs(float64(int64(diffFirst)))) > 5*r.params.ClockRate {
|
||||
if uint32(math.Abs(float64(int64(diffHighest)))) > 5*r.clockRate || uint32(math.Abs(float64(int64(diffFirst)))) > 5*r.clockRate {
|
||||
r.clockSkewMediaPathCount++
|
||||
if (r.clockSkewMediaPathCount-1)%100 == 0 {
|
||||
if shouldLog(r.clockSkewMediaPathCount) {
|
||||
r.logger.Infow(
|
||||
"received sender report, clock skew against media path",
|
||||
"current", WrappedRTCPSenderReportStateLogger{srData},
|
||||
@@ -496,7 +757,8 @@ func (r *RTPStatsReceiver) checkRTPClockSkewAgainstMediaPathForSenderReport(srDa
|
||||
}
|
||||
|
||||
func (r *RTPStatsReceiver) updatePropagationDelayAndRecordSenderReport(srData *livekit.RTCPSenderReportState) {
|
||||
senderClockTime := mediatransportutil.NtpTime(srData.NtpTimestamp).Time().UnixNano()
|
||||
srTime := mediatransportutil.NtpTime(srData.NtpTimestamp).Time()
|
||||
senderClockTime := srTime.UnixNano()
|
||||
estimatedPropagationDelay, stepChange := r.propagationDelayEstimator.Update(senderClockTime, srData.At)
|
||||
if stepChange {
|
||||
r.logger.Debugw(
|
||||
@@ -518,7 +780,7 @@ func (r *RTPStatsReceiver) SetRtcpSenderReportData(srData *livekit.RTCPSenderRep
|
||||
r.lock.Lock()
|
||||
defer r.lock.Unlock()
|
||||
|
||||
if srData == nil || !r.initialized {
|
||||
if srData == nil || !r.initialized || r.clockRate == 0 {
|
||||
return false
|
||||
}
|
||||
|
||||
@@ -542,9 +804,11 @@ func (r *RTPStatsReceiver) SetRtcpSenderReportData(srData *livekit.RTCPSenderRep
|
||||
r.updatePropagationDelayAndRecordSenderReport(srDataExt)
|
||||
r.checkRTPClockSkewAgainstMediaPathForSenderReport(srDataExt)
|
||||
|
||||
if err, loggingFields := r.maybeAdjustFirstPacketTime(r.srNewest, 0, r.timestamp.GetExtendedStart()); err != nil {
|
||||
adjustment, loggingFields, err := r.maybeAdjustFirstPacketTime(r.srNewest, 0, r.timestamp.GetExtendedStart())
|
||||
if err != nil {
|
||||
r.logger.Infow(err.Error(), append(loggingFields, "rtpStats", lockedRTPStatsReceiverLogEncoder{r})...)
|
||||
}
|
||||
r.propagationDelayEstimator.InitialAdjustment(adjustment)
|
||||
return true
|
||||
}
|
||||
|
||||
@@ -571,8 +835,8 @@ func (r *RTPStatsReceiver) GetRtcpReceptionReport(ssrc uint32, proxyFracLost uin
|
||||
defer r.lock.Unlock()
|
||||
|
||||
extHighestSN := r.sequenceNumber.GetExtendedHighest()
|
||||
then, now := r.getAndResetSnapshot(snapshotID, r.sequenceNumber.GetExtendedStart(), extHighestSN)
|
||||
if now == nil || then == nil {
|
||||
then, now, ok := r.getAndResetSnapshot(snapshotID, r.sequenceNumber.GetExtendedStart(), extHighestSN)
|
||||
if !ok {
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -594,15 +858,9 @@ func (r *RTPStatsReceiver) GetRtcpReceptionReport(ssrc uint32, proxyFracLost uin
|
||||
packetsLost = 0
|
||||
}
|
||||
lossRate := float32(packetsLost) / float32(packetsExpected)
|
||||
fracLost := uint8(lossRate * 256.0)
|
||||
if proxyFracLost > fracLost {
|
||||
fracLost = proxyFracLost
|
||||
}
|
||||
fracLost := max(proxyFracLost, uint8(lossRate*256.0))
|
||||
|
||||
totalLost := r.packetsLost
|
||||
if totalLost > 0xffffff { // 24-bits max
|
||||
totalLost = 0xffffff
|
||||
}
|
||||
totalLost := min(r.packetsLost, 0xffffff) // 24-bits max
|
||||
|
||||
lastSR := uint32(0)
|
||||
dlsr := uint32(0)
|
||||
@@ -629,7 +887,7 @@ func (r *RTPStatsReceiver) DeltaInfo(snapshotID uint32) *RTPDeltaInfo {
|
||||
r.lock.Lock()
|
||||
defer r.lock.Unlock()
|
||||
|
||||
deltaInfo, err, loggingFields := r.deltaInfo(
|
||||
deltaInfo, loggingFields, err := r.deltaInfo(
|
||||
snapshotID,
|
||||
r.sequenceNumber.GetExtendedStart(),
|
||||
r.sequenceNumber.GetExtendedHighest(),
|
||||
@@ -653,6 +911,10 @@ func (r *RTPStatsReceiver) MarshalLogObject(e zapcore.ObjectEncoder) error {
|
||||
}
|
||||
|
||||
func (r *RTPStatsReceiver) ToProto() *livekit.RTPStats {
|
||||
if r == nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
r.lock.RLock()
|
||||
defer r.lock.RUnlock()
|
||||
|
||||
@@ -696,6 +958,37 @@ func (r *RTPStatsReceiver) ExtendedHighestSequenceNumber() uint64 {
|
||||
return r.sequenceNumber.GetExtendedHighest()
|
||||
}
|
||||
|
||||
func (r *RTPStatsReceiver) maybeRestart(sn uint16, ts uint32, payloadSize int) bool {
|
||||
if payloadSize > 0 {
|
||||
if r.restartPacketsN < restartThreshold {
|
||||
r.restartPacketsBuf[r.restartPacketsN] = packet{sn, ts}
|
||||
r.restartPacketsN++
|
||||
} else {
|
||||
// keep last restartThreshold entries: shift left and append
|
||||
copy(r.restartPacketsBuf[:], r.restartPacketsBuf[1:])
|
||||
r.restartPacketsBuf[restartThreshold-1] = packet{sn, ts}
|
||||
}
|
||||
}
|
||||
if r.restartPacketsN < restartThreshold {
|
||||
return false
|
||||
}
|
||||
|
||||
// check for contiguous sequence numbers and equal or increasing timestamps
|
||||
for i := 1; i < r.restartPacketsN; i++ {
|
||||
p := &r.restartPacketsBuf[i]
|
||||
prev := &r.restartPacketsBuf[i-1]
|
||||
if p.sequenceNumber != prev.sequenceNumber+1 || (p.timestamp-prev.timestamp) > (1<<31) {
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
return true
|
||||
}
|
||||
|
||||
func (r *RTPStatsReceiver) resetRestart() {
|
||||
r.restartPacketsN = 0
|
||||
}
|
||||
|
||||
// ----------------------------------
|
||||
|
||||
type lockedRTPStatsReceiverLogEncoder struct {
|
||||
@@ -725,6 +1018,18 @@ func (r lockedRTPStatsReceiverLogEncoder) MarshalLogObject(e zapcore.ObjectEncod
|
||||
e.AddUint64("extHighestTS", extHighestTS)
|
||||
|
||||
e.AddObject("propagationDelayEstimator", r.propagationDelayEstimator)
|
||||
|
||||
e.AddInt("clockSkewCount", r.clockSkewCount)
|
||||
e.AddInt("clockSkewMediaPathCount", r.clockSkewMediaPathCount)
|
||||
e.AddInt("outOfOrderSenderReportCount", r.outOfOrderSenderReportCount)
|
||||
e.AddInt("timeReversedCount", r.timeReversedCount)
|
||||
|
||||
e.AddInt("packetsDroppedPreStartTimestamp", r.packetsDroppedPreStartTimestamp)
|
||||
e.AddInt("packetsDroppedOldTimestamp", r.packetsDroppedOldTimestamp)
|
||||
e.AddInt("packetsDroppedPreStartSequenceNumber", r.packetsDroppedPreStartSequenceNumber)
|
||||
e.AddInt("packetsDroppedOldSequenceNumber", r.packetsDroppedOldSequenceNumber)
|
||||
|
||||
e.AddArray("restartPackets", logger.ObjectSlice(r.restartPacketsBuf[:r.restartPacketsN]))
|
||||
return nil
|
||||
}
|
||||
|
||||
|
||||
@@ -17,7 +17,7 @@ package rtpstats
|
||||
import (
|
||||
"go.uber.org/zap/zapcore"
|
||||
|
||||
"github.com/livekit/livekit-server/pkg/sfu/utils"
|
||||
"github.com/livekit/mediatransportutil/pkg/utils"
|
||||
"github.com/livekit/protocol/livekit"
|
||||
"github.com/livekit/protocol/utils/mono"
|
||||
)
|
||||
@@ -102,7 +102,9 @@ func (r *RTPStatsReceiverLite) Update(packetTime int64, packetSize int, sequence
|
||||
gapSN := int64(resSN.ExtendedVal - resSN.PreExtendedHighest)
|
||||
if gapSN <= 0 { // duplicate OR out-of-order
|
||||
r.packetsOutOfOrder++ // counting duplicate as out-of-order
|
||||
r.packetsLost--
|
||||
if gapSN != 0 && r.packetsLost > 0 {
|
||||
r.packetsLost--
|
||||
}
|
||||
} else { // in-order
|
||||
r.updateGapHistogram(int(gapSN))
|
||||
r.packetsLost += uint64(gapSN - 1)
|
||||
@@ -119,7 +121,7 @@ func (r *RTPStatsReceiverLite) DeltaInfoLite(snapshotLiteID uint32) *RTPDeltaInf
|
||||
r.lock.Lock()
|
||||
defer r.lock.Unlock()
|
||||
|
||||
deltaInfoLite, err, loggingFields := r.deltaInfoLite(
|
||||
deltaInfoLite, loggingFields, err := r.deltaInfoLite(
|
||||
snapshotLiteID,
|
||||
r.sequenceNumber.GetExtendedStart(),
|
||||
r.sequenceNumber.GetExtendedHighest(),
|
||||
|
||||
+241
-135
@@ -21,12 +21,12 @@ import (
|
||||
"time"
|
||||
|
||||
"github.com/pion/rtcp"
|
||||
"go.uber.org/zap"
|
||||
"go.uber.org/zap/zapcore"
|
||||
"google.golang.org/protobuf/types/known/timestamppb"
|
||||
|
||||
"github.com/livekit/mediatransportutil"
|
||||
"github.com/livekit/protocol/livekit"
|
||||
"github.com/livekit/protocol/logger"
|
||||
"github.com/livekit/protocol/utils/mono"
|
||||
)
|
||||
|
||||
@@ -40,6 +40,9 @@ const (
|
||||
snInfoFlagOutOfOrder
|
||||
)
|
||||
|
||||
// max number of receiver reports kept per sender snapshot for logging
|
||||
const cMaxProcessedReceptionReports = 64
|
||||
|
||||
type snInfo struct {
|
||||
pktSize uint16
|
||||
hdrSize uint8
|
||||
@@ -103,7 +106,8 @@ type wrappedReceptionReportsLogger struct {
|
||||
}
|
||||
|
||||
func (w wrappedReceptionReportsLogger) MarshalLogObject(e zapcore.ObjectEncoder) error {
|
||||
for i, rr := range w.senderSnapshotReceiverView.processedReceptionReports {
|
||||
for i := 0; i < w.senderSnapshotReceiverView.processedReceptionReportsSize; i++ {
|
||||
rr := w.senderSnapshotReceiverView.processedReceptionReports[(w.senderSnapshotReceiverView.processedReceptionReportsHead+i)%cMaxProcessedReceptionReports]
|
||||
e.AddReflected(fmt.Sprintf("%d", i), rr)
|
||||
}
|
||||
|
||||
@@ -192,9 +196,13 @@ type senderSnapshotReceiverView struct {
|
||||
maxRtt uint32
|
||||
maxJitter float64
|
||||
|
||||
extLastRRSN uint64
|
||||
intervalStats intervalStats
|
||||
processedReceptionReports []rtcp.ReceptionReport
|
||||
extLastRRSN uint64
|
||||
intervalStats intervalStats
|
||||
|
||||
processedReceptionReports [cMaxProcessedReceptionReports]rtcp.ReceptionReport
|
||||
processedReceptionReportsHead int
|
||||
processedReceptionReportsSize int
|
||||
|
||||
metadataCacheOverflowCount int
|
||||
}
|
||||
|
||||
@@ -272,6 +280,70 @@ func (s *senderSnapshot) maybeUpdateMaxJitter(jitter float64) {
|
||||
|
||||
// -------------------------------------------------------------------
|
||||
|
||||
type senderUpdateLoggingFields struct {
|
||||
packetTime int64
|
||||
extSequenceNumber uint64
|
||||
extTimestamp uint64
|
||||
marker bool
|
||||
hdrSize int
|
||||
payloadSize int
|
||||
paddingSize int
|
||||
gapSN int64
|
||||
gapTS int64
|
||||
timeSinceHighest int64
|
||||
rtpStats *RTPStatsSender
|
||||
}
|
||||
|
||||
func (sulf *senderUpdateLoggingFields) MarshalLogObject(e zapcore.ObjectEncoder) error {
|
||||
if sulf != nil {
|
||||
e.AddUint64("currSN", sulf.extSequenceNumber)
|
||||
e.AddInt64("gapSN", sulf.gapSN)
|
||||
e.AddUint64("currTS", sulf.extTimestamp)
|
||||
e.AddInt64("gapTS", sulf.gapTS)
|
||||
e.AddTime("packetTime", time.Unix(0, sulf.packetTime))
|
||||
e.AddDuration("timeSinceHighest", time.Duration(sulf.timeSinceHighest))
|
||||
e.AddBool("marker", sulf.marker)
|
||||
e.AddInt("hdrSize", sulf.hdrSize)
|
||||
e.AddInt("payloadSize", sulf.payloadSize)
|
||||
e.AddInt("paddingSize", sulf.paddingSize)
|
||||
e.AddObject("rtpStats", lockedRTPStatsSenderLogEncoder{sulf.rtpStats})
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// -------------------------------------------------------------------
|
||||
|
||||
type senderReportLoggingFields struct {
|
||||
srData *livekit.RTCPSenderReportState
|
||||
publisherSRData *livekit.RTCPSenderReportState
|
||||
tsOffset uint64
|
||||
reportTime int64
|
||||
reportTimeAdjusted int64
|
||||
nowNano int64
|
||||
nowRTPExt uint64
|
||||
rtpStats *RTPStatsSender
|
||||
}
|
||||
|
||||
func (srlf *senderReportLoggingFields) MarshalLogObject(e zapcore.ObjectEncoder) error {
|
||||
if srlf != nil {
|
||||
e.AddObject("curr", WrappedRTCPSenderReportStateLogger{srlf.srData})
|
||||
e.AddObject("feed", WrappedRTCPSenderReportStateLogger{srlf.publisherSRData})
|
||||
e.AddUint64("tsOffset", srlf.tsOffset)
|
||||
e.AddTime("timeNow", time.Now())
|
||||
e.AddInt64("reportTime", srlf.reportTime)
|
||||
e.AddInt64("reportTimeAdjusted", srlf.reportTimeAdjusted)
|
||||
e.AddDuration("timeSinceHighest", time.Duration(srlf.nowNano-srlf.rtpStats.highestTime))
|
||||
e.AddDuration("timeSinceFirst", time.Duration(srlf.nowNano-srlf.rtpStats.firstTime))
|
||||
e.AddDuration("timeSincePublisherSRAdjusted", time.Duration(srlf.nowNano-srlf.publisherSRData.AtAdjusted))
|
||||
e.AddDuration("timeSincePublisherSR", time.Duration(srlf.nowNano-srlf.publisherSRData.At))
|
||||
e.AddUint64("nowRTPExt", srlf.nowRTPExt)
|
||||
e.AddObject("rtpStats", lockedRTPStatsSenderLogEncoder{srlf.rtpStats})
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// -------------------------------------------------------------------
|
||||
|
||||
type rttMarker struct {
|
||||
ntpTime mediatransportutil.NtpTime
|
||||
sentAt time.Time
|
||||
@@ -308,10 +380,8 @@ type RTPStatsSender struct {
|
||||
nextSenderSnapshotID uint32
|
||||
senderSnapshots []senderSnapshot
|
||||
|
||||
clockSkewCount int
|
||||
largeJumpNegativeCount int
|
||||
largeJumpCount int
|
||||
timeReversedCount int
|
||||
clockSkewCount int
|
||||
timeReversedCount int
|
||||
}
|
||||
|
||||
func NewRTPStatsSender(params RTPStatsParams, cacheSize int) *RTPStatsSender {
|
||||
@@ -399,7 +469,7 @@ func (r *RTPStatsSender) Update(
|
||||
r.lock.Lock()
|
||||
defer r.lock.Unlock()
|
||||
|
||||
if r.endTime != 0 {
|
||||
if r.endTime != 0 || r.clockRate == 0 {
|
||||
return
|
||||
}
|
||||
|
||||
@@ -444,21 +514,6 @@ func (r *RTPStatsSender) Update(
|
||||
pktSize := uint64(hdrSize + payloadSize + paddingSize)
|
||||
isDuplicate := false
|
||||
gapSN := int64(extSequenceNumber - r.extHighestSN)
|
||||
ulgr := func() logger.UnlikelyLogger {
|
||||
return r.logger.WithUnlikelyValues(
|
||||
"currSN", extSequenceNumber,
|
||||
"gapSN", gapSN,
|
||||
"currTS", extTimestamp,
|
||||
"gapTS", int64(extTimestamp-r.extHighestTS),
|
||||
"packetTime", time.Unix(0, packetTime),
|
||||
"timeSinceHighest", time.Duration(packetTime-r.highestTime),
|
||||
"marker", marker,
|
||||
"hdrSize", hdrSize,
|
||||
"payloadSize", payloadSize,
|
||||
"paddingSize", paddingSize,
|
||||
"rtpStats", lockedRTPStatsSenderLogEncoder{r},
|
||||
)
|
||||
}
|
||||
if gapSN <= 0 { // duplicate OR out-of-order
|
||||
if payloadSize == 0 && extSequenceNumber < r.extStartSN && !r.params.IsRTX {
|
||||
// do not start on a padding only packet
|
||||
@@ -479,8 +534,22 @@ func (r *RTPStatsSender) Update(
|
||||
r.senderSnapshots[i].maybeReinit(r.extStartSN, extSequenceNumber)
|
||||
}
|
||||
|
||||
ulgr().Infow(
|
||||
sulf := &senderUpdateLoggingFields{
|
||||
packetTime: packetTime,
|
||||
extSequenceNumber: extSequenceNumber,
|
||||
extTimestamp: extTimestamp,
|
||||
marker: marker,
|
||||
hdrSize: hdrSize,
|
||||
payloadSize: payloadSize,
|
||||
paddingSize: paddingSize,
|
||||
gapSN: gapSN,
|
||||
gapTS: int64(extTimestamp - r.extHighestTS),
|
||||
timeSinceHighest: packetTime - r.highestTime,
|
||||
rtpStats: r,
|
||||
}
|
||||
r.logger.Infow(
|
||||
"adjusting start sequence number",
|
||||
zap.Inline(sulf),
|
||||
"snAfter", extSequenceNumber,
|
||||
"tsAfter", extTimestamp,
|
||||
)
|
||||
@@ -500,32 +569,26 @@ func (r *RTPStatsSender) Update(
|
||||
r.packetsLost--
|
||||
r.setSnInfo(extSequenceNumber, r.extHighestSN, uint16(pktSize), uint8(hdrSize), uint16(payloadSize), marker, true)
|
||||
}
|
||||
|
||||
if !isDuplicate && -gapSN >= cSequenceNumberLargeJumpThreshold {
|
||||
r.largeJumpNegativeCount++
|
||||
if (r.largeJumpNegativeCount-1)%100 == 0 {
|
||||
ulgr().Warnw(
|
||||
"large sequence number gap negative", nil,
|
||||
"count", r.largeJumpNegativeCount,
|
||||
)
|
||||
}
|
||||
}
|
||||
} else { // in-order
|
||||
if gapSN >= cSequenceNumberLargeJumpThreshold {
|
||||
r.largeJumpCount++
|
||||
if (r.largeJumpCount-1)%100 == 0 {
|
||||
ulgr().Warnw(
|
||||
"large sequence number gap", nil,
|
||||
"count", r.largeJumpCount,
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
if extTimestamp < r.extHighestTS {
|
||||
r.timeReversedCount++
|
||||
if (r.timeReversedCount-1)%100 == 0 {
|
||||
ulgr().Warnw(
|
||||
if shouldLog(r.timeReversedCount) {
|
||||
sulf := &senderUpdateLoggingFields{
|
||||
packetTime: packetTime,
|
||||
extSequenceNumber: extSequenceNumber,
|
||||
extTimestamp: extTimestamp,
|
||||
marker: marker,
|
||||
hdrSize: hdrSize,
|
||||
payloadSize: payloadSize,
|
||||
paddingSize: paddingSize,
|
||||
gapSN: gapSN,
|
||||
gapTS: int64(extTimestamp - r.extHighestTS),
|
||||
timeSinceHighest: packetTime - r.highestTime,
|
||||
rtpStats: r,
|
||||
}
|
||||
r.logger.Warnw(
|
||||
"time reversed", nil,
|
||||
zap.Inline(sulf),
|
||||
"count", r.timeReversedCount,
|
||||
)
|
||||
}
|
||||
@@ -544,8 +607,22 @@ func (r *RTPStatsSender) Update(
|
||||
}
|
||||
|
||||
if extTimestamp < r.extStartTS {
|
||||
ulgr().Infow(
|
||||
sulf := &senderUpdateLoggingFields{
|
||||
packetTime: packetTime,
|
||||
extSequenceNumber: extSequenceNumber,
|
||||
extTimestamp: extTimestamp,
|
||||
marker: marker,
|
||||
hdrSize: hdrSize,
|
||||
payloadSize: payloadSize,
|
||||
paddingSize: paddingSize,
|
||||
gapSN: gapSN,
|
||||
gapTS: int64(extTimestamp - r.extHighestTS),
|
||||
timeSinceHighest: packetTime - r.highestTime,
|
||||
rtpStats: r,
|
||||
}
|
||||
r.logger.Infow(
|
||||
"adjusting start timestamp",
|
||||
zap.Inline(sulf),
|
||||
"snAfter", extSequenceNumber,
|
||||
"tsAfter", extTimestamp,
|
||||
)
|
||||
@@ -628,14 +705,24 @@ func (r *RTPStatsSender) UpdateFromReceiverReport(rr rtcp.ReceptionReport) (rtt
|
||||
r.lastRR = rr
|
||||
}()
|
||||
|
||||
timeSinceLastRR := func() time.Duration {
|
||||
if r.lastRRTime != 0 {
|
||||
return time.Duration(nowNano - r.lastRRTime)
|
||||
}
|
||||
return time.Duration(nowNano - r.startTime)
|
||||
timeSinceLastRR := time.Duration(nowNano - r.startTime)
|
||||
if r.lastRRTime != 0 {
|
||||
timeSinceLastRR = time.Duration(nowNano - r.lastRRTime)
|
||||
}
|
||||
|
||||
extReceivedRRSN := extHighestSNFromRR + (r.extStartSN & 0xFFFF_FFFF_FFFF_0000)
|
||||
if int64(r.extHighestSN-extReceivedRRSN) < 0 || int64(r.extHighestSN-extReceivedRRSN) > (1<<32) {
|
||||
r.logger.Infow(
|
||||
"receiver report runaway, dropping",
|
||||
"timeSinceLastRR", timeSinceLastRR,
|
||||
"receivedRR", rr,
|
||||
"extHighestSNFromRR", extHighestSNFromRR,
|
||||
"extReceivedRRSN", extReceivedRRSN,
|
||||
"rtpStats", lockedRTPStatsSenderLogEncoder{r},
|
||||
)
|
||||
return
|
||||
}
|
||||
|
||||
if r.extHighestSNFromRR != extHighestSNFromRR && int64(r.extHighestSN-extReceivedRRSN) >= (1<<16) {
|
||||
// there are cases where remote does not send RTCP Receiver Report for extended periods of time,
|
||||
// some times several minutes, in that interval the sequence number rolls over,
|
||||
@@ -645,7 +732,7 @@ func (r *RTPStatsSender) UpdateFromReceiverReport(rr rtcp.ReceptionReport) (rtt
|
||||
// should not roll over for long time, for e. g. it will approximately take 100 days at 500 pps).
|
||||
// So, there seems to be a remote reporter issue where the sequence number rollover is missed.
|
||||
//
|
||||
// catch up till diffrence between highest sent and highest received via receiver report is
|
||||
// catch up till difference between highest sent and highest received via receiver report is
|
||||
// less than full 16-bit range.
|
||||
//
|
||||
// in a different flavor, there are clients that do not report properly,
|
||||
@@ -654,7 +741,7 @@ func (r *RTPStatsSender) UpdateFromReceiverReport(rr rtcp.ReceptionReport) (rtt
|
||||
// RTCP RR does not change.
|
||||
r.logger.Infow(
|
||||
"receiver report missed rollover, adjusting",
|
||||
"timeSinceLastRR", timeSinceLastRR(),
|
||||
"timeSinceLastRR", timeSinceLastRR,
|
||||
"receivedRR", rr,
|
||||
"extHighestSNFromRR", extHighestSNFromRR,
|
||||
"extReceivedRRSN", extReceivedRRSN,
|
||||
@@ -667,7 +754,7 @@ func (r *RTPStatsSender) UpdateFromReceiverReport(rr rtcp.ReceptionReport) (rtt
|
||||
}
|
||||
r.logger.Infow(
|
||||
"receiver report missed rollover, adjusted",
|
||||
"timeSinceLastRR", timeSinceLastRR(),
|
||||
"timeSinceLastRR", timeSinceLastRR,
|
||||
"receivedRR", rr,
|
||||
"extHighestSNFromRR", extHighestSNFromRR,
|
||||
"extReceivedRRSN", extReceivedRRSN,
|
||||
@@ -679,7 +766,7 @@ func (r *RTPStatsSender) UpdateFromReceiverReport(rr rtcp.ReceptionReport) (rtt
|
||||
// if remote adjusts somehow, roll back alignment
|
||||
r.logger.Infow(
|
||||
"receiver report caught up rollover, adjusting",
|
||||
"timeSinceLastRR", timeSinceLastRR(),
|
||||
"timeSinceLastRR", timeSinceLastRR,
|
||||
"receivedRR", rr,
|
||||
"extHighestSNFromRR", extHighestSNFromRR,
|
||||
"extReceivedRRSN", extReceivedRRSN,
|
||||
@@ -692,7 +779,7 @@ func (r *RTPStatsSender) UpdateFromReceiverReport(rr rtcp.ReceptionReport) (rtt
|
||||
}
|
||||
r.logger.Infow(
|
||||
"receiver report caught up rollover, adjusted",
|
||||
"timeSinceLastRR", timeSinceLastRR(),
|
||||
"timeSinceLastRR", timeSinceLastRR,
|
||||
"receivedRR", rr,
|
||||
"extHighestSNFromRR", extHighestSNFromRR,
|
||||
"extReceivedRRSN", extReceivedRRSN,
|
||||
@@ -701,9 +788,9 @@ func (r *RTPStatsSender) UpdateFromReceiverReport(rr rtcp.ReceptionReport) (rtt
|
||||
}
|
||||
|
||||
if r.extHighestSNFromRR > extHighestSNFromRR {
|
||||
r.logger.Infow(
|
||||
r.logger.Debugw(
|
||||
"receiver report out-of-order, dropping",
|
||||
"timeSinceLastRR", timeSinceLastRR(),
|
||||
"timeSinceLastRR", timeSinceLastRR,
|
||||
"receivedRR", rr,
|
||||
"extHighestSNFromRR", extHighestSNFromRR,
|
||||
"extReceivedRRSN", extReceivedRRSN,
|
||||
@@ -768,7 +855,7 @@ func (r *RTPStatsSender) UpdateFromReceiverReport(rr rtcp.ReceptionReport) (rtt
|
||||
"metadata cache overflow",
|
||||
"senderSnapshotID", i+cFirstSnapshotID,
|
||||
"senderSnapshot", s,
|
||||
"timeSinceLastRR", timeSinceLastRR(),
|
||||
"timeSinceLastRR", timeSinceLastRR,
|
||||
"receivedRR", rr,
|
||||
"extReceivedRRSN", extReceivedRRSN,
|
||||
"packetsInInterval", extReceivedRRSN-s.receiverView.extLastRRSN,
|
||||
@@ -779,7 +866,16 @@ func (r *RTPStatsSender) UpdateFromReceiverReport(rr rtcp.ReceptionReport) (rtt
|
||||
}
|
||||
}
|
||||
s.receiverView.extLastRRSN = extReceivedRRSN
|
||||
s.receiverView.processedReceptionReports = append(s.receiverView.processedReceptionReports, rr)
|
||||
|
||||
s.receiverView.processedReceptionReports[s.receiverView.processedReceptionReportsHead] = rr
|
||||
s.receiverView.processedReceptionReportsHead++
|
||||
if s.receiverView.processedReceptionReportsHead >= cMaxProcessedReceptionReports {
|
||||
s.receiverView.processedReceptionReportsHead = 0
|
||||
}
|
||||
s.receiverView.processedReceptionReportsSize++
|
||||
if s.receiverView.processedReceptionReportsSize > cMaxProcessedReceptionReports {
|
||||
s.receiverView.processedReceptionReportsSize = cMaxProcessedReceptionReports
|
||||
}
|
||||
}
|
||||
|
||||
return
|
||||
@@ -796,11 +892,11 @@ func (r *RTPStatsSender) MaybeAdjustFirstPacketTime(publisherSRData *livekit.RTC
|
||||
r.lock.Lock()
|
||||
defer r.lock.Unlock()
|
||||
|
||||
if !r.initialized || publisherSRData == nil {
|
||||
if !r.initialized || publisherSRData == nil || r.clockRate == 0 {
|
||||
return
|
||||
}
|
||||
|
||||
if err, loggingFields := r.maybeAdjustFirstPacketTime(publisherSRData, tsOffset, r.extStartTS); err != nil {
|
||||
if _, loggingFields, err := r.maybeAdjustFirstPacketTime(publisherSRData, tsOffset, r.extStartTS); err != nil {
|
||||
r.logger.Infow(err.Error(), append(loggingFields, "rtpStats", lockedRTPStatsSenderLogEncoder{r})...)
|
||||
}
|
||||
}
|
||||
@@ -809,14 +905,13 @@ func (r *RTPStatsSender) GetExpectedRTPTimestamp(at time.Time) (expectedTSExt ui
|
||||
r.lock.RLock()
|
||||
defer r.lock.RUnlock()
|
||||
|
||||
if r.firstTime == 0 {
|
||||
if r.firstTime == 0 || r.clockRate == 0 {
|
||||
err = errors.New("uninitialized")
|
||||
return
|
||||
}
|
||||
|
||||
timeDiff := at.Sub(time.Unix(0, r.firstTime))
|
||||
expectedRTPDiff := timeDiff.Nanoseconds() * int64(r.params.ClockRate) / 1e9
|
||||
expectedTSExt = r.extStartTS + uint64(expectedRTPDiff)
|
||||
expectedTSExt = r.extStartTS + r.rtpConverter.ToRTPExt(timeDiff)
|
||||
return
|
||||
}
|
||||
|
||||
@@ -824,7 +919,7 @@ func (r *RTPStatsSender) GetRtcpSenderReport(ssrc uint32, publisherSRData *livek
|
||||
r.lock.Lock()
|
||||
defer r.lock.Unlock()
|
||||
|
||||
if !r.initialized || publisherSRData == nil {
|
||||
if !r.initialized || publisherSRData == nil || r.clockRate == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -834,19 +929,21 @@ func (r *RTPStatsSender) GetRtcpSenderReport(ssrc uint32, publisherSRData *livek
|
||||
nowNTP mediatransportutil.NtpTime
|
||||
nowRTPExt uint64
|
||||
)
|
||||
nowNano := mono.UnixNano()
|
||||
if passThrough {
|
||||
reportTime = publisherSRData.At
|
||||
reportTimeAdjusted = publisherSRData.AtAdjusted
|
||||
timeSincePublisherSR := time.Duration(nowNano - publisherSRData.At)
|
||||
reportTime = publisherSRData.At + timeSincePublisherSR.Nanoseconds()
|
||||
reportTimeAdjusted = publisherSRData.AtAdjusted + timeSincePublisherSR.Nanoseconds()
|
||||
|
||||
nowNTP = mediatransportutil.NtpTime(publisherSRData.NtpTimestamp)
|
||||
nowRTPExt = publisherSRData.RtpTimestampExt - tsOffset
|
||||
nowNTP = mediatransportutil.ToNtpTime(mediatransportutil.NtpTime(publisherSRData.NtpTimestamp).Time().Add(timeSincePublisherSR))
|
||||
nowRTPExt = publisherSRData.RtpTimestampExt - tsOffset + r.rtpConverter.ToRTPExt(timeSincePublisherSR)
|
||||
} else {
|
||||
timeSincePublisherSRAdjusted := time.Duration(mono.UnixNano() - publisherSRData.AtAdjusted)
|
||||
timeSincePublisherSRAdjusted := time.Duration(nowNano - publisherSRData.AtAdjusted)
|
||||
reportTimeAdjusted = publisherSRData.AtAdjusted + timeSincePublisherSRAdjusted.Nanoseconds()
|
||||
reportTime = reportTimeAdjusted
|
||||
|
||||
nowNTP = mediatransportutil.ToNtpTime(time.Unix(0, reportTime))
|
||||
nowRTPExt = publisherSRData.RtpTimestampExt - tsOffset + uint64(timeSincePublisherSRAdjusted.Nanoseconds()*int64(r.params.ClockRate)/1e9)
|
||||
nowRTPExt = publisherSRData.RtpTimestampExt - tsOffset + r.rtpConverter.ToRTPExt(timeSincePublisherSRAdjusted)
|
||||
}
|
||||
|
||||
packetCount := uint32(r.getPacketsSeenPlusDuplicates(r.extStartSN, r.extHighestSN))
|
||||
@@ -861,33 +958,28 @@ func (r *RTPStatsSender) GetRtcpSenderReport(ssrc uint32, publisherSRData *livek
|
||||
Octets: octetCount,
|
||||
}
|
||||
|
||||
ulgr := func() logger.UnlikelyLogger {
|
||||
nowNano := mono.UnixNano()
|
||||
return r.logger.WithUnlikelyValues(
|
||||
"curr", WrappedRTCPSenderReportStateLogger{srData},
|
||||
"feed", WrappedRTCPSenderReportStateLogger{publisherSRData},
|
||||
"tsOffset", tsOffset,
|
||||
"timeNow", mono.Now(),
|
||||
"reportTime", time.Unix(0, reportTime),
|
||||
"reportTimeAdjusted", time.Unix(0, reportTimeAdjusted),
|
||||
"timeSinceHighest", time.Duration(nowNano-r.highestTime),
|
||||
"timeSinceFirst", time.Duration(nowNano-r.firstTime),
|
||||
"timeSincePublisherSRAdjusted", time.Duration(nowNano-publisherSRData.AtAdjusted),
|
||||
"timeSincePublisherSR", time.Duration(nowNano-publisherSRData.At),
|
||||
"nowRTPExt", nowRTPExt,
|
||||
"rtpStats", lockedRTPStatsSenderLogEncoder{r},
|
||||
)
|
||||
}
|
||||
|
||||
if r.srNewest != nil && nowRTPExt >= r.srNewest.RtpTimestampExt {
|
||||
timeSinceLastReport := nowNTP.Time().Sub(mediatransportutil.NtpTime(r.srNewest.NtpTimestamp).Time())
|
||||
if r.srNewest != nil && nowRTPExt >= r.srNewest.RtpTimestampExt && r.clockRate != 0 {
|
||||
nowTime := nowNTP.Time()
|
||||
srNewestTime := mediatransportutil.NtpTime(r.srNewest.NtpTimestamp).Time()
|
||||
timeSinceLastReport := nowTime.Sub(srNewestTime)
|
||||
rtpDiffSinceLastReport := nowRTPExt - r.srNewest.RtpTimestampExt
|
||||
windowClockRate := float64(rtpDiffSinceLastReport) / timeSinceLastReport.Seconds()
|
||||
if timeSinceLastReport.Seconds() > 0.2 && math.Abs(float64(r.params.ClockRate)-windowClockRate) > 0.2*float64(r.params.ClockRate) {
|
||||
if timeSinceLastReport.Seconds() > 0.2 && math.Abs(float64(r.clockRate)-windowClockRate) > 0.2*float64(r.clockRate) {
|
||||
r.clockSkewCount++
|
||||
if (r.clockSkewCount-1)%100 == 0 {
|
||||
ulgr().Infow(
|
||||
if shouldLog(r.clockSkewCount) {
|
||||
srlf := &senderReportLoggingFields{
|
||||
srData: srData,
|
||||
publisherSRData: publisherSRData,
|
||||
tsOffset: tsOffset,
|
||||
reportTime: reportTime,
|
||||
reportTimeAdjusted: reportTimeAdjusted,
|
||||
nowNano: nowNano,
|
||||
nowRTPExt: nowRTPExt,
|
||||
rtpStats: r,
|
||||
}
|
||||
r.logger.Infow(
|
||||
"sending sender report, clock skew",
|
||||
zap.Inline(srlf),
|
||||
"timeSinceLastReport", timeSinceLastReport,
|
||||
"rtpDiffSinceLastReport", rtpDiffSinceLastReport,
|
||||
"windowClockRate", windowClockRate,
|
||||
@@ -900,7 +992,20 @@ func (r *RTPStatsSender) GetRtcpSenderReport(ssrc uint32, publisherSRData *livek
|
||||
if r.srNewest != nil && nowRTPExt < r.srNewest.RtpTimestampExt {
|
||||
// If report being generated is behind the last report, skip it.
|
||||
// Should not happen.
|
||||
ulgr().Infow("sending sender report, out-of-order, skipping")
|
||||
srlf := &senderReportLoggingFields{
|
||||
srData: srData,
|
||||
publisherSRData: publisherSRData,
|
||||
tsOffset: tsOffset,
|
||||
reportTime: reportTime,
|
||||
reportTimeAdjusted: reportTimeAdjusted,
|
||||
nowNano: nowNano,
|
||||
nowRTPExt: nowRTPExt,
|
||||
rtpStats: r,
|
||||
}
|
||||
r.logger.Infow(
|
||||
"sending sender report, out-of-order, skipping",
|
||||
zap.Inline(srlf),
|
||||
)
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -911,7 +1016,7 @@ func (r *RTPStatsSender) GetRtcpSenderReport(ssrc uint32, publisherSRData *livek
|
||||
|
||||
r.rttMarker = rttMarker{
|
||||
ntpTime: nowNTP,
|
||||
sentAt: mono.Now(),
|
||||
sentAt: time.Now(),
|
||||
}
|
||||
|
||||
return &rtcp.SenderReport{
|
||||
@@ -927,7 +1032,7 @@ func (r *RTPStatsSender) DeltaInfo(snapshotID uint32) *RTPDeltaInfo {
|
||||
r.lock.Lock()
|
||||
defer r.lock.Unlock()
|
||||
|
||||
deltaInfo, err, loggingFields := r.deltaInfo(
|
||||
deltaInfo, loggingFields, err := r.deltaInfo(
|
||||
snapshotID,
|
||||
r.extStartSN,
|
||||
r.extHighestSN,
|
||||
@@ -943,9 +1048,13 @@ func (r *RTPStatsSender) DeltaInfoSender(senderSnapshotID uint32) (*RTPDeltaInfo
|
||||
r.lock.Lock()
|
||||
defer r.lock.Unlock()
|
||||
|
||||
if r.clockRate == 0 {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
var deltaStatsSenderView *RTPDeltaInfo
|
||||
thenSenderView, nowSenderView := r.getAndResetSenderSnapshotWindow(senderSnapshotID)
|
||||
if thenSenderView != nil && nowSenderView != nil {
|
||||
thenSenderView, nowSenderView, ok := r.getAndResetSenderSnapshotWindow(senderSnapshotID)
|
||||
if ok {
|
||||
startTime := thenSenderView.startTime
|
||||
endTime := nowSenderView.startTime
|
||||
|
||||
@@ -979,7 +1088,7 @@ func (r *RTPStatsSender) DeltaInfoSender(senderSnapshotID uint32) (*RTPDeltaInfo
|
||||
packetsLostFeed = packetsExpected
|
||||
}
|
||||
|
||||
maxJitterTime := thenSenderView.maxJitterFeed / float64(r.params.ClockRate) * 1e6
|
||||
maxJitterTime := thenSenderView.maxJitterFeed / float64(r.clockRate) * 1e6
|
||||
|
||||
deltaStatsSenderView = &RTPDeltaInfo{
|
||||
StartTime: time.Unix(0, startTime),
|
||||
@@ -1007,8 +1116,8 @@ func (r *RTPStatsSender) DeltaInfoSender(senderSnapshotID uint32) (*RTPDeltaInfo
|
||||
|
||||
var deltaStatsReceiverView *RTPDeltaInfo
|
||||
if r.lastRRTime != 0 {
|
||||
thenReceiverView, nowReceiverView := r.getAndResetSenderSnapshotReceiverView(senderSnapshotID)
|
||||
if thenReceiverView != nil && nowReceiverView != nil {
|
||||
thenReceiverView, nowReceiverView, ok := r.getAndResetSenderSnapshotReceiverView(senderSnapshotID)
|
||||
if ok {
|
||||
startTime := thenReceiverView.startTime
|
||||
endTime := nowReceiverView.startTime
|
||||
|
||||
@@ -1048,12 +1157,7 @@ func (r *RTPStatsSender) DeltaInfoSender(senderSnapshotID uint32) (*RTPDeltaInfo
|
||||
packetsLost = packetsExpected
|
||||
}
|
||||
|
||||
// discount jitter from publisher side + internal processing
|
||||
maxJitter := thenReceiverView.maxJitter - thenReceiverView.maxJitterFeed
|
||||
if maxJitter < 0.0 {
|
||||
maxJitter = 0.0
|
||||
}
|
||||
maxJitterTime := maxJitter / float64(r.params.ClockRate) * 1e6
|
||||
maxJitterTime := thenReceiverView.maxJitter / float64(r.clockRate) * 1e6
|
||||
|
||||
deltaStatsReceiverView = &RTPDeltaInfo{
|
||||
StartTime: time.Unix(0, startTime),
|
||||
@@ -1117,9 +1221,9 @@ func (r *RTPStatsSender) ToProto() *livekit.RTPStats {
|
||||
return p
|
||||
}
|
||||
|
||||
func (r *RTPStatsSender) getAndResetSenderSnapshotWindow(senderSnapshotID uint32) (*senderSnapshotWindow, *senderSnapshotWindow) {
|
||||
if !r.initialized {
|
||||
return nil, nil
|
||||
func (r *RTPStatsSender) getAndResetSenderSnapshotWindow(senderSnapshotID uint32) (senderSnapshotWindow, senderSnapshotWindow, bool) {
|
||||
if !r.initialized || senderSnapshotID < cFirstSnapshotID {
|
||||
return senderSnapshotWindow{}, senderSnapshotWindow{}, false
|
||||
}
|
||||
|
||||
idx := senderSnapshotID - cFirstSnapshotID
|
||||
@@ -1131,7 +1235,7 @@ func (r *RTPStatsSender) getAndResetSenderSnapshotWindow(senderSnapshotID uint32
|
||||
|
||||
// snapshot now
|
||||
r.senderSnapshots[idx].senderView = r.getSenderSnapshotWindow(mono.UnixNano())
|
||||
return &then.senderView, &r.senderSnapshots[idx].senderView
|
||||
return then.senderView, r.senderSnapshots[idx].senderView, true
|
||||
}
|
||||
|
||||
func (r *RTPStatsSender) getSenderSnapshotWindow(startTime int64) senderSnapshotWindow {
|
||||
@@ -1158,9 +1262,9 @@ func (r *RTPStatsSender) getSenderSnapshotWindow(startTime int64) senderSnapshot
|
||||
}
|
||||
}
|
||||
|
||||
func (r *RTPStatsSender) getAndResetSenderSnapshotReceiverView(senderSnapshotID uint32) (*senderSnapshotReceiverView, *senderSnapshotReceiverView) {
|
||||
if !r.initialized || r.lastRRTime == 0 {
|
||||
return nil, nil
|
||||
func (r *RTPStatsSender) getAndResetSenderSnapshotReceiverView(senderSnapshotID uint32) (senderSnapshotReceiverView, senderSnapshotReceiverView, bool) {
|
||||
if !r.initialized || r.lastRRTime == 0 || senderSnapshotID < cFirstSnapshotID {
|
||||
return senderSnapshotReceiverView{}, senderSnapshotReceiverView{}, false
|
||||
}
|
||||
|
||||
idx := senderSnapshotID - cFirstSnapshotID
|
||||
@@ -1172,7 +1276,7 @@ func (r *RTPStatsSender) getAndResetSenderSnapshotReceiverView(senderSnapshotID
|
||||
|
||||
// snapshot now
|
||||
r.senderSnapshots[idx].receiverView = r.getSenderSnapshotReceiverView(r.lastRRTime, &then.receiverView)
|
||||
return &then.receiverView, &r.senderSnapshots[idx].receiverView
|
||||
return then.receiverView, r.senderSnapshots[idx].receiverView, true
|
||||
}
|
||||
|
||||
func (r *RTPStatsSender) getSenderSnapshotReceiverView(startTime int64, s *senderSnapshotReceiverView) senderSnapshotReceiverView {
|
||||
@@ -1251,11 +1355,13 @@ func (r *RTPStatsSender) clearSnInfos(extStartInclusive uint64, extEndExclusive
|
||||
return
|
||||
}
|
||||
|
||||
if extEndExclusive-extStartInclusive > uint64(len(r.snInfos)) {
|
||||
clear(r.snInfos)
|
||||
return
|
||||
}
|
||||
|
||||
for esn := extStartInclusive; esn != extEndExclusive; esn++ {
|
||||
snInfo := &r.snInfos[int(esn)%len(r.snInfos)]
|
||||
snInfo.pktSize = 0
|
||||
snInfo.hdrSize = 0
|
||||
snInfo.flags = 0
|
||||
r.snInfos[int(esn)%len(r.snInfos)] = snInfo{}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1273,14 +1379,18 @@ func (r *RTPStatsSender) getIntervalStats(
|
||||
extEndExclusive uint64,
|
||||
ehsn uint64,
|
||||
) (intervalStats intervalStats) {
|
||||
processESN := func(esn uint64, ehsn uint64) {
|
||||
slot := r.getSnInfoOutOfOrderSlot(esn, ehsn)
|
||||
if slot < 0 {
|
||||
intervalStats.packetsNotFoundMetadata++
|
||||
return
|
||||
}
|
||||
upperBound := ehsn + 1
|
||||
lowerBound := uint64(0)
|
||||
if n := uint64(len(r.snInfos)); n != 0 && ehsn >= n-1 {
|
||||
lowerBound = ehsn - n + 1
|
||||
}
|
||||
extStartInclusiveClamped := max(min(extStartInclusive, upperBound), lowerBound)
|
||||
extEndExclusiveClamped := max(min(extEndExclusive, upperBound), extStartInclusiveClamped)
|
||||
|
||||
snInfo := &r.snInfos[slot]
|
||||
intervalStats.packetsNotFoundMetadata = (extEndExclusive - extStartInclusive) - (extEndExclusiveClamped - extStartInclusiveClamped)
|
||||
|
||||
for esn := extStartInclusiveClamped; esn != extEndExclusiveClamped; esn++ {
|
||||
snInfo := &r.snInfos[int(esn)%len(r.snInfos)]
|
||||
switch {
|
||||
case snInfo.pktSize == 0:
|
||||
intervalStats.packetsLostFeed++
|
||||
@@ -1303,10 +1413,6 @@ func (r *RTPStatsSender) getIntervalStats(
|
||||
intervalStats.frames++
|
||||
}
|
||||
}
|
||||
|
||||
for esn := extStartInclusive; esn != extEndExclusive; esn++ {
|
||||
processESN(esn, ehsn)
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
|
||||
@@ -21,8 +21,6 @@ import (
|
||||
|
||||
"github.com/pion/rtp"
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"github.com/livekit/protocol/logger"
|
||||
)
|
||||
|
||||
func getPacket(sn uint16, ts uint32, payloadSize int) *rtp.Packet {
|
||||
@@ -37,10 +35,8 @@ func getPacket(sn uint16, ts uint32, payloadSize int) *rtp.Packet {
|
||||
|
||||
func Test_RTPStatsReceiver_Update(t *testing.T) {
|
||||
clockRate := uint32(90000)
|
||||
r := NewRTPStatsReceiver(RTPStatsParams{
|
||||
ClockRate: clockRate,
|
||||
Logger: logger.GetLogger(),
|
||||
})
|
||||
r := NewRTPStatsReceiver(RTPStatsParams{})
|
||||
r.SetClockRate(clockRate)
|
||||
|
||||
sequenceNumber := uint16(rand.Float64() * float64(1<<16))
|
||||
timestamp := uint32(rand.Float64() * float64(1<<32))
|
||||
@@ -89,7 +85,7 @@ func Test_RTPStatsReceiver_Update(t *testing.T) {
|
||||
len(packet.Payload),
|
||||
0,
|
||||
)
|
||||
require.True(t, flowState.IsNotHandled)
|
||||
require.Equal(t, RTPFlowUnhandledReasonPreStartTimestamp, flowState.UnhandledReason)
|
||||
require.Equal(t, sequenceNumber, r.sequenceNumber.GetHighest())
|
||||
require.Equal(t, sequenceNumber, uint16(r.sequenceNumber.GetExtendedHighest()))
|
||||
require.Equal(t, timestamp, r.timestamp.GetHighest())
|
||||
@@ -108,7 +104,7 @@ func Test_RTPStatsReceiver_Update(t *testing.T) {
|
||||
len(packet.Payload),
|
||||
0,
|
||||
)
|
||||
require.True(t, flowState.IsNotHandled)
|
||||
require.Equal(t, RTPFlowUnhandledReasonPreStartTimestamp, flowState.UnhandledReason)
|
||||
require.Equal(t, sequenceNumber, r.sequenceNumber.GetHighest())
|
||||
require.Equal(t, sequenceNumber, uint16(r.sequenceNumber.GetExtendedHighest()))
|
||||
require.Equal(t, timestamp, r.timestamp.GetHighest())
|
||||
@@ -218,7 +214,7 @@ func Test_RTPStatsReceiver_Update(t *testing.T) {
|
||||
len(packet.Payload),
|
||||
0,
|
||||
)
|
||||
require.True(t, flowState.IsNotHandled)
|
||||
require.Equal(t, RTPFlowUnhandledReasonOldSequenceNumber, flowState.UnhandledReason)
|
||||
require.Equal(t, sequenceNumber, r.sequenceNumber.GetHighest())
|
||||
require.Equal(t, sequenceNumber, uint16(r.sequenceNumber.GetExtendedHighest()))
|
||||
require.Equal(t, timestamp, r.timestamp.GetHighest())
|
||||
@@ -226,3 +222,65 @@ func Test_RTPStatsReceiver_Update(t *testing.T) {
|
||||
|
||||
r.Stop()
|
||||
}
|
||||
|
||||
func Test_RTPStatsReceiver_Restart(t *testing.T) {
|
||||
clockRate := uint32(90000)
|
||||
r := NewRTPStatsReceiver(RTPStatsParams{})
|
||||
r.SetClockRate(clockRate)
|
||||
|
||||
// should not restart till there are at least threshold packets
|
||||
require.False(t, r.maybeRestart(10, 20, 1000))
|
||||
require.False(t, r.maybeRestart(11, 20, 1000))
|
||||
require.False(t, r.maybeRestart(13, 20, 1000))
|
||||
require.False(t, r.maybeRestart(14, 20, 1000))
|
||||
// although adding 5th packet should have enough packets for a check,
|
||||
// still should not restart as there is a sequence number gap between 11 and 13
|
||||
require.False(t, r.maybeRestart(15, 20, 1000))
|
||||
require.False(t, r.maybeRestart(16, 19, 1000))
|
||||
// has enough packets, but still cannot restart because timestamps are not increasing
|
||||
require.False(t, r.maybeRestart(17, 21, 1000))
|
||||
require.False(t, r.maybeRestart(18, 21, 1000))
|
||||
require.False(t, r.maybeRestart(19, 21, 1000))
|
||||
// can restart as there are enough packets with proper sequencing
|
||||
require.True(t, r.maybeRestart(20, 21, 1000))
|
||||
require.Equal(t, restartThreshold, r.restartPacketsN)
|
||||
|
||||
r.resetRestart()
|
||||
require.Zero(t, r.restartPacketsN)
|
||||
|
||||
r.Stop()
|
||||
}
|
||||
|
||||
func Test_RTPStatsSender_getIntervalStats(t *testing.T) {
|
||||
t.Run("packetsNotFoundMetadata should match lost packets", func(t *testing.T) {
|
||||
r := NewRTPStatsSender(RTPStatsParams{}, 1024)
|
||||
stats := r.getIntervalStats(0, 10000, 10000)
|
||||
require.EqualValues(t, 8977, stats.packetsNotFoundMetadata)
|
||||
})
|
||||
}
|
||||
|
||||
func BenchmarkRTPStatsReceiver_Update(b *testing.B) {
|
||||
const clockRate = 90000
|
||||
const hdrSize = 12
|
||||
const payloadSize = 1200
|
||||
const paddingSize = 0
|
||||
const tsIncrement = 3000 // ~33ms at 90kHz
|
||||
|
||||
r := NewRTPStatsReceiver(RTPStatsParams{})
|
||||
r.SetClockRate(clockRate)
|
||||
|
||||
// initialize before running benchmark loop
|
||||
packetTime := time.Now().UnixNano()
|
||||
sn := uint16(0)
|
||||
ts := uint32(0)
|
||||
r.Update(packetTime, sn, ts, false, hdrSize, payloadSize, paddingSize)
|
||||
|
||||
b.ResetTimer()
|
||||
for i := 0; i < b.N; i++ {
|
||||
sn++
|
||||
ts += tsIncrement
|
||||
packetTime += 33 * int64(time.Millisecond) // ~30fps
|
||||
marker := (i+1)%100 == 0 // marker every 100 packets
|
||||
r.Update(packetTime, sn, ts, marker, hdrSize, payloadSize, paddingSize)
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user