Squashed 'livekit-server/' changes from 154b4d26..46c43095
46c43095 fix goreleaser workflow, version 1.13.1 (#4577) e0815be2 chore: improve docker test shutdown reliability (#4576) bfd9deff expose TCPFallbackRTTThreshold and AllowUDPUnstableFallback via config (#4556) b93c1e16 Release v1.13.0. (#4573) fd452212 Update mediatransportutil to get ICE candidate timeout config (#4572) 8be8c74a Update github workflows (#4463) c4e41872 Update go deps to v1.17.2 (#4462) dc8e0310 Update go deps to v4 (#4482) 20fd1ad2 turn: allow for providing secret via file (#4564) 6590570d Pin pion/dtls to v3.1.2 (#4570) cdbbee1f deps: bump protocol + psrpc to latest tips (#4565) d290de81 Correct config comment (#4563) 77ecf920 rtc: report participant session end time on room move (#4561) 63be96f6 Prevent panic from nil(illegal) syncState.Subscriptions message (#4560) 835ef1b3 Metrics for participant active, i. e. fully established. (#4557) 5bd42534 Document of advertise_internal_ip and external_ip_only (#4554) 356ae211 Config documentation for advertise_internal_ip and skip_external_ip_validation (#4552) 7c319a67 rtc: prevent duration reporting for inactive participants (#4550) 2dd5e632 telemetry: split webhook-processed hook out of NewTelemetryService (#4548) 222177a9 service: prevent nil deref in validate with wrapped join request (#4547) dd7580b4 Protect against nil clientInfo (#4546) 145689e6 Start tracking Twirp method request latency in prometheus too, not just in logs (#4545) cde89627 rtc: emit per-data-track bytes via BytesTrackStats (#4540) 2e22911d Remove backwards compatibility support for TURN auth. (#4539) 062d1219 Use NACKQuueInterface type. (#4538) 7f08b04c Add IsIntentionalDisconnect helper (#4537) 1ab2bf04 Clean up packet size logging (#4536) 8ab92a80 Don't require media sections when joining (#4535) 019a6640 rtc: report participant kind code and details (#4534) 77595d38 TEL-336: fix sip error categorization (#4528) f303f499 Always enable rtx codec (#4533) e4a8a55c Check Less and LessEq in version compare. (#4532) 37eb7a32 Release v1.12.0 (#4529) 4a7b1e85 Create NACK tracker only once. (#4527) 89faaeba Apply ttl check only when authenticate allocation creating (#4526) b32933b0 Log details of RTCP packets. (#4525) 8b79ec9e Support SIP auth realm for inbound. (#4522) 4b8db3cf Add integration test for TURN auth failures (#4524) ef2e5efe Log large packets receive/send. (#4521) d1236750 feat: auto create rooms for tokens with the RoomCreate grant (#4320) 7a3e595b apply room tags from JWT grant room configuration (#4518) ab7fdeab add AssignmentHook to AssignJob; propagate websocket write errors (#4516) cf20c9cd Add expiry to TURN password. (#4515) 20d4a3a1 Populate data track loggers with context (#4514) 12fff29a allow setting agent job assignment url (#4512) ba366fc7 Fix SIP media config upgrade. (#4511) 8fbc5adf update protocol for protojson (#4510) 3de6f517 Add TURN permission handler. (#4505) 8ffcef93 Update protocol to support SIP media config. (#4509) 1ab1e072 test: verify upstream and downstream connection stats end-to-end (#4508) c4fd71a5 Fix sense check in DeltaInfo gathering (#4507) 803999ef rename agent environment to deployment (#4506) bacc21e6 add helper to check for agent worker endpoint (#4503) 253f977d add duration seconds reporting (#4500) ffab3bd3 add agent environment (#4498) ccdf23c8 Use mediatransportutil/codec package, no functional change (#4497) 680703f2 Include reception reoprts in receiver report callback. (#4496) f51798bc Fix publish-only limitations being incorrectly applied to receivers (#4495) a002337d Legacy TrackInfo.Simulcast flag. (#4493) af1dcc88 Add CloseWithReason to agent SignalConn interface (#4492) d7c2daf1 report all simulcast layers (#4491) c1ad2b22 Misc optimisations. (#4490) 19b9e8c0 Additional data tracks logging (#4489) 743d9c8b add support for client capabilities (#4461) fc47e478 Close peer connection unconditionally to unblock set local/remote (#4485) 639406eb Update module github.com/pion/ice/v4 to v4.2.3 (#4481) dc6b7505 reduce some heap use in packet path (#4478) f3b80b28 fix: wrap IPv6 addresses in brackets in UDP TURN URLs (RFC 3986) (#4476) 3a7f2628 Turn off transceiver re-use on Safari. (#4474) d84f3d7a add more types to signum (#4473) 701a37c2 Convert sort.Slice -> slices.SortFunc (#4472) 85be9d70 Avoid stream allocator event data cast to interface and back. (#4471) b43685e8 Keep a shadow copy of tracks for use by different stream allocator state (#4470) 27c2b149 Consolidate RTCP packets and do RTCP callback outside lock. (#4469) 31083307 do not log data track stats if not started (#4468) 9ee06635 feat(pion/ice): replace deprecated NAT1To1 with SetAddressRewriteRules (#4466) 8ccad68d Release v1.11.0 (#4459) dbf5cf61 Store concrete ICE candidate for remote candidates. (#4458) 2a04bc3c fix publisher frame count reporting for simulcast streams (#4457) 1d804737 fix: limit join request and WHIP request body to http.DefaultMaxHeaderBytes (#4450) 3cfb71e7 Use Muted in TrackInfo to propagated published track muted. (#4453) 69aa9479 Some drive-by clean up (#4452) 6c81f678 Add subscriber stream start event notification (#4449) ce1bf47b Revert "fix: ensure num_participants is accurate in webhook events (#4265) (#…" (#4448) cdb0769c fix: ensure num_participants is accurate in webhook events (#4265) (#4422) c91e79af Switch to stdlib maps, slices (#4445) ea7b9c6f Update module github.com/livekit/protocol to v1.45.3 (#4435) 97378368 Update go deps (major) (#3179) d6aef547 Update go deps (#3862) afc9feae Update github workflows (#4331) 4b385612 chore: pin GH commits and switch to golangci-lint (#4444) 2974ba87 Unsubscribe from data track on close (#4443) 5dc4e90d Apply IPFilter when get local ip (#4440) 88c77dc6 compute agent dispatch affinity from target load (#4442) 8fe99377 Log join duration. (#4433) 0a503a57 Add `Close` method for UpDataTrackManager and call it on participant (#4432) 55912dff Add some simple data track stats (#4431) 050909e6 Enable data tracks by default. (#4429) 72c7e65c chore: log API key during worker registration (#4428) 8a67dd1b Do not close publisher peer connection to aid migration. (#4427) 91e90c10 Add some more logging around migration. (#4426) c6ddc879 isExpectedToResume is based on whether flushing or not. (#4425) 7d06cfca Keep subscription synchronous when publisher is expected to resume. (#4424) 934f8598 Clean up data track observers on unsubscribe. (#4421) 9674ac48 Cleaning up some logs and standardising log frequency. (#4420) 7b925304 Drop time inverted packets in RED -> Opus conversion. (#4418) 4d8d232a ensure participant init is correctly serialized for logging (#4417) 4fe80877 Log time inversion between incoming packets (#4415) 248d7394 Guard against timestamp inversion in RED -> Opus conversion. (#4414) 9ab8c1d5 clear track notifier observers on subscription teardown (#4413) 397cd09a Embedded turn test (#4412) 56326654 Prepare release v1.10.1 (#4408) e9b113c8 Make the TURN bind address configurable and allow for multiple addresses. (#4315) 4bc5e6bb Address malformed H264/H265 parsing issues. (#4407) 77a0a4fc AV1 parser overflow fix. (#4405) ff7fd7ed feat(agent-dispatch): add job restart policy (#4401) 34bd1e08 do not log roll over for padding only packets (#4396) 13d02ee9 add deadline to dtls connect context (#4395) 9055a349 Path check helpers (#4392) 1f1eeb68 Fallback to servicestore if rpc is unavailable (#4391) 59e9bb41 Fix TURN server URL (#4389) 9e0a7e54 Close both peer connections to aid migration. (#4382) 9474c807 route participant reads through PSRPC instead of Redis (#4387) a5333a86 add packet trailer stripping support (#4361) 0e3d765d Release v1.10.0 (#4380) bc3aeaf3 Update grpc to address CVE-2026-33186 (#4381) 8cdd6f4c Replace deprecated io/ioutil with io in whipservice (#4375) 89410df7 handle AGENT_ERROR disconnect reason (#4339) 8f984c77 Fix repair stream ID reporting for RTX pairing. (#4369) cdfaacfc Restart nacker on OOB sequence number restart. (#4368) 4a9e0045 Update protocol. (#4367) 750d5904 Add API to restart lite stats. (#4366) c8bb2578 Rename log field "pID" to "participantID" for consistency (#4365) 77fc74a7 Do not block all ext ID determination on stream allocator listener (#4364) 90a46fab Do not kick off migration of closed participant (#4363) 5dc2e7b1 Switch data track extension to 1-byte ID/length. (#4362) 7323ad02 Sample data send error logging. (#4358) 0d34e455 Add option to not re-use transceiver in e2ee. (#4356) 95225ff2 don't require media section for dual peerconnection mode (#4354) e9639538 Refine ipv6 support (#4352) b34b0472 Add StopEgress function to the EgressLauncher interface (#4353) 69235ed2 update readme (#4340) db1a8046 defensive check for peer connection instance (#4350) cb7dc2d0 TEL-405: support originating calls from custom domains (#4349) 7eaaaada Mark last run of grow bucket outside goroutine. (#4348) caa47522 Add option to require media sections when participant joining (#4347) 087050d1 Release v1.9.12 (#4346) 493e87df Fix SIP client timeout. (#4345) 52c28a93 Log a bit more details of out-of-order TWCC feedback report. (#4343) 516aeabf Use ParticipantTelemetryListener of LocalParticipant. (#4342) b3510565 Exclude ice restart case from offer answer id mismatch warning (#4341) 303657bc feat: make INSTALL_PATH overridable in install script (#3954) 9d418689 Send participant left event after track unpublished for moved (#4334) bab91868 do not discount packets lost on duplicate packets (#4333) 939794cf mark + restart (#4329) 75f9c462 Add debug for receiver restart. (#4328) 74891f30 Protect against incorrect temporal layer. (#4327) f51b2732 Update pion/webrtc and deps to update dtls (#4326) b81bac0e Key telemetry stats worker using combination of roomID, participantID (#4323) 2d40449f Update self-hosting deployment documentation link (#4312) 03e90dd7 Fix for some CodeQL reported issues (#4314) 77c858f0 Log stats worker when it not closable. (#4313) a6035212 ESP32 client info (#4267) 478e486a Copilot suggested improvement to Github Actions permissions (#4310) cbd2f82d Copilot suggested improvement to Github Actions permissions (#4311) a9b8d40d Publish is always on publisher peer connection. (#4307) 8ae56406 generate & log egressID for start egress request (#4303) 35d7ef88 Avoid alloc in RTPStatsReceiver.Update (#4302) bb744916 More optimisation in RTPStats module. (#4298) cefd5da9 Optimise some bits in rtpstats_receiver (#4297) 52a4b848 Read client protocol from query param (#4294) 195b17f6 Populate client_protocol field in ParticipantInfo (#4293) 370e0a4d Set up audio config in audio level module when config is updated. (#4290) f3e9b688 Do not increase max expected layer on track info update. (#4285) a9849340 Avoid logger data race. (#4284) 97016627 Do not hold lock when creating buffer (#4283) 6b68e3d5 Create buffer if needed when a PLI is requested. (#4282) 3cca7180 use separate allocation for signal stats telemetry guard (#4281) d2bae34d refresh telemetry guard on participant move (#4280) 333f0349 clear reference guard when resetting signal stats (#4279) d1bab17b Add session duration and participant kind to closing log. (#4277) 1e689e1a Reducing some info level logs. (#4274) 88facc02 adds a test to ensure agent worker errors cause disconnection (#4273) 76a41a7a Generate config flags (#4268) 700e1788 require participant broadcast when metadata/attributes are set in token (#4266) b61799ec Ignore parse addr error when add remote candidate (#4264) 01bd966f Add silent frame for pcmu/a (#4258) 0c33b8c6 chore: move codecs/mime stuff to protocol (#4255) 165c1735 Update livekit protocol to v1.44.0 (#4254) 343f12b8 Wrapping SIP errors for invalid argument and not found (#4253) aea044c5 Defer setting clock rate in RTPStats module till codec is bound. (#4250) d9f716c1 FIx receiver restart race (#4248) 0508ee9f remove copy/paste left over line (#4246) 40408407 Release v1.9.11 (#4245) f8675507 Do not remove participant from cache on disconnect. (#4241) a35a6ae7 Add participant option for data track auto-subscribe. (#4240) 07572511 Wrapping the invalid request errors for CreateSipParticipant (#4239) 843d8c3e Update Pion transport package. (#4237) a05690d2 Changing field naming of data track packet (#4235) c4015008 Clear participant version cache on disconnect (#4234) 18db4ec1 Log modified timeout of API context. (#4232) ac20ccda Log timeout in API. (#4231) 541a7d6c Change some logs to debugw (#4229) 52ab3374 Return on SDP fragment read error. (#4228) 7fae5ac9 Do not restart receiver on codec change mid-session. (#4225) dafdc36e Add option to force simuclast codec. (#4226) 0a7dd40b Use only layer 0 for SVC codecs. (#4224) 4ec0f8f4 Support OpenTelemetry tracing. Add Jaeger support. (#4222) 80ba93fa Do NACK updates as soon as flow state is processed. (#4221) 4405afe2 Use atomic pointer and return interface from RED transformer constructors (#4220) b649c2fe Remove method not needed from REDTransformer. (#4219) f0080f35 Remove enable arrival time forwarding method. (#4217) 335f4c33 Swap result sink atomically rather than closing and setting. (#4216) 46651c19 Release v1.9.10 (#4214) 08ac4ecd Support preserving external supplied time. (#4212) f6efccce report video size from media data for whip (#4211) d92f6a79 return iceservers for whip (#4210) 1a4758ed Skip restart callback if external. (#4208) dde4fb49 configurable dependency descriptor restart (#4207) 08793bea Use active at time to check for track not bound timeout. (#4206) 3606ce54 Do not warn about track not bound if participant is not ready. (#4205) b8ddd0f9 Taking interface{} -> any modernize bits (#4204) b91cd2e4 Rework receiver restart. (#4202) bb00c864 Restart API on receiver. (#4200) 25ece1e9 Minor refactor in buffer base and audio level (#4198) 599002f8 ignore PLI requests for non-video (#4196) 2510b946 Taking a bunch of go modernize suggestions. (#4194) ed8e6afc Handle repair SSRC of simulcast tracks during migration. (#4193) c6bf7a27 Fix logging key and other clean up around stream restart. (#4192) 3cb9abb6 Update pion/webrtc to v4.2.1 (#4191) 8b0efb8c Resolve RTX pair via OnTrack also. (#4190) 381bce03 Return extended sequence number only and not packet. (#4189) 6bcbf54e Always instantiate nacker when using out-of-band sequence numbers. (#4187) e71184de Store buffer after creating it. (#4186) 7c8ea115 Refactor receiver and buffer into Base and higher layer. (#4185) 4104b827 update protocol (#4183) cd99fec2 Make new path for signalling v1.5 support. (#4180) 32cd0370 Flush the ext packets on restart/close and release packets. (#4179) 1df1316b Move OnDataTrackMessage to participant listener to support replay. (#4178) e7601251 Make data message naming a bit more consistent. (#4177) a04e566d Use published track for model access in data down track. (#4176) 47c86be1 Add support for TURN static auth secret credentials (#3796) 24559a28 chore(deps): update github workflows to v6 (#3866) 0a824386 add explicit room exists servicestore op (#4175) 39bd077d Release v1.9.9 (#4174) cbb2c617 Publish/Unpublish counter match. (#4173) fb849edc Minor clean up (#4172) c28e5e45 fix(kindToProto): update protocol (#4171) 47324abd Drop run away receiver reports. (#4170) 462ec324 prevent uint overflow setting packet not found count (#4169) 5c841b8e Some logging changes. (#4168) 2f2d0a57 skip lost sequence number ranges in getIntervalStats (#4166) 898ebe05 clean up manual roomservice log redaction (#4165) 3e417253 move delete to oss service store (#4164) 5964efbb Ensure subscribe data track handles are unique (#4162) a26c4830 Add support for RTP stream restart. (#4161) 386f0b38 fix typo in clearing index when removing track from room track manager (#4158) 0abfb251 deregister observability function when participant is closed (#4157) 97aba5e7 Consistently undo update to sequence number and timestamp when the (#4156) 2317c295 Fix panic while removing track from room track manager. (#4153) a0a28ac5 Avoid duplicate track add to room track manager. (#4152) f01008f8 Revert telemetry stats worker wait configuration. (#4151) ca4b56d2 Handle case of sequence number jump just after start. (#4150) 97099cae Configurable telemetry stats worker clean up wait. (#4148) d7db7cb3 chore: fix a large number of spelling issues (#4147) cadf2ad9 Release v1.9.8 (#4145) 498304cd defensive nil check (#4144) 20f6a497 Store ddParser in atomic.Pointer (#4143) 037cb906 release ext packet if patching fails (#4142) dd598ef2 Release ExtPacket if dependency descriptor or other parsing fails (#4141) 35b0e2bc update webrtc to 4.1.8 to pick up DTLS fingerprint check during handshake (#4140) 1c1a836c Mark RTCP buffer Write as noinline. (#4138) ea9b2177 protocol deps to get inactive file adjusted memory usage. (#4137) 64f3d1e9 switch participant callbacks to room to listener interface (#4136) c6e6c021 add debug metric for tracking references (#4134) a30c79fa Use isEnding to indicate if down track could be resumed. (#4132) 8c241ecf Fix RTCP reader leak in DownTrack. (#4131) 3eef869a Do not pause rid in SDP (#4129) 8e01e595 Release 1.9.7 (#4128) 7c1a0fab Fix concurrent map access. (#4127) 14446b1c Let participant close remove the published tracks. (#4125) fa0633aa move utils.WrapAround to mediatransportutil (#4124) f8706cd4 Update pion/ice to stop gather first on close (#4123) 7954748d Data tracks (#4089) 7158d983 log bucket growth (#4122) 04b35eb6 Release v1.9.6 (#4121) ebdcead5 Update mediatransportutil to get bucket packet size limit. (#4120) 411b09f6 Release v1.9.5 (#4119) 8dcf235a Update pion/ice - attempt to address tcp packet conn close hang (#4116) 64c65143 Update mediatransportutil (#4115) 0a2943bb Clean up bits added to debug peer connection close hang. (#4114) 9c483a69 Use released version v1.8.41 of pion/sctp (#4113) 35c79a57 Update SCTP hash, had the wrong one in previous PR (#4111) e0fbbef1 Update pion/sctp with RFC9260 revert (#4110) f3c80917 Try SCTP with read deadline to unblock abort. (#4109) bd5382da Splitting transport close timeout logs. (#4108) 6d4154b8 Update pion/ice. (#4107) a6418ae2 Log more peer conenction state on close timeout. (#4105) 06d99974 Check for cancel on unsubscription/source track going away. (#4104) 7f10e18b Record join/publish/subscribe cancellations. (#4102) 40293632 Clear stereo=1 if stereo is not enabled. (#4101) 70f6def3 Add checks for participant and sub-components close. (#4100) ffbabcc7 Switch forwarding latency log to Debugw (#4098) 27d82a72 Fix "address" typo in transport logs (addddress → address) (#4097) 37a06821 logger proto redaction. (#4090) 54cf7d46 Control latency of lossy data channel (#4088) 5175c1af Lock x/tools at 0.37.0 (#4085) d510fff1 Downgrade x/tools to be able to make a release (#4084) c3ea5890 Prepare release v1.9.4. (#4083) 3a128e61 protocol bump for SIP error mapping and validation (#4081) c3964ba2 Use sync.Pool for objects in packet path. (#4066) f8b994d4 Forwarding latency measurement tweaks. (#4080) f4929f09 Revert "Revert pion/transpor to v3.0.8 (#4073)" (#4074) a04d9c48 Revert pion/transpor to v3.0.8 (#4073) 2d5054ad kind details for connector (#4072) a272e28a Log raeson for subscriber not being to determine codec. (#4071) b9b4eec9 Update pion/transport to v3.1.1 (#4070) b23d093c update protocol (#4069) 4ce07bed Higher resolution forwarding latency histogram. (#4067) 858db7ab fix(deps): update module github.com/livekit/protocol to v1.43.0 (#4015) 1dc9b8fc Use buffered indicator to exclude from forwarding latency. (#4062) f117ee51 Track start up delay. (#4061) 4872f205 Return write count from WriteRTP. (#4059) d0ba46b4 Log write count atomic. (#4057) ae5fb7e8 Add packet to forwarding stats only if packet is forwarded. (#4056) f6909192 Update PsRPC to get redis pipeliner implementation. (#4055) ca3c507b Prevent invalid track access while peer connection is shutting down. (#4054) 9ca6ee00 Use replace so that x/tools does not get overridden (#4048) b9323eab chore(deps): downgrade x/tools for counterfeiter (#4047) 2f1e6c36 Prep release v1.9.3 (#4046) 9d5c351d Fix prom units for forwarding latency/jitter. (#4045) e183657c Add prom histogram for forwarding latency and jitter. (#4044) 1eefeb30 Enable AbsCaptureTimeURI in RTC configuration (#4043) 075a7576 Use simulcast codec as default policy for audio track (#4040) c264b504 Don't warn 0 payload type for PCMU (#4039) 32fc3525 Broadcast cond var on RTX write. (#4038) 061eb8b4 AddDownTrack to regressed codec after restarting forwarder. (#4037) c87eb8ed fix: add missing Unlock() in AddReceiver (#4036) 70444924 if RingingTimeout is provided, deadline should be set to that timeout. (#4018) ab906d71 Prevent leakage of previous codec after codec regression. (#4035) 79b03f97 Log queueing latency when encountering high forwarding latency (#4034) 29117b14 set max layer in allocation (#4033) 15b19ccd Remove ~ from rid which indicates disabled layer to get the actual rid (#4032) 34e16a87 Check more conditions for opportunistic alloc. (#4031) 81fbd355 Use the optimal allocation function for opportunistic allocation. (#4030) a2ce73e0 Do not bind buffer if codec is invalid. (#4028) cef6fdb7 Correct direction for request/response for prom counters. (#4027) 5042c06c Use rtp converter from protocol/utils/rtputil (#4020) 5a426d15 Use rtp converter from protocol/utils (#4019) 35fb8877 feat: use env var for GOARCH (#4012) c0397696 Issue #1 only: Fix spatial layer initialization in Forwarder (#4003) 2afbf0e8 Some golang modernisation bits. (#4016) 484f784a Prepare release v1.9.2 (#4011) ad074ed2 counterfeiter needs an older version of x/tools (#4009) e63e8b6f Include mid -> trackID in both SDP offer and answer. (#4007) 781dfede Do not call receiver methods under settings lock. (#4006) 69ff25a0 Use answer with mid -> trackID mapping when in single peer connection (#4005) fe912acf Update pion/webrtc to prevent GetStats panic. (#4004) 7930dcde Do not try to read stats from peer connection after close. (#4002) ca0d5ee9 Count request/response packets on both client and server side. (#4001) dd62eb00 Resort to full search for requested quality is not available. (#4000) f6ca82d1 Revert to using silence packets for audio dummy start. (#3999) 0e2c59c8 Sort codec layers when adding track (#3998) 100bb46a Adding ProviderInfo to GetSIPTrunkAuthenticationResponse (#3993) a8d4df66 "Power of Two Random Choices" option for node selection (#3785) a20bbe34 Log RPC details. (#3991) 158496bc Increment RTP timestamp on padding when using dummy start. (#3989) 4f6ed65d Limit check to red + opus when looking for primary codec match. (#3988) a87f6c4b Allow passing inline trunk for outbound calls. (#3987) bf06596f Support Opus mixed with RED when encrypted. (#3986) ea208a1c Add encryption datapacket type (#3869) 2a6adbe8 Use padding only packets for dummy start of audio. (#3984) be018f97 Provide the InputVideo/AudioState to Ingress in WHIPRTCConnectionNotify (#3982) 146bd969 Do not panic of redis is not configured (#3981) 01337ba7 Do not start forawarding on out-of-order packet. (#3985) c7f625d6 Do not force codec regression between opus and red. (#3980) 3bd20ddb Revert unintentional change to not handle transport fallback on (#3970) 89a2f46c Update deps to fix redis issue when 1 cluster address is provided (#3969) 060719d1 add config for user data recording (#3966) b3ee219c fix stats worker closed condition (#3965) 3d737031 add idempotent reference count to telemetry stats worker (#3964) 735c663a Update protocol for EventKey helper. (#3963) 646b9de8 Add node_ip to config-sample.yaml (#3960) 0bf7b178 avoid logging on small values (#3958) 00ff2ab9 Adjust for hold time when fowarding RTCP report. (#3956) e180be06 short circuit participant broadcast filter in livestream mode (#3955) bfba6fee Adjust stream allocator ping interval based on state. (#3951) 3837006b Revert "Switch ops queue a singly linked list. (#3949)" (#3950) 990c5faf feat: server rpc apis (#3904) 80b11662 Switch ops queue a singly linked list. (#3949) 56ee2328 handle terminated job requests (#3948) 49f9b9c8 Flush stats when there are no packets. (#3947) e6a3df1e ForwarStats.GetStats needs to be public (#3946) 824d116b Tweaks tresholds for logging high forwarding latency/jitter. (#3945) 408492e0 Log some information around high forwarding latency. (#3944) 6a41fae5 Use microseconds for forwarding stats. (#3943) 856e0871 mediatransportutil to log local address when validating external IP (#3942) 40101cf7 Update protocol for SipCreateParticipant (#3939) b07e7a38 Use difference in key frame counter to stop seeder. (#3936) d7f92878 Avoid matching on empty track id. (#3937) 56fb2885 Do DD restart only if DD structure is present. (#3935) 86facce9 More debugging of DD jump (#3934) 6058a3f6 Add debugging from DD frame number wrap around. (#3933) dc6825c0 mediatransportutil crash fix for logging local address (#3930) d6f0588f Update mediatransportutil to log external IP found via STUN. (#3929) 2c30a064 Fix dynacast subscriber node clearing on move participant. (#3926) 6489237e Simulcast audio fixes (#3925) 9f0ab870 Wait for `SetRemoteDescription` before configuring senders. (#3924) df6c26db Subscrbed audio codecs - update from remote nodes. (#3921) 798fa761 Support simulcasting of audio (#3920) f4a06cf0 Clean code as there is no oss sweeper for ingress (#3918) 5f561b4f Include agent_name as a participant attribute (#3914) 782a35e8 update protocol for psrpc (#3915) eee2001a Set publisher codec preferences after setting remote description (#3913) fc995533 add incoming request id to request response message (#3912) 76645fad Rpcs for ingress proxy WHIP (#3911) 991a4a4f Refactor subscribedTrack + mediaTrackSubscriptions. (#3908) e16b3ba9 Use gzip reader pool (#3903) 17c34921 update protocol for sip api change (#3902) 2f43a575 Release candidate for v1.9.1 (#3899) 07c40cf3 Use `RequestResponse` to report protocol handling errors (#3895) 98352fd0 Prevent race in determining BWE type. (#3891) f7291fda Do not send both asb-send-time and twcc. (#3890) 21b42fa6 Do not advertise NACK for RED. (#3889) 6633bf93 Use departure timeout from room preset. (#3888) 38f7906e Handle migration better in single peer connection case. (#3886) 5026de2b handle frame number wrap back in svc (#3885) 091e3c13 Revert to using answer for migration case. (#3884) 2aeadf14 init ua parser once (#3883) 998a9f94 Switch known rids from 012 -> 210, used by OBS. (#3882) 890fd942 Single peer connection mode (#3873) bfe98eaa fix: ensure the participant kind is set on refresh tokens (#3881) 8d270e2a chunk room updates (#3880) b4e146c5 update mediatransport util for ice port 3478 (#3877) dc3a7753 Fix timeout handing in StopEgress (#3876) d62336e1 Remove unnecessary check (#3870) c58e5d23 Update golang Docker tag to v1.25 (#3864) 98d577ee Update module github.com/livekit/protocol to v1.40.0 (#3865) afbf541e Update pion deps (#3863) b660c3b5 Extract video size from media stream (#3856) 456b8709 Fix missed unlock (#3861) d500806e Handle no codecs in track info. (#3859) 11b240d6 Log track settings more. (#3853) 1aa0f963 Log signal messages on media node. (#3852) b182d07b Log signal messages as debug. (#3851) a370bb20 Support G.711 A-law and U-law (#3849) fa5f4ef3 Populate SDP cid in track info when available. (#3845) eed27885 Send `participant_connection_aborted` when participant session is closed (#3848) 61e59346 Update go deps (#3439) 1b228913 Support video layer mode from client and make most of the code mime aware (#3843) f2da4444 Support per simulcast codec layers. (#3840) f275f592 handle SyncState in join request (#3839) 5d44cf6d Use wrapped join request to be able to support compressed and uncompressed. (#3838) 5ca16264 Support join request as proto + base64 encoded query param (#3836) 7dea1012 Clean up missed v2 pieces (#3837) 34a49130 Delete v2 signalling (#3835) 1fe33716 Fix: RingingTimeout was being skipped for transferParticipant (#3831) 5751692a deps (#3829) db4bc127 Get to the point of connecting publisher PC and using it for async signalling (#3822) 5e483e75 update readme (#3809) e3155b14 Get to the point of establishing subscriber peer connection. (#3821) a7ce1382 HTTP DELETE of participant session (#3819) 01de0e36 Do not send leave if nil (to older clients) (#3817) 10103449 Add country label to edge prom stats. (#3816) 68387b41 Minor tweak to keep RPC type at service level. (#3815) a75295fc More v2 signalling changes (#3814) b20db94d Validation end point for v2 signalling. (#3811) f2f595f4 update readme (#3808) fffc2ac0 Use signalling utils from protocol (#3807) f5fc82d3 Filling out messages unlikely to change in v2. (#3806) 1c99b9ad Split signal segmenter and reassembler. (#3805) 0a1bfd30 Signal handling interfaces and participant specific HTTP PATCH. (#3804) 7837c8e5 starting signaller interface (#3802) 18ce5244 Handle Metadata field from RoomConfig (#3798) 2a6a9b8a Grouping all signal messages into participant_signal. (#3801) 078c01fa Signal v2: envelope and fragments as wire message format. (#3800) b9a44c3f Signalling V2 protocol implementation start (#3794) ba702a53 forward agent id to job state (#3786) 1f31d430 Map ErrNoResponse to ErrRequestTimedOut in StopEgress to avoid returning 503 (#3788) 51bbe8c5 Set participant active when peerconnection connected (#3790) 40028dc3 Normalize known rids. (#3779) ddd92329 Return default layer for invalid rid + track info combination. (#3778) 8c033ce9 Enable H265 by default (#3773) 7678e087 Set rids for all codecs. (#3772) 5d636acf Limit taking rids from SDP only in WHIP path. (#3771) 4d09e5b5 Log SDP rids to understand the mapping better. (#3770) c69f1aae Revert "Temporary change: use pre-defined rids" (#3769) 8197438e bounds check layer index (#3768) d11da5f5 Temporary change: use pre-defined rids (#3767) cb4da533 fix(deps): update module github.com/livekit/protocol to v1.39.3 (#3733) d6d2b6d8 feat(cli-flags): add option for cpu profiling (#3765) 9fc4ddbe ClearAllReceivers interface is used to pause relay tracks. (#3761) 1216113b Do not need to just clean up receivers. Remove that interface. (#3760) ef6c38ce Log previous allocation to see changes. (#3759) 01bf9685 SVC with RID -> spatial layer mapping (#3754) c481396f offer could be nil when migrating. (#3752) 8c2fc0bc Fix svc encoding for chrome mobile on iOS (#3751) e467daa0 move egress roomID load to launcher (#3748) 3783ebb3 feat(cli): update to urfave/cli/v3 (#3745) 03d3fcab Fix data packet ParticipantIdentity override logic in participant.go (#3735) 068b4366 reuse compiled client config scripts (#3743) e754a860 return error when moving egree/agent participant (#3741) 7542cf07 remove unused code (#3740) 9d569e2f Take ClientInfo from request. (#3738) 80774263 chore: set workerid on job creation (#3737) 5549ab55 Revert clearing RIDs. (#3732) ae967313 Clear rids if not present in SDP. (#3731) 0e033907 Return highest available layer if requested quality is higher than (#3729) 9ce737db Add log for dropping out of order reliable message (#3728) 1b95e818 Don't check bindState on downtrack.Bind (#3726) 670f927f Set and use rid/spatial layer in TrackInfo. (#3724) a9e29116 Add Id to SDP signalling messages. (#3722) 4ec828ce Fix bug with SDP rid, clear only overflow. (#3723) 8f6c3a9b Clear rids from default for layers not published. (#3721) ce07740e Add simulcast support for WHIP. (#3719) e98fb94f Create client config manager in room manager constructor. (#3718) fdf9b852 e2e reliability for data channel (#3716) 35dda8ea swap pub/sub track metrics (#3717) 1d9a4366 Do not require create permission for WHIP participant. (#3715) e0aea17a Flush stats on close (#3713) 630aa7d9 implement observability for room metrics (#3712) e7f0294e remove unused ws signal read loop (#3709) 77f70b18 for real, pick up protocol change for webhooks queue length bucnkets (#3707) 7b180646 protocol dep for webhook stats buckets (#3706) b0ab95ba warn about credentials when used in tokens (#3705) a72ce30f Small changes to add/use helper functions for length checks. (#3704) 425f6bb3 Allow passing extra attributes to RTC endpoint. (#3693) 758e1762 Add a trend check before declaring joint queuing region. (#3701) fe81e411 Adds Devin to readme so it auto updates DeepWiki weekly (#3699) 09dede35 version bump v1.9.0 (#3698) fc867c5b Webhook prom stats (#3697) 0e17916f Do not use Redis pipeline for SIP delete. Fixes Redis clustering support. (#3694) 1b760393 WHIP support. (#3692) e4f7d81b add client ip to agent worker registration (#3675) 6b849a87 update mediatransportutil for sctp congestion control (#3673) 83b189b0 Add ServerInfo to ReconnectResponse (#3671) 5f87a35b Prevent operating on swapped out map. (#3670) 13b55a80 move agent token (#3669) c9385edd handle agent worker jwt (#3668) 3b359d8b Use logger resolver reset to reset contexts. (#3665) dbb70e0f Fix dynacast quality for moving out tracks (#3664) 0a5f3c2a resolve new room name logger earlier when moving participant (#3662) 2df05517 Revert unbound transceiver stop. (#3661) 5172af15 ~Send initial participant update only after a participant becomes active.~ - General clean up (#3655) 5d4d86f8 protocol to pick resolver values replace (#3659) 7f8e6323 Send self participant update immediately. (#3656) 11630878 Use unordered for lossy data channel. (#3653) aee34ffe log request for agent dispatch api (#3650) 793b383a Add Moving participant to another room (#3648) 2fff36cb Stub MoveParticipant so that cloud can include the latest protocol. (#3646) d4ab1142 Redact address (#3643) b83190a3 protocol update to fix memory stats path (#3642) a1f4e88e Update protocol to latest, got bit by tag (#3641) 8d3902af Protocol to pick up cgroups v2 memory path fix (#3640) 2f002388 Use participant close reason in remove. (#3639) 58822c26 Include clientInfo in connectivity logs. (#3638) 6d6393a6 Take AudioFeatures from AddTrack. (#3635) 08670412 Limit buffer queue before Bind. (#3634) 9f5bc9b9 Avoid synthesising duplicate feature. (#3632) 847239c3 Disable vp9 for safari 18.4 (#3631) f69ab680 Populate the sender identity when translating to user packet. (#3628) e1490558 Forward data between WHIP client and non-WHIP client (#3627) 6739e7bc Broadcast inside lock (#3626) f24152b4 Call Broadcast in lock scope. (#3625) b760918a Use logger from request context. (#3623) 34a2e2c1 Check for multiple layers for managed track. (#3622) 4955ebe4 Forward transfer headers to internal request (#3615) d9ee9214 Set up RTX for WHIP publish (#3619) d8cf5439 Determine TURN connection type and no fallback for TURN/TLS. (#3612) d0d212fd Fix WHIP ICE restart. (#3616) 5e7f8a12 Update mediatransportutil for max sctp message (65K) (#3611) 28dfac14 Use exported GetEgressNotifyOptions (#3604) 75236bef protocol update to fix IPv6 SDP fragment parsing (#3603) 2130980d Add basic video support to WHIP. (#3602) e5cbb227 Allow specifying extra webhooks with egress requests (#3597) 7e16106a Add OnSubscirberReady callback on LocalParticipant. (#3600) 5c2d96b9 Check DestinationRoom of VideoGrant for participant forwarding (#3599) 2e236a19 Revert participant state ACTIVE change. (#3598) 35ac5f56 Add support for WHIP ICE Trickle/Restart. (#3596) ec2dff96 Fix SIP updates when replacing slices. (#3592) e24fe77b map PEER_CONNECTION_DISCONNECTED -> CONNECTION_TIMEOUT (#3591) 6ee6eb43 Do not drop audio codecs (#3590) 68357ba6 List audio codecs after video codecs. (#3589) 05a891ff Fix rule (had an extra bracket) (#3588) d7c41091 Exclude RED from enabled codecs for Flutter + 2.4.2 + Android. (#3587) ee08aede skip out of order participant state updates (#3583) 15a8d9a2 Break track published fuse when there are no tracks (#3581) 35f83c51 Replace Promise with Fuse. (#3580) 07fe9b72 Prevent migration race. (#3579) ac8082ff Use older SDP module to accommodate bad SDP. (#3578) 1c8307c7 Use cgroup for memstats. (#3573) e9be0fca log SDP offer on error (#3577) 3238ab8d Calculate rates for memory used and total. (#3570) d08487bf Unlabeled (pass through) data channels. (#3567) 52ce18d5 fix: revert recent changes to determine simulcast from sdp (#3565) cdfbb106 Audio uses signal SignalCid and SdpCid. (#3564) ed5e2f16 Keep simulcast information tied to receiver. (#3563) ad010cfc chore(logs): log VLS type for VP9/AV1 (#3561) 8cc17f8f Rework node stats a bit. (#3555) 15f56551 fix(video): determine svc/simulcast from SDP for advanced codecs (#3549) 2b6a46f4 Handle `prefer_regression` for backup codec (#3554) b0abb0ae Add option to use different pacer with send side bwe. (#3552) 26822b6b ParseUsername utility for TURN user name. (#3547) 55909ed7 log the initial join response (#3546) 97fcb82a Fix: Return NotFoundErr instead of Unavailable when the participant does not exist in UpdateParticipant. (#3543) 75d0e18e Implement SIP update API. (#3141) e118aff1 Fire track subscribed when the subscriber connected (#3540) 13417c01 Send mute event only on change (#3537) 7f4c4597 Stubs for SIP update API. (#3533) fe673bb2 Send regressed codec upstream stats to analytics. (#3532) 8eb81388 Use a generation to counter to stop key frame seeder on codec change (#3531) 188470a2 Do not accept unsupported track type in AddTrack (#3530) 507fc9cf Do not instantiate 0 sized sequencer. (#3529) 20bddfea Clean up published track on participant removal. (#3527) 65d8aa28 Handle subscribe race with track close better. (#3526) a6cb00b3 Reduce seeder duration to 30s and also do not force send PLI. (#3525) c8233205 Add a key frame seeder in up track. (#3524) 0f61ff3a Remove redundant log (#3523) 7685cd25 Log ParticipantInit on signal start to get a picture of join params (#3522) ac9e62ef add server agent load threshold config (#3520) cd5d32f0 Add pID and connID to log context to make it easier to search using pID. (#3518) 2d9aa6dd Update api call info method (#3515) b3779a90 WebHookConfig (#3517) 6121b9af Check ForwardParticipant room name (#3514) 9a7c9442 mediatransportutil update (#3511) 50ab47c1 Log packet drops/forward. (#3510) 139d1b13 Add ForwardParticipant method to room service (#3507) 6c04909f Use atomic to store codec. (#3505) 7f6afe05 Prevent bind lock deadlock on muted. (#3504) 48063df5 load mime type before calling writeBlankFrameRTP (#3502) d2e6cd15 Do not bind lock across flush which could take time (#3501) 47896f50 Update protocol and IO service. (#3499) 1dc42eef Bump github.com/go-jose/go-jose/v3 from 3.0.3 to 3.0.4 (#3497) 3a35cbc4 Log migration complete only when coming from sync (#3496) c2f17a10 refactor: using slices.Contains to simplify the code (#3495) 01e51dbd fix: fix the wrong error return value (#3493) ff9115b2 Disable dd parser for vp8 if extension is not found (#3492) c3e06f05 Do not attempt to create objects for URL ingresses as the ingress service will do so (#3491) f0edfbba Fix receiver rtt/jitter. (#3487) 05dfd30d Take RTT and jitter from receiver view while reporting track stats for (#3483) 04ed5683 Don't issue TrackPublished/Unpublished event on migrated track (#3482) 1cffe30c Use a RED transformer to consolidate both RED -> Opus OR Opus -> RED (#3481) 591888f7 Fix missing RTCP sender report when forwarding RED as Opus. (#3480) git-subtree-dir: livekit-server git-subtree-split: 46c4309554d37d23ee8da88a8a7e02a68fba09c1
This commit is contained in:
@@ -22,6 +22,7 @@ import (
|
||||
|
||||
"github.com/livekit/protocol/livekit"
|
||||
"github.com/livekit/protocol/logger"
|
||||
"github.com/livekit/protocol/observability/roomobs"
|
||||
"github.com/livekit/protocol/rpc"
|
||||
"github.com/livekit/protocol/utils/guid"
|
||||
|
||||
@@ -34,8 +35,24 @@ type AnalyticsService interface {
|
||||
SendStats(ctx context.Context, stats []*livekit.AnalyticsStat)
|
||||
SendEvent(ctx context.Context, events *livekit.AnalyticsEvent)
|
||||
SendNodeRoomStates(ctx context.Context, nodeRooms *livekit.AnalyticsNodeRooms)
|
||||
RoomProjectReporter(ctx context.Context) roomobs.ProjectReporter
|
||||
}
|
||||
|
||||
// ----------------------------
|
||||
|
||||
var _ AnalyticsService = &NullAnalyticService{}
|
||||
|
||||
type NullAnalyticService struct{}
|
||||
|
||||
func (n NullAnalyticService) SendStats(_ context.Context, _ []*livekit.AnalyticsStat) {}
|
||||
func (n NullAnalyticService) SendEvent(_ context.Context, _ *livekit.AnalyticsEvent) {}
|
||||
func (n NullAnalyticService) SendNodeRoomStates(_ context.Context, _ *livekit.AnalyticsNodeRooms) {}
|
||||
func (n NullAnalyticService) RoomProjectReporter(_ctx context.Context) roomobs.ProjectReporter {
|
||||
return nil
|
||||
}
|
||||
|
||||
// ----------------------------
|
||||
|
||||
type analyticsService struct {
|
||||
analyticsKey string
|
||||
nodeID string
|
||||
@@ -95,3 +112,7 @@ func (a *analyticsService) SendNodeRoomStates(_ context.Context, nodeRooms *live
|
||||
logger.Errorw("failed to send node room states", err)
|
||||
}
|
||||
}
|
||||
|
||||
func (a *analyticsService) RoomProjectReporter(_ context.Context) roomobs.ProjectReporter {
|
||||
return roomobs.NewNoopProjectReporter()
|
||||
}
|
||||
|
||||
+103
-51
@@ -20,15 +20,16 @@ import (
|
||||
|
||||
"google.golang.org/protobuf/types/known/timestamppb"
|
||||
|
||||
"github.com/livekit/livekit-server/pkg/sfu/mime"
|
||||
"github.com/livekit/livekit-server/pkg/telemetry/prometheus"
|
||||
"github.com/livekit/protocol/codecs/mime"
|
||||
"github.com/livekit/protocol/egress"
|
||||
"github.com/livekit/protocol/livekit"
|
||||
"github.com/livekit/protocol/logger"
|
||||
"github.com/livekit/protocol/utils/guid"
|
||||
"github.com/livekit/protocol/webhook"
|
||||
)
|
||||
|
||||
func (t *telemetryService) NotifyEvent(ctx context.Context, event *livekit.WebhookEvent) {
|
||||
func (t *telemetryService) NotifyEvent(ctx context.Context, event *livekit.WebhookEvent, opts ...webhook.NotifyOption) {
|
||||
if t.notifier == nil {
|
||||
return
|
||||
}
|
||||
@@ -36,7 +37,7 @@ func (t *telemetryService) NotifyEvent(ctx context.Context, event *livekit.Webho
|
||||
event.CreatedAt = time.Now().Unix()
|
||||
event.Id = guid.New("EV_")
|
||||
|
||||
if err := t.notifier.QueueNotify(ctx, event); err != nil {
|
||||
if err := t.notifier.QueueNotify(ctx, event, opts...); err != nil {
|
||||
logger.Warnw("failed to notify webhook", err, "event", event.Event)
|
||||
}
|
||||
}
|
||||
@@ -79,6 +80,7 @@ func (t *telemetryService) ParticipantJoined(
|
||||
clientInfo *livekit.ClientInfo,
|
||||
clientMeta *livekit.AnalyticsClientMeta,
|
||||
shouldSendEvent bool,
|
||||
guard *ReferenceGuard,
|
||||
) {
|
||||
t.enqueue(func() {
|
||||
_, found := t.getOrCreateWorker(
|
||||
@@ -87,6 +89,7 @@ func (t *telemetryService) ParticipantJoined(
|
||||
livekit.RoomName(room.Name),
|
||||
livekit.ParticipantID(participant.Sid),
|
||||
livekit.ParticipantIdentity(participant.Identity),
|
||||
guard,
|
||||
)
|
||||
if !found {
|
||||
prometheus.IncrementParticipantRtcConnected(1)
|
||||
@@ -108,10 +111,11 @@ func (t *telemetryService) ParticipantActive(
|
||||
participant *livekit.ParticipantInfo,
|
||||
clientMeta *livekit.AnalyticsClientMeta,
|
||||
isMigration bool,
|
||||
guard *ReferenceGuard,
|
||||
) {
|
||||
t.enqueue(func() {
|
||||
if !isMigration {
|
||||
// consider participant joined only when they became active
|
||||
// a participant is considered "joined" only when they become "active"
|
||||
t.NotifyEvent(ctx, &livekit.WebhookEvent{
|
||||
Event: webhook.EventParticipantJoined,
|
||||
Room: room,
|
||||
@@ -125,12 +129,13 @@ func (t *telemetryService) ParticipantActive(
|
||||
livekit.RoomName(room.Name),
|
||||
livekit.ParticipantID(participant.Sid),
|
||||
livekit.ParticipantIdentity(participant.Identity),
|
||||
guard,
|
||||
)
|
||||
if !found {
|
||||
// need to also account for participant count
|
||||
prometheus.AddParticipant()
|
||||
}
|
||||
worker.SetConnected()
|
||||
prometheus.IncrementParticipantRtcActive(1)
|
||||
|
||||
ev := newParticipantEvent(livekit.AnalyticsEventType_PARTICIPANT_ACTIVE, room, participant)
|
||||
ev.ClientMeta = clientMeta
|
||||
@@ -161,6 +166,7 @@ func (t *telemetryService) ParticipantResumed(
|
||||
livekit.RoomName(room.Name),
|
||||
livekit.ParticipantID(participant.Sid),
|
||||
livekit.ParticipantIdentity(participant.Identity),
|
||||
nil,
|
||||
)
|
||||
if !found {
|
||||
prometheus.AddParticipant()
|
||||
@@ -179,37 +185,55 @@ func (t *telemetryService) ParticipantLeft(ctx context.Context,
|
||||
room *livekit.Room,
|
||||
participant *livekit.ParticipantInfo,
|
||||
shouldSendEvent bool,
|
||||
guard *ReferenceGuard,
|
||||
) {
|
||||
t.enqueue(func() {
|
||||
isConnected := false
|
||||
if worker, ok := t.getWorker(livekit.ParticipantID(participant.Sid)); ok {
|
||||
if worker, ok := t.getWorker(livekit.RoomID(room.Sid), livekit.ParticipantID(participant.Sid)); ok {
|
||||
isConnected = worker.IsConnected()
|
||||
if worker.Close() {
|
||||
if worker.Close(guard) {
|
||||
prometheus.SubParticipant()
|
||||
} else {
|
||||
logger.Infow(
|
||||
"stats worker active",
|
||||
"room", room.Name,
|
||||
"roomID", room.Sid,
|
||||
"participant", participant.Identity,
|
||||
"participantID", participant.Sid,
|
||||
"worker", worker,
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
if isConnected && shouldSendEvent {
|
||||
if shouldSendEvent {
|
||||
webhookEvent := webhook.EventParticipantLeft
|
||||
analyticsEvent := livekit.AnalyticsEventType_PARTICIPANT_LEFT
|
||||
if !isConnected {
|
||||
webhookEvent = webhook.EventParticipantConnectionAborted
|
||||
analyticsEvent = livekit.AnalyticsEventType_PARTICIPANT_CONNECTION_ABORTED
|
||||
}
|
||||
t.NotifyEvent(ctx, &livekit.WebhookEvent{
|
||||
Event: webhook.EventParticipantLeft,
|
||||
Event: webhookEvent,
|
||||
Room: room,
|
||||
Participant: participant,
|
||||
})
|
||||
|
||||
t.SendEvent(ctx, newParticipantEvent(livekit.AnalyticsEventType_PARTICIPANT_LEFT, room, participant))
|
||||
t.SendEvent(ctx, newParticipantEvent(analyticsEvent, room, participant))
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func (t *telemetryService) TrackPublishRequested(
|
||||
ctx context.Context,
|
||||
roomID livekit.RoomID,
|
||||
roomName livekit.RoomName,
|
||||
participantID livekit.ParticipantID,
|
||||
identity livekit.ParticipantIdentity,
|
||||
track *livekit.TrackInfo,
|
||||
) {
|
||||
t.enqueue(func() {
|
||||
prometheus.AddPublishAttempt(track.Type.String())
|
||||
room := t.getRoomDetails(participantID)
|
||||
prometheus.RecordTrackPublishAttempt(track.Type.String())
|
||||
room := toMinimalRoomProto(roomID, roomName)
|
||||
ev := newTrackEvent(livekit.AnalyticsEventType_TRACK_PUBLISH_REQUESTED, room, participantID, track)
|
||||
if ev.Participant != nil {
|
||||
ev.Participant.Identity = string(identity)
|
||||
@@ -220,15 +244,21 @@ func (t *telemetryService) TrackPublishRequested(
|
||||
|
||||
func (t *telemetryService) TrackPublished(
|
||||
ctx context.Context,
|
||||
roomID livekit.RoomID,
|
||||
roomName livekit.RoomName,
|
||||
participantID livekit.ParticipantID,
|
||||
identity livekit.ParticipantIdentity,
|
||||
track *livekit.TrackInfo,
|
||||
shouldSendEvent bool,
|
||||
) {
|
||||
t.enqueue(func() {
|
||||
prometheus.AddPublishedTrack(track.Type.String())
|
||||
prometheus.AddPublishSuccess(track.Type.String())
|
||||
prometheus.RecordTrackPublishSuccess(track.Type.String())
|
||||
if !shouldSendEvent {
|
||||
return
|
||||
}
|
||||
|
||||
room := t.getRoomDetails(participantID)
|
||||
room := toMinimalRoomProto(roomID, roomName)
|
||||
participant := &livekit.ParticipantInfo{
|
||||
Sid: string(participantID),
|
||||
Identity: string(identity),
|
||||
@@ -246,22 +276,30 @@ func (t *telemetryService) TrackPublished(
|
||||
})
|
||||
}
|
||||
|
||||
func (t *telemetryService) TrackPublishedUpdate(ctx context.Context, participantID livekit.ParticipantID, track *livekit.TrackInfo) {
|
||||
func (t *telemetryService) TrackPublishedUpdate(
|
||||
ctx context.Context,
|
||||
roomID livekit.RoomID,
|
||||
roomName livekit.RoomName,
|
||||
participantID livekit.ParticipantID,
|
||||
track *livekit.TrackInfo,
|
||||
) {
|
||||
t.enqueue(func() {
|
||||
room := t.getRoomDetails(participantID)
|
||||
room := toMinimalRoomProto(roomID, roomName)
|
||||
t.SendEvent(ctx, newTrackEvent(livekit.AnalyticsEventType_TRACK_PUBLISHED_UPDATE, room, participantID, track))
|
||||
})
|
||||
}
|
||||
|
||||
func (t *telemetryService) TrackMaxSubscribedVideoQuality(
|
||||
ctx context.Context,
|
||||
roomID livekit.RoomID,
|
||||
roomName livekit.RoomName,
|
||||
participantID livekit.ParticipantID,
|
||||
track *livekit.TrackInfo,
|
||||
mime mime.MimeType,
|
||||
maxQuality livekit.VideoQuality,
|
||||
) {
|
||||
t.enqueue(func() {
|
||||
room := t.getRoomDetails(participantID)
|
||||
room := toMinimalRoomProto(roomID, roomName)
|
||||
ev := newTrackEvent(livekit.AnalyticsEventType_TRACK_MAX_SUBSCRIBED_VIDEO_QUALITY, room, participantID, track)
|
||||
ev.MaxSubscribedVideoQuality = maxQuality
|
||||
ev.Mime = mime.String()
|
||||
@@ -271,13 +309,15 @@ func (t *telemetryService) TrackMaxSubscribedVideoQuality(
|
||||
|
||||
func (t *telemetryService) TrackSubscribeRequested(
|
||||
ctx context.Context,
|
||||
roomID livekit.RoomID,
|
||||
roomName livekit.RoomName,
|
||||
participantID livekit.ParticipantID,
|
||||
track *livekit.TrackInfo,
|
||||
) {
|
||||
t.enqueue(func() {
|
||||
prometheus.RecordTrackSubscribeAttempt()
|
||||
|
||||
room := t.getRoomDetails(participantID)
|
||||
room := toMinimalRoomProto(roomID, roomName)
|
||||
ev := newTrackEvent(livekit.AnalyticsEventType_TRACK_SUBSCRIBE_REQUESTED, room, participantID, track)
|
||||
t.SendEvent(ctx, ev)
|
||||
})
|
||||
@@ -285,6 +325,8 @@ func (t *telemetryService) TrackSubscribeRequested(
|
||||
|
||||
func (t *telemetryService) TrackSubscribed(
|
||||
ctx context.Context,
|
||||
roomID livekit.RoomID,
|
||||
roomName livekit.RoomName,
|
||||
participantID livekit.ParticipantID,
|
||||
track *livekit.TrackInfo,
|
||||
publisher *livekit.ParticipantInfo,
|
||||
@@ -297,7 +339,7 @@ func (t *telemetryService) TrackSubscribed(
|
||||
return
|
||||
}
|
||||
|
||||
room := t.getRoomDetails(participantID)
|
||||
room := toMinimalRoomProto(roomID, roomName)
|
||||
ev := newTrackEvent(livekit.AnalyticsEventType_TRACK_SUBSCRIBED, room, participantID, track)
|
||||
ev.Publisher = publisher
|
||||
t.SendEvent(ctx, ev)
|
||||
@@ -306,6 +348,8 @@ func (t *telemetryService) TrackSubscribed(
|
||||
|
||||
func (t *telemetryService) TrackSubscribeFailed(
|
||||
ctx context.Context,
|
||||
roomID livekit.RoomID,
|
||||
roomName livekit.RoomName,
|
||||
participantID livekit.ParticipantID,
|
||||
trackID livekit.TrackID,
|
||||
err error,
|
||||
@@ -314,7 +358,7 @@ func (t *telemetryService) TrackSubscribeFailed(
|
||||
t.enqueue(func() {
|
||||
prometheus.RecordTrackSubscribeFailure(err, isUserError)
|
||||
|
||||
room := t.getRoomDetails(participantID)
|
||||
room := toMinimalRoomProto(roomID, roomName)
|
||||
ev := newTrackEvent(livekit.AnalyticsEventType_TRACK_SUBSCRIBE_FAILED, room, participantID, &livekit.TrackInfo{
|
||||
Sid: string(trackID),
|
||||
})
|
||||
@@ -325,6 +369,8 @@ func (t *telemetryService) TrackSubscribeFailed(
|
||||
|
||||
func (t *telemetryService) TrackUnsubscribed(
|
||||
ctx context.Context,
|
||||
roomID livekit.RoomID,
|
||||
roomName livekit.RoomName,
|
||||
participantID livekit.ParticipantID,
|
||||
track *livekit.TrackInfo,
|
||||
shouldSendEvent bool,
|
||||
@@ -333,7 +379,7 @@ func (t *telemetryService) TrackUnsubscribed(
|
||||
prometheus.RecordTrackUnsubscribed(track.Type.String())
|
||||
|
||||
if shouldSendEvent {
|
||||
room := t.getRoomDetails(participantID)
|
||||
room := toMinimalRoomProto(roomID, roomName)
|
||||
t.SendEvent(ctx, newTrackEvent(livekit.AnalyticsEventType_TRACK_UNSUBSCRIBED, room, participantID, track))
|
||||
}
|
||||
})
|
||||
@@ -341,6 +387,8 @@ func (t *telemetryService) TrackUnsubscribed(
|
||||
|
||||
func (t *telemetryService) TrackUnpublished(
|
||||
ctx context.Context,
|
||||
roomID livekit.RoomID,
|
||||
roomName livekit.RoomName,
|
||||
participantID livekit.ParticipantID,
|
||||
identity livekit.ParticipantIdentity,
|
||||
track *livekit.TrackInfo,
|
||||
@@ -352,7 +400,7 @@ func (t *telemetryService) TrackUnpublished(
|
||||
return
|
||||
}
|
||||
|
||||
room := t.getRoomDetails(participantID)
|
||||
room := toMinimalRoomProto(roomID, roomName)
|
||||
participant := &livekit.ParticipantInfo{
|
||||
Sid: string(participantID),
|
||||
Identity: string(identity),
|
||||
@@ -370,28 +418,34 @@ func (t *telemetryService) TrackUnpublished(
|
||||
|
||||
func (t *telemetryService) TrackMuted(
|
||||
ctx context.Context,
|
||||
roomID livekit.RoomID,
|
||||
roomName livekit.RoomName,
|
||||
participantID livekit.ParticipantID,
|
||||
track *livekit.TrackInfo,
|
||||
) {
|
||||
t.enqueue(func() {
|
||||
room := t.getRoomDetails(participantID)
|
||||
room := toMinimalRoomProto(roomID, roomName)
|
||||
t.SendEvent(ctx, newTrackEvent(livekit.AnalyticsEventType_TRACK_MUTED, room, participantID, track))
|
||||
})
|
||||
}
|
||||
|
||||
func (t *telemetryService) TrackUnmuted(
|
||||
ctx context.Context,
|
||||
roomID livekit.RoomID,
|
||||
roomName livekit.RoomName,
|
||||
participantID livekit.ParticipantID,
|
||||
track *livekit.TrackInfo,
|
||||
) {
|
||||
t.enqueue(func() {
|
||||
room := t.getRoomDetails(participantID)
|
||||
room := toMinimalRoomProto(roomID, roomName)
|
||||
t.SendEvent(ctx, newTrackEvent(livekit.AnalyticsEventType_TRACK_UNMUTED, room, participantID, track))
|
||||
})
|
||||
}
|
||||
|
||||
func (t *telemetryService) TrackPublishRTPStats(
|
||||
ctx context.Context,
|
||||
roomID livekit.RoomID,
|
||||
roomName livekit.RoomName,
|
||||
participantID livekit.ParticipantID,
|
||||
trackID livekit.TrackID,
|
||||
mimeType mime.MimeType,
|
||||
@@ -399,7 +453,7 @@ func (t *telemetryService) TrackPublishRTPStats(
|
||||
stats *livekit.RTPStats,
|
||||
) {
|
||||
t.enqueue(func() {
|
||||
room := t.getRoomDetails(participantID)
|
||||
room := toMinimalRoomProto(roomID, roomName)
|
||||
ev := newRoomEvent(livekit.AnalyticsEventType_TRACK_PUBLISH_STATS, room)
|
||||
ev.ParticipantId = string(participantID)
|
||||
ev.TrackId = string(trackID)
|
||||
@@ -412,13 +466,15 @@ func (t *telemetryService) TrackPublishRTPStats(
|
||||
|
||||
func (t *telemetryService) TrackSubscribeRTPStats(
|
||||
ctx context.Context,
|
||||
roomID livekit.RoomID,
|
||||
roomName livekit.RoomName,
|
||||
participantID livekit.ParticipantID,
|
||||
trackID livekit.TrackID,
|
||||
mimeType mime.MimeType,
|
||||
stats *livekit.RTPStats,
|
||||
) {
|
||||
t.enqueue(func() {
|
||||
room := t.getRoomDetails(participantID)
|
||||
room := toMinimalRoomProto(roomID, roomName)
|
||||
ev := newRoomEvent(livekit.AnalyticsEventType_TRACK_SUBSCRIBE_STATS, room)
|
||||
ev.ParticipantId = string(participantID)
|
||||
ev.TrackId = string(trackID)
|
||||
@@ -428,12 +484,19 @@ func (t *telemetryService) TrackSubscribeRTPStats(
|
||||
})
|
||||
}
|
||||
|
||||
func (t *telemetryService) NotifyEgressEvent(ctx context.Context, event string, info *livekit.EgressInfo) {
|
||||
opts := egress.GetEgressNotifyOptions(info)
|
||||
|
||||
t.NotifyEvent(ctx, &livekit.WebhookEvent{
|
||||
Event: event,
|
||||
EgressInfo: info,
|
||||
}, opts...)
|
||||
}
|
||||
|
||||
func (t *telemetryService) EgressStarted(ctx context.Context, info *livekit.EgressInfo) {
|
||||
|
||||
t.enqueue(func() {
|
||||
t.NotifyEvent(ctx, &livekit.WebhookEvent{
|
||||
Event: webhook.EventEgressStarted,
|
||||
EgressInfo: info,
|
||||
})
|
||||
t.NotifyEgressEvent(ctx, webhook.EventEgressStarted, info)
|
||||
|
||||
t.SendEvent(ctx, newEgressEvent(livekit.AnalyticsEventType_EGRESS_STARTED, info))
|
||||
})
|
||||
@@ -441,20 +504,15 @@ func (t *telemetryService) EgressStarted(ctx context.Context, info *livekit.Egre
|
||||
|
||||
func (t *telemetryService) EgressUpdated(ctx context.Context, info *livekit.EgressInfo) {
|
||||
t.enqueue(func() {
|
||||
t.NotifyEvent(ctx, &livekit.WebhookEvent{
|
||||
Event: webhook.EventEgressUpdated,
|
||||
EgressInfo: info,
|
||||
})
|
||||
t.NotifyEgressEvent(ctx, webhook.EventEgressUpdated, info)
|
||||
|
||||
t.SendEvent(ctx, newEgressEvent(livekit.AnalyticsEventType_EGRESS_UPDATED, info))
|
||||
})
|
||||
}
|
||||
|
||||
func (t *telemetryService) EgressEnded(ctx context.Context, info *livekit.EgressInfo) {
|
||||
t.enqueue(func() {
|
||||
t.NotifyEvent(ctx, &livekit.WebhookEvent{
|
||||
Event: webhook.EventEgressEnded,
|
||||
EgressInfo: info,
|
||||
})
|
||||
t.NotifyEgressEvent(ctx, webhook.EventEgressEnded, info)
|
||||
|
||||
t.SendEvent(ctx, newEgressEvent(livekit.AnalyticsEventType_EGRESS_ENDED, info))
|
||||
})
|
||||
@@ -533,19 +591,6 @@ func (t *telemetryService) Webhook(ctx context.Context, webhookInfo *livekit.Web
|
||||
})
|
||||
}
|
||||
|
||||
// returns a livekit.Room with only name and sid filled out
|
||||
// returns nil if room is not found
|
||||
func (t *telemetryService) getRoomDetails(participantID livekit.ParticipantID) *livekit.Room {
|
||||
if worker, ok := t.getWorker(participantID); ok {
|
||||
return &livekit.Room{
|
||||
Sid: string(worker.roomID),
|
||||
Name: string(worker.roomName),
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func newRoomEvent(event livekit.AnalyticsEventType, room *livekit.Room) *livekit.AnalyticsEvent {
|
||||
ev := &livekit.AnalyticsEvent{
|
||||
Type: event,
|
||||
@@ -596,3 +641,10 @@ func newIngressEvent(event livekit.AnalyticsEventType, ingress *livekit.IngressI
|
||||
Ingress: ingress,
|
||||
}
|
||||
}
|
||||
|
||||
func toMinimalRoomProto(roomID livekit.RoomID, roomName livekit.RoomName) *livekit.Room {
|
||||
return &livekit.Room{
|
||||
Sid: string(roomID),
|
||||
Name: string(roomName),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -21,6 +21,7 @@ import (
|
||||
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"github.com/livekit/livekit-server/pkg/telemetry"
|
||||
"github.com/livekit/protocol/livekit"
|
||||
)
|
||||
|
||||
@@ -46,9 +47,10 @@ func Test_OnParticipantJoin_EventIsSent(t *testing.T) {
|
||||
ClientConnectTime: 420,
|
||||
}
|
||||
participantInfo := &livekit.ParticipantInfo{Sid: partSID}
|
||||
guard := &telemetry.ReferenceGuard{}
|
||||
|
||||
// do
|
||||
fixture.sut.ParticipantJoined(context.Background(), room, participantInfo, clientInfo, clientMeta, true)
|
||||
fixture.sut.ParticipantJoined(context.Background(), room, participantInfo, clientInfo, clientMeta, true, guard)
|
||||
time.Sleep(time.Millisecond * 500)
|
||||
|
||||
// test
|
||||
@@ -81,10 +83,11 @@ func Test_OnParticipantLeft_EventIsSent(t *testing.T) {
|
||||
room := &livekit.Room{Sid: "RoomSid", Name: "RoomName"}
|
||||
partSID := "part1"
|
||||
participantInfo := &livekit.ParticipantInfo{Sid: partSID}
|
||||
guard := &telemetry.ReferenceGuard{}
|
||||
|
||||
// do
|
||||
fixture.sut.ParticipantActive(context.Background(), room, participantInfo, &livekit.AnalyticsClientMeta{}, false)
|
||||
fixture.sut.ParticipantLeft(context.Background(), room, participantInfo, true)
|
||||
fixture.sut.ParticipantActive(context.Background(), room, participantInfo, &livekit.AnalyticsClientMeta{}, false, guard)
|
||||
fixture.sut.ParticipantLeft(context.Background(), room, participantInfo, true, guard)
|
||||
time.Sleep(time.Millisecond * 500)
|
||||
|
||||
// test
|
||||
@@ -100,6 +103,8 @@ func Test_OnTrackUpdate_EventIsSent(t *testing.T) {
|
||||
fixture := createFixture()
|
||||
|
||||
// prepare
|
||||
roomID := "room1"
|
||||
roomName := "RoomName"
|
||||
partID := "part1"
|
||||
trackID := "track1"
|
||||
layer := &livekit.VideoLayer{
|
||||
@@ -119,7 +124,7 @@ func Test_OnTrackUpdate_EventIsSent(t *testing.T) {
|
||||
}
|
||||
|
||||
// do
|
||||
fixture.sut.TrackPublishedUpdate(context.Background(), livekit.ParticipantID(partID), trackInfo)
|
||||
fixture.sut.TrackPublishedUpdate(context.Background(), livekit.RoomID(roomID), livekit.RoomName(roomName), livekit.ParticipantID(partID), trackInfo)
|
||||
time.Sleep(time.Millisecond * 500)
|
||||
|
||||
// test
|
||||
@@ -158,9 +163,10 @@ func Test_OnParticipantActive_EventIsSent(t *testing.T) {
|
||||
ClientAddr: "127.0.0.1",
|
||||
}
|
||||
participantInfo := &livekit.ParticipantInfo{Sid: partSID}
|
||||
guard := &telemetry.ReferenceGuard{}
|
||||
|
||||
// do
|
||||
fixture.sut.ParticipantJoined(context.Background(), room, participantInfo, clientInfo, clientMeta, true)
|
||||
fixture.sut.ParticipantJoined(context.Background(), room, participantInfo, clientInfo, clientMeta, true, guard)
|
||||
time.Sleep(time.Millisecond * 500)
|
||||
|
||||
// test
|
||||
@@ -173,7 +179,7 @@ func Test_OnParticipantActive_EventIsSent(t *testing.T) {
|
||||
ClientConnectTime: 420,
|
||||
}
|
||||
|
||||
fixture.sut.ParticipantActive(context.Background(), room, participantInfo, clientMetaConnect, false)
|
||||
fixture.sut.ParticipantActive(context.Background(), room, participantInfo, clientMetaConnect, false, guard)
|
||||
time.Sleep(time.Millisecond * 500)
|
||||
|
||||
require.Equal(t, 2, fixture.analytics.SendEventCallCount())
|
||||
@@ -210,9 +216,10 @@ func Test_OnTrackSubscribed_EventIsSent(t *testing.T) {
|
||||
ClientAddr: "127.0.0.1",
|
||||
}
|
||||
participantInfo := &livekit.ParticipantInfo{Sid: partSID}
|
||||
guard := &telemetry.ReferenceGuard{}
|
||||
|
||||
// do
|
||||
fixture.sut.ParticipantJoined(context.Background(), room, participantInfo, clientInfo, clientMeta, true)
|
||||
fixture.sut.ParticipantJoined(context.Background(), room, participantInfo, clientInfo, clientMeta, true, guard)
|
||||
time.Sleep(time.Millisecond * 500)
|
||||
|
||||
// test
|
||||
@@ -221,7 +228,7 @@ func Test_OnTrackSubscribed_EventIsSent(t *testing.T) {
|
||||
require.Equal(t, room, event.Room)
|
||||
|
||||
// do
|
||||
fixture.sut.TrackSubscribed(context.Background(), livekit.ParticipantID(partSID), trackInfo, publisherInfo, true)
|
||||
fixture.sut.TrackSubscribed(context.Background(), livekit.RoomID(room.Sid), livekit.RoomName(room.Name), livekit.ParticipantID(partSID), trackInfo, publisherInfo, true)
|
||||
time.Sleep(time.Millisecond * 500)
|
||||
|
||||
require.Eventually(t, func() bool {
|
||||
|
||||
@@ -0,0 +1,40 @@
|
||||
// Copyright 2023 LiveKit, Inc.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package prometheus
|
||||
|
||||
import (
|
||||
"github.com/prometheus/client_golang/prometheus"
|
||||
|
||||
"github.com/livekit/protocol/livekit"
|
||||
)
|
||||
|
||||
var (
|
||||
refCounts *prometheus.GaugeVec
|
||||
)
|
||||
|
||||
func initDebugStats(nodeID string, nodeType livekit.NodeType) {
|
||||
refCounts = prometheus.NewGaugeVec(prometheus.GaugeOpts{
|
||||
Namespace: livekitNamespace,
|
||||
Subsystem: "debug",
|
||||
Name: "ref_count",
|
||||
ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String()},
|
||||
}, []string{"referrer"})
|
||||
|
||||
prometheus.MustRegister(refCounts)
|
||||
}
|
||||
|
||||
func AddRef(referrer string, n int) {
|
||||
refCounts.WithLabelValues(referrer).Add(float64(n))
|
||||
}
|
||||
+190
-157
@@ -17,35 +17,34 @@ package prometheus
|
||||
import (
|
||||
"time"
|
||||
|
||||
"github.com/mackerelio/go-osstat/memory"
|
||||
"github.com/prometheus/client_golang/prometheus"
|
||||
"github.com/twitchtv/twirp"
|
||||
"go.uber.org/atomic"
|
||||
|
||||
"github.com/livekit/protocol/livekit"
|
||||
"github.com/livekit/protocol/rpc"
|
||||
"github.com/livekit/protocol/utils/hwstats"
|
||||
"github.com/livekit/protocol/webhook"
|
||||
)
|
||||
|
||||
const (
|
||||
livekitNamespace string = "livekit"
|
||||
|
||||
statsUpdateInterval = time.Second * 10
|
||||
)
|
||||
|
||||
var (
|
||||
initialized atomic.Bool
|
||||
|
||||
MessageCounter *prometheus.CounterVec
|
||||
MessageBytes *prometheus.CounterVec
|
||||
ServiceOperationCounter *prometheus.CounterVec
|
||||
TwirpRequestStatusCounter *prometheus.CounterVec
|
||||
promMessageCounter *prometheus.CounterVec
|
||||
promServiceOperationCounter *prometheus.CounterVec
|
||||
promTwirpRequestStatusCounter *prometheus.CounterVec
|
||||
promTwirpRequestLatency *prometheus.HistogramVec
|
||||
|
||||
sysPacketsStart uint32
|
||||
sysDroppedPacketsStart uint32
|
||||
promSysPacketGauge *prometheus.GaugeVec
|
||||
promSysDroppedPacketPctGauge prometheus.Gauge
|
||||
sysPacketsStart uint32
|
||||
sysDroppedPacketsStart uint32
|
||||
promSysPacketGauge *prometheus.GaugeVec
|
||||
|
||||
cpuStats *hwstats.CPUStats
|
||||
cpuStats *hwstats.CPUStats
|
||||
memoryStats *hwstats.MemoryStats
|
||||
)
|
||||
|
||||
func Init(nodeID string, nodeType livekit.NodeType) error {
|
||||
@@ -53,27 +52,17 @@ func Init(nodeID string, nodeType livekit.NodeType) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
MessageCounter = prometheus.NewCounterVec(
|
||||
promMessageCounter = prometheus.NewCounterVec(
|
||||
prometheus.CounterOpts{
|
||||
Namespace: livekitNamespace,
|
||||
Subsystem: "node",
|
||||
Name: "messages",
|
||||
ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String()},
|
||||
},
|
||||
[]string{"type", "status"},
|
||||
[]string{"type", "status", "direction"},
|
||||
)
|
||||
|
||||
MessageBytes = prometheus.NewCounterVec(
|
||||
prometheus.CounterOpts{
|
||||
Namespace: livekitNamespace,
|
||||
Subsystem: "node",
|
||||
Name: "message_bytes",
|
||||
ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String()},
|
||||
},
|
||||
[]string{"type", "message_type"},
|
||||
)
|
||||
|
||||
ServiceOperationCounter = prometheus.NewCounterVec(
|
||||
promServiceOperationCounter = prometheus.NewCounterVec(
|
||||
prometheus.CounterOpts{
|
||||
Namespace: livekitNamespace,
|
||||
Subsystem: "node",
|
||||
@@ -83,7 +72,7 @@ func Init(nodeID string, nodeType livekit.NodeType) error {
|
||||
[]string{"type", "status", "error_type"},
|
||||
)
|
||||
|
||||
TwirpRequestStatusCounter = prometheus.NewCounterVec(
|
||||
promTwirpRequestStatusCounter = prometheus.NewCounterVec(
|
||||
prometheus.CounterOpts{
|
||||
Namespace: livekitNamespace,
|
||||
Subsystem: "node",
|
||||
@@ -93,6 +82,17 @@ func Init(nodeID string, nodeType livekit.NodeType) error {
|
||||
[]string{"service", "method", "status", "code"},
|
||||
)
|
||||
|
||||
promTwirpRequestLatency = prometheus.NewHistogramVec(
|
||||
prometheus.HistogramOpts{
|
||||
Namespace: livekitNamespace,
|
||||
Subsystem: "node",
|
||||
Name: "twirp_request_latency_ms",
|
||||
ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String()},
|
||||
Buckets: []float64{5, 10, 25, 50, 100, 250, 500, 1000, 2500, 5000, 10000, 30000, 60000},
|
||||
},
|
||||
[]string{"service", "method"},
|
||||
)
|
||||
|
||||
promSysPacketGauge = prometheus.NewGaugeVec(
|
||||
prometheus.GaugeOpts{
|
||||
Namespace: livekitNamespace,
|
||||
@@ -104,30 +104,21 @@ func Init(nodeID string, nodeType livekit.NodeType) error {
|
||||
[]string{"type"},
|
||||
)
|
||||
|
||||
promSysDroppedPacketPctGauge = prometheus.NewGauge(
|
||||
prometheus.GaugeOpts{
|
||||
Namespace: livekitNamespace,
|
||||
Subsystem: "node",
|
||||
Name: "dropped_packets",
|
||||
ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String()},
|
||||
Help: "System level dropped outgoing packet percentage.",
|
||||
},
|
||||
)
|
||||
|
||||
prometheus.MustRegister(MessageCounter)
|
||||
prometheus.MustRegister(MessageBytes)
|
||||
prometheus.MustRegister(ServiceOperationCounter)
|
||||
prometheus.MustRegister(TwirpRequestStatusCounter)
|
||||
prometheus.MustRegister(promMessageCounter)
|
||||
prometheus.MustRegister(promServiceOperationCounter)
|
||||
prometheus.MustRegister(promTwirpRequestStatusCounter)
|
||||
prometheus.MustRegister(promTwirpRequestLatency)
|
||||
prometheus.MustRegister(promSysPacketGauge)
|
||||
prometheus.MustRegister(promSysDroppedPacketPctGauge)
|
||||
|
||||
sysPacketsStart, sysDroppedPacketsStart, _ = getTCStats()
|
||||
|
||||
initPacketStats(nodeID, nodeType)
|
||||
initRoomStats(nodeID, nodeType)
|
||||
rpc.InitPSRPCStats(prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String()})
|
||||
webhook.InitWebhookStats(prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String()})
|
||||
initQualityStats(nodeID, nodeType)
|
||||
initDataPacketStats(nodeID, nodeType)
|
||||
initDebugStats(nodeID, nodeType)
|
||||
|
||||
var err error
|
||||
cpuStats, err = hwstats.NewCPUStats(nil)
|
||||
@@ -135,145 +126,187 @@ func Init(nodeID string, nodeType livekit.NodeType) error {
|
||||
return err
|
||||
}
|
||||
|
||||
memoryStats, err = hwstats.NewMemoryStats()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func GetUpdatedNodeStats(prev *livekit.NodeStats, prevAverage *livekit.NodeStats) (*livekit.NodeStats, bool, error) {
|
||||
func GetNodeStats(nodeStartedAt int64, prevStats []*livekit.NodeStats, rateIntervals []time.Duration) (*livekit.NodeStats, error) {
|
||||
loadAvg, err := getLoadAvg()
|
||||
if err != nil {
|
||||
return nil, false, err
|
||||
}
|
||||
|
||||
var cpuLoad float64
|
||||
cpuIdle := cpuStats.GetCPUIdle()
|
||||
if cpuIdle > 0 {
|
||||
cpuLoad = 1 - (cpuIdle / cpuStats.NumCPU())
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// On MacOS, get "\"vm_stat\": executable file not found in $PATH" although it is in /usr/bin
|
||||
// So, do not error out. Use the information if it is available.
|
||||
memTotal := uint64(0)
|
||||
memUsed := uint64(0)
|
||||
memInfo, _ := memory.Get()
|
||||
if memInfo != nil {
|
||||
memTotal = memInfo.Total
|
||||
memUsed = memInfo.Used
|
||||
}
|
||||
memUsed, memTotal, _ := memoryStats.GetMemory()
|
||||
|
||||
// do not error out, and use the information if it is available
|
||||
sysPackets, sysDroppedPackets, _ := getTCStats()
|
||||
promSysPacketGauge.WithLabelValues("out").Set(float64(sysPackets - sysPacketsStart))
|
||||
promSysPacketGauge.WithLabelValues("dropped").Set(float64(sysDroppedPackets - sysDroppedPacketsStart))
|
||||
|
||||
bytesInNow := bytesIn.Load()
|
||||
bytesOutNow := bytesOut.Load()
|
||||
packetsInNow := packetsIn.Load()
|
||||
packetsOutNow := packetsOut.Load()
|
||||
nackTotalNow := nackTotal.Load()
|
||||
retransmitBytesNow := retransmitBytes.Load()
|
||||
retransmitPacketsNow := retransmitPackets.Load()
|
||||
participantSignalConnectedNow := participantSignalConnected.Load()
|
||||
participantRTCInitNow := participantRTCInit.Load()
|
||||
participantRTConnectedCNow := participantRTCConnected.Load()
|
||||
trackPublishAttemptsNow := trackPublishAttempts.Load()
|
||||
trackPublishSuccessNow := trackPublishSuccess.Load()
|
||||
trackSubscribeAttemptsNow := trackSubscribeAttempts.Load()
|
||||
trackSubscribeSuccessNow := trackSubscribeSuccess.Load()
|
||||
forwardLatencyNow := forwardLatency.Load()
|
||||
forwardJitterNow := forwardJitter.Load()
|
||||
|
||||
updatedAt := time.Now().Unix()
|
||||
elapsed := updatedAt - prevAverage.UpdatedAt
|
||||
// include sufficient buffer to be sure a stats update had taken place
|
||||
computeAverage := elapsed > int64(statsUpdateInterval.Seconds()+2)
|
||||
if bytesInNow != prevAverage.BytesIn ||
|
||||
bytesOutNow != prevAverage.BytesOut ||
|
||||
packetsInNow != prevAverage.PacketsIn ||
|
||||
packetsOutNow != prevAverage.PacketsOut ||
|
||||
retransmitBytesNow != prevAverage.RetransmitBytesOut ||
|
||||
retransmitPacketsNow != prevAverage.RetransmitPacketsOut {
|
||||
computeAverage = true
|
||||
}
|
||||
|
||||
stats := &livekit.NodeStats{
|
||||
StartedAt: prev.StartedAt,
|
||||
UpdatedAt: updatedAt,
|
||||
NumRooms: roomCurrent.Load(),
|
||||
NumClients: participantCurrent.Load(),
|
||||
NumTracksIn: trackPublishedCurrent.Load(),
|
||||
NumTracksOut: trackSubscribedCurrent.Load(),
|
||||
NumTrackPublishAttempts: trackPublishAttemptsNow,
|
||||
NumTrackPublishSuccess: trackPublishSuccessNow,
|
||||
NumTrackSubscribeAttempts: trackSubscribeAttemptsNow,
|
||||
NumTrackSubscribeSuccess: trackSubscribeSuccessNow,
|
||||
BytesIn: bytesInNow,
|
||||
BytesOut: bytesOutNow,
|
||||
PacketsIn: packetsInNow,
|
||||
PacketsOut: packetsOutNow,
|
||||
RetransmitBytesOut: retransmitBytesNow,
|
||||
RetransmitPacketsOut: retransmitPacketsNow,
|
||||
NackTotal: nackTotalNow,
|
||||
ParticipantSignalConnected: participantSignalConnectedNow,
|
||||
ParticipantRtcInit: participantRTCInitNow,
|
||||
ParticipantRtcConnected: participantRTConnectedCNow,
|
||||
BytesInPerSec: prevAverage.BytesInPerSec,
|
||||
BytesOutPerSec: prevAverage.BytesOutPerSec,
|
||||
PacketsInPerSec: prevAverage.PacketsInPerSec,
|
||||
PacketsOutPerSec: prevAverage.PacketsOutPerSec,
|
||||
RetransmitBytesOutPerSec: prevAverage.RetransmitBytesOutPerSec,
|
||||
RetransmitPacketsOutPerSec: prevAverage.RetransmitPacketsOutPerSec,
|
||||
NackPerSec: prevAverage.NackPerSec,
|
||||
ForwardLatency: forwardLatencyNow,
|
||||
ForwardJitter: forwardJitterNow,
|
||||
ParticipantSignalConnectedPerSec: prevAverage.ParticipantSignalConnectedPerSec,
|
||||
ParticipantRtcInitPerSec: prevAverage.ParticipantRtcInitPerSec,
|
||||
ParticipantRtcConnectedPerSec: prevAverage.ParticipantRtcConnectedPerSec,
|
||||
NumCpus: uint32(cpuStats.NumCPU()), // this will round down to the nearest integer
|
||||
CpuLoad: float32(cpuLoad),
|
||||
MemoryTotal: memTotal,
|
||||
MemoryUsed: memUsed,
|
||||
LoadAvgLast1Min: float32(loadAvg.Loadavg1),
|
||||
LoadAvgLast5Min: float32(loadAvg.Loadavg5),
|
||||
LoadAvgLast15Min: float32(loadAvg.Loadavg15),
|
||||
SysPacketsOut: sysPackets,
|
||||
SysPacketsDropped: sysDroppedPackets,
|
||||
TrackPublishAttemptsPerSec: prevAverage.TrackPublishAttemptsPerSec,
|
||||
TrackPublishSuccessPerSec: prevAverage.TrackPublishSuccessPerSec,
|
||||
TrackSubscribeAttemptsPerSec: prevAverage.TrackSubscribeAttemptsPerSec,
|
||||
TrackSubscribeSuccessPerSec: prevAverage.TrackSubscribeSuccessPerSec,
|
||||
StartedAt: nodeStartedAt,
|
||||
UpdatedAt: time.Now().Unix(),
|
||||
NumRooms: roomCurrent.Load(),
|
||||
NumClients: participantCurrent.Load(),
|
||||
NumTracksIn: trackPublishedCurrent.Load(),
|
||||
NumTracksOut: trackSubscribedCurrent.Load(),
|
||||
NumTrackPublishAttempts: trackPublishAttempts.Load(),
|
||||
NumTrackPublishSuccess: trackPublishSuccess.Load(),
|
||||
NumTrackPublishCancels: trackPublishCancels.Load(),
|
||||
NumTrackSubscribeAttempts: trackSubscribeAttempts.Load(),
|
||||
NumTrackSubscribeSuccess: trackSubscribeSuccess.Load(),
|
||||
NumTrackSubscribeCancels: trackSubscribeCancels.Load(),
|
||||
BytesIn: bytesIn.Load(),
|
||||
BytesOut: bytesOut.Load(),
|
||||
PacketsIn: packetsIn.Load(),
|
||||
PacketsOut: packetsOut.Load(),
|
||||
RetransmitBytesOut: retransmitBytes.Load(),
|
||||
RetransmitPacketsOut: retransmitPackets.Load(),
|
||||
NackTotal: nackTotal.Load(),
|
||||
ParticipantSignalConnected: participantSignalConnected.Load(),
|
||||
ParticipantSignalFailed: participantSignalFailed.Load(),
|
||||
ParticipantSignalValidationFailed: participantSignalValidationFailed.Load(),
|
||||
ParticipantRtcInit: participantRTCInit.Load(),
|
||||
ParticipantRtcConnected: participantRTCConnected.Load(),
|
||||
ParticipantRtcCanceled: participantRTCCanceled.Load(),
|
||||
ParticipantRtcActive: participantRTCActive.Load(),
|
||||
ForwardLatency: forwardLatency.Load(),
|
||||
ForwardJitter: forwardJitter.Load(),
|
||||
NumCpus: uint32(cpuStats.NumCPU()), // this will round down to the nearest integer
|
||||
CpuLoad: float32(cpuStats.GetCPULoad()),
|
||||
MemoryTotal: memTotal,
|
||||
MemoryUsed: memUsed,
|
||||
LoadAvgLast1Min: float32(loadAvg.Loadavg1),
|
||||
LoadAvgLast5Min: float32(loadAvg.Loadavg5),
|
||||
LoadAvgLast15Min: float32(loadAvg.Loadavg15),
|
||||
SysPacketsOut: sysPackets,
|
||||
SysPacketsDropped: sysDroppedPackets,
|
||||
}
|
||||
|
||||
// update stats
|
||||
if computeAverage {
|
||||
stats.BytesInPerSec = perSec(prevAverage.BytesIn, bytesInNow, elapsed)
|
||||
stats.BytesOutPerSec = perSec(prevAverage.BytesOut, bytesOutNow, elapsed)
|
||||
stats.PacketsInPerSec = perSec(prevAverage.PacketsIn, packetsInNow, elapsed)
|
||||
stats.PacketsOutPerSec = perSec(prevAverage.PacketsOut, packetsOutNow, elapsed)
|
||||
stats.RetransmitBytesOutPerSec = perSec(prevAverage.RetransmitBytesOut, retransmitBytesNow, elapsed)
|
||||
stats.RetransmitPacketsOutPerSec = perSec(prevAverage.RetransmitPacketsOut, retransmitPacketsNow, elapsed)
|
||||
stats.NackPerSec = perSec(prevAverage.NackTotal, nackTotalNow, elapsed)
|
||||
stats.ParticipantSignalConnectedPerSec = perSec(prevAverage.ParticipantSignalConnected, participantSignalConnectedNow, elapsed)
|
||||
stats.ParticipantRtcInitPerSec = perSec(prevAverage.ParticipantRtcInit, participantRTCInitNow, elapsed)
|
||||
stats.ParticipantRtcConnectedPerSec = perSec(prevAverage.ParticipantRtcConnected, participantRTConnectedCNow, elapsed)
|
||||
stats.SysPacketsOutPerSec = perSec(uint64(prevAverage.SysPacketsOut), uint64(sysPackets), elapsed)
|
||||
stats.SysPacketsDroppedPerSec = perSec(uint64(prevAverage.SysPacketsDropped), uint64(sysDroppedPackets), elapsed)
|
||||
stats.TrackPublishAttemptsPerSec = perSec(uint64(prevAverage.NumTrackPublishAttempts), uint64(trackPublishAttemptsNow), elapsed)
|
||||
stats.TrackPublishSuccessPerSec = perSec(uint64(prevAverage.NumTrackPublishSuccess), uint64(trackPublishSuccessNow), elapsed)
|
||||
stats.TrackSubscribeAttemptsPerSec = perSec(uint64(prevAverage.NumTrackSubscribeAttempts), uint64(trackSubscribeAttemptsNow), elapsed)
|
||||
stats.TrackSubscribeSuccessPerSec = perSec(uint64(prevAverage.NumTrackSubscribeSuccess), uint64(trackSubscribeSuccessNow), elapsed)
|
||||
for _, rateInterval := range rateIntervals {
|
||||
for idx := len(prevStats) - 1; idx >= 0; idx-- {
|
||||
prev := prevStats[idx]
|
||||
if prev == nil {
|
||||
continue
|
||||
}
|
||||
|
||||
packetTotal := stats.SysPacketsOutPerSec + stats.SysPacketsDroppedPerSec
|
||||
if packetTotal == 0 {
|
||||
stats.SysPacketsDroppedPctPerSec = 0
|
||||
} else {
|
||||
stats.SysPacketsDroppedPctPerSec = stats.SysPacketsDroppedPerSec / packetTotal
|
||||
if stats.UpdatedAt-prev.UpdatedAt >= int64(rateInterval.Seconds()) {
|
||||
if rate := getNodeStatsRate(append(prevStats[idx:], stats)); rate != nil {
|
||||
stats.Rates = append(stats.Rates, rate)
|
||||
}
|
||||
break
|
||||
}
|
||||
}
|
||||
promSysDroppedPacketPctGauge.Set(float64(stats.SysPacketsDroppedPctPerSec))
|
||||
}
|
||||
|
||||
return stats, computeAverage, nil
|
||||
return stats, nil
|
||||
}
|
||||
|
||||
func getNodeStatsRate(statsHistory []*livekit.NodeStats) *livekit.NodeStatsRate {
|
||||
if len(statsHistory) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
elapsed := statsHistory[len(statsHistory)-1].UpdatedAt - statsHistory[0].UpdatedAt
|
||||
if elapsed <= 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
// time weighted averages
|
||||
var cpuLoad, memoryUsed, memoryTotal, memoryLoad float32
|
||||
for idx := len(statsHistory) - 1; idx > 0; idx-- {
|
||||
stats := statsHistory[idx]
|
||||
prevStats := statsHistory[idx-1]
|
||||
if stats == nil || prevStats == nil {
|
||||
continue
|
||||
}
|
||||
|
||||
spanElapsed := stats.UpdatedAt - prevStats.UpdatedAt
|
||||
if spanElapsed <= 0 {
|
||||
continue
|
||||
}
|
||||
|
||||
cpuLoad += stats.CpuLoad * float32(spanElapsed)
|
||||
memoryUsed += float32(stats.MemoryUsed) * float32(spanElapsed)
|
||||
memoryTotal += float32(stats.MemoryTotal) * float32(spanElapsed)
|
||||
if stats.MemoryTotal > 0 {
|
||||
memoryLoad += float32(stats.MemoryUsed) / float32(stats.MemoryTotal) * float32(spanElapsed)
|
||||
}
|
||||
}
|
||||
|
||||
earlier := statsHistory[0]
|
||||
later := statsHistory[len(statsHistory)-1]
|
||||
rate := &livekit.NodeStatsRate{
|
||||
StartedAt: earlier.UpdatedAt,
|
||||
EndedAt: later.UpdatedAt,
|
||||
Duration: elapsed,
|
||||
BytesIn: perSec(earlier.BytesIn, later.BytesIn, elapsed),
|
||||
BytesOut: perSec(earlier.BytesOut, later.BytesOut, elapsed),
|
||||
PacketsIn: perSec(earlier.PacketsIn, later.PacketsIn, elapsed),
|
||||
PacketsOut: perSec(earlier.PacketsOut, later.PacketsOut, elapsed),
|
||||
RetransmitBytesOut: perSec(earlier.RetransmitBytesOut, later.RetransmitBytesOut, elapsed),
|
||||
RetransmitPacketsOut: perSec(earlier.RetransmitPacketsOut, later.RetransmitPacketsOut, elapsed),
|
||||
NackTotal: perSec(earlier.NackTotal, later.NackTotal, elapsed),
|
||||
ParticipantSignalConnected: perSec(earlier.ParticipantSignalConnected, later.ParticipantSignalConnected, elapsed),
|
||||
ParticipantSignalFailed: perSec(earlier.ParticipantSignalFailed, later.ParticipantSignalFailed, elapsed),
|
||||
ParticipantSignalValidationFailed: perSec(earlier.ParticipantSignalValidationFailed, later.ParticipantSignalValidationFailed, elapsed),
|
||||
ParticipantRtcInit: perSec(earlier.ParticipantRtcInit, later.ParticipantRtcInit, elapsed),
|
||||
ParticipantRtcConnected: perSec(earlier.ParticipantRtcConnected, later.ParticipantRtcConnected, elapsed),
|
||||
ParticipantRtcCanceled: perSec(earlier.ParticipantRtcCanceled, later.ParticipantRtcCanceled, elapsed),
|
||||
ParticipantRtcActive: perSec(earlier.ParticipantRtcActive, later.ParticipantRtcActive, elapsed),
|
||||
SysPacketsOut: perSec(uint64(earlier.SysPacketsOut), uint64(later.SysPacketsOut), elapsed),
|
||||
SysPacketsDropped: perSec(uint64(earlier.SysPacketsDropped), uint64(later.SysPacketsDropped), elapsed),
|
||||
TrackPublishAttempts: perSec(uint64(earlier.NumTrackPublishAttempts), uint64(later.NumTrackPublishAttempts), elapsed),
|
||||
TrackPublishSuccess: perSec(uint64(earlier.NumTrackPublishSuccess), uint64(later.NumTrackPublishSuccess), elapsed),
|
||||
TrackPublishCancels: perSec(uint64(earlier.NumTrackPublishCancels), uint64(later.NumTrackPublishCancels), elapsed),
|
||||
TrackSubscribeAttempts: perSec(uint64(earlier.NumTrackSubscribeAttempts), uint64(later.NumTrackSubscribeAttempts), elapsed),
|
||||
TrackSubscribeSuccess: perSec(uint64(earlier.NumTrackSubscribeSuccess), uint64(later.NumTrackSubscribeSuccess), elapsed),
|
||||
TrackSubscribeCancels: perSec(uint64(earlier.NumTrackSubscribeCancels), uint64(later.NumTrackSubscribeCancels), elapsed),
|
||||
CpuLoad: cpuLoad / float32(elapsed),
|
||||
MemoryLoad: memoryLoad / float32(elapsed),
|
||||
MemoryUsed: memoryUsed / float32(elapsed),
|
||||
MemoryTotal: memoryTotal / float32(elapsed),
|
||||
}
|
||||
return rate
|
||||
}
|
||||
|
||||
func perSec(prev, curr uint64, secs int64) float32 {
|
||||
return float32(curr-prev) / float32(secs)
|
||||
}
|
||||
|
||||
func RecordSignalRequestSuccess() {
|
||||
promMessageCounter.WithLabelValues("signal", "success", "request").Add(1)
|
||||
}
|
||||
|
||||
func RecordSignalRequestFailure() {
|
||||
promMessageCounter.WithLabelValues("signal", "failure", "request").Add(1)
|
||||
}
|
||||
|
||||
func RecordSignalResponseSuccess() {
|
||||
promMessageCounter.WithLabelValues("signal", "success", "response").Add(1)
|
||||
}
|
||||
|
||||
func RecordSignalResponseFailure() {
|
||||
promMessageCounter.WithLabelValues("signal", "failure", "response").Add(1)
|
||||
}
|
||||
|
||||
func RecordServiceOperationSuccess(op string) {
|
||||
promServiceOperationCounter.WithLabelValues(op, "success", "").Add(1)
|
||||
}
|
||||
|
||||
func RecordServiceOperationError(op string, error string) {
|
||||
promServiceOperationCounter.WithLabelValues(op, "error", error).Add(1)
|
||||
}
|
||||
|
||||
func RecordTwirpRequestStatus(service string, method string, statusFamily string, code twirp.ErrorCode) {
|
||||
promTwirpRequestStatusCounter.WithLabelValues(service, method, statusFamily, string(code)).Add(1)
|
||||
}
|
||||
|
||||
func RecordTwirpRequestLatency(service, method string, duration time.Duration) {
|
||||
promTwirpRequestLatency.WithLabelValues(service, method).Observe(float64(duration.Milliseconds()))
|
||||
}
|
||||
|
||||
@@ -24,31 +24,40 @@ import (
|
||||
type Direction string
|
||||
|
||||
const (
|
||||
Incoming Direction = "incoming"
|
||||
Outgoing Direction = "outgoing"
|
||||
transmissionInitial = "initial"
|
||||
transmissionRetransmit = "retransmit"
|
||||
Incoming Direction = "incoming"
|
||||
Outgoing Direction = "outgoing"
|
||||
)
|
||||
|
||||
type TransmissionType string
|
||||
|
||||
const (
|
||||
TransmissionInitial TransmissionType = "initial"
|
||||
TransmissionRetransmit TransmissionType = "retransmit"
|
||||
)
|
||||
|
||||
var (
|
||||
bytesIn atomic.Uint64
|
||||
bytesOut atomic.Uint64
|
||||
packetsIn atomic.Uint64
|
||||
packetsOut atomic.Uint64
|
||||
nackTotal atomic.Uint64
|
||||
retransmitBytes atomic.Uint64
|
||||
retransmitPackets atomic.Uint64
|
||||
participantSignalConnected atomic.Uint64
|
||||
participantRTCConnected atomic.Uint64
|
||||
participantRTCInit atomic.Uint64
|
||||
forwardLatency atomic.Uint32
|
||||
forwardJitter atomic.Uint32
|
||||
bytesIn atomic.Uint64
|
||||
bytesOut atomic.Uint64
|
||||
packetsIn atomic.Uint64
|
||||
packetsOut atomic.Uint64
|
||||
nackTotal atomic.Uint64
|
||||
retransmitBytes atomic.Uint64
|
||||
retransmitPackets atomic.Uint64
|
||||
participantSignalConnected atomic.Uint64
|
||||
participantSignalFailed atomic.Uint64
|
||||
participantSignalValidationFailed atomic.Uint64
|
||||
participantRTCConnected atomic.Uint64
|
||||
participantRTCInit atomic.Uint64
|
||||
participantRTCCanceled atomic.Uint64
|
||||
participantRTCActive atomic.Uint64
|
||||
forwardLatency atomic.Uint32
|
||||
forwardJitter atomic.Uint32
|
||||
|
||||
promPacketLabels = []string{"direction", "transmission"}
|
||||
promPacketLabels = []string{"direction", "transmission", "country"}
|
||||
promPacketTotal *prometheus.CounterVec
|
||||
promPacketBytes *prometheus.CounterVec
|
||||
promRTCPLabels = []string{"direction"}
|
||||
promStreamLabels = []string{"direction", "source", "type"}
|
||||
promRTCPLabels = []string{"direction", "country"}
|
||||
promStreamLabels = []string{"direction", "source", "type", "country"}
|
||||
promNackTotal *prometheus.CounterVec
|
||||
promPliTotal *prometheus.CounterVec
|
||||
promFirTotal *prometheus.CounterVec
|
||||
@@ -62,15 +71,7 @@ var (
|
||||
promConnections *prometheus.GaugeVec
|
||||
promForwardLatency prometheus.Gauge
|
||||
promForwardJitter prometheus.Gauge
|
||||
|
||||
promPacketTotalIncomingInitial prometheus.Counter
|
||||
promPacketTotalIncomingRetransmit prometheus.Counter
|
||||
promPacketTotalOutgoingInitial prometheus.Counter
|
||||
promPacketTotalOutgoingRetransmit prometheus.Counter
|
||||
promPacketBytesIncomingInitial prometheus.Counter
|
||||
promPacketBytesIncomingRetransmit prometheus.Counter
|
||||
promPacketBytesOutgoingInitial prometheus.Counter
|
||||
promPacketBytesOutgoingRetransmit prometheus.Counter
|
||||
promForwardLatencyHist prometheus.Histogram
|
||||
)
|
||||
|
||||
func initPacketStats(nodeID string, nodeType livekit.NodeType) {
|
||||
@@ -135,7 +136,6 @@ func initPacketStats(nodeID string, nodeType livekit.NodeType) {
|
||||
Subsystem: "jitter",
|
||||
Name: "us",
|
||||
ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String()},
|
||||
|
||||
// 1ms, 10ms, 30ms, 50ms, 70ms, 100ms, 300ms, 600ms, 1s
|
||||
Buckets: []float64{1000, 10000, 30000, 50000, 70000, 100000, 300000, 600000, 1000000},
|
||||
}, promStreamLabels)
|
||||
@@ -170,6 +170,25 @@ func initPacketStats(nodeID string, nodeType livekit.NodeType) {
|
||||
Name: "jitter",
|
||||
ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String()},
|
||||
})
|
||||
promForwardLatencyHist = prometheus.NewHistogram(prometheus.HistogramOpts{
|
||||
Namespace: livekitNamespace,
|
||||
Subsystem: "forward_latency",
|
||||
Name: "ns",
|
||||
ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String()},
|
||||
// 50us, 100us, 250us, 500us, 1ms, 2ms, 3ms, 5ms, 10ms, 20ms
|
||||
Buckets: []float64{
|
||||
50 * 1000,
|
||||
100 * 1000,
|
||||
250 * 1000,
|
||||
500 * 1000,
|
||||
1 * 1000 * 1000,
|
||||
2 * 1000 * 1000,
|
||||
3 * 1000 * 1000,
|
||||
5 * 1000 * 1000,
|
||||
10 * 1000 * 1000,
|
||||
20 * 1000 * 1000,
|
||||
},
|
||||
})
|
||||
|
||||
prometheus.MustRegister(promPacketTotal)
|
||||
prometheus.MustRegister(promPacketBytes)
|
||||
@@ -186,31 +205,17 @@ func initPacketStats(nodeID string, nodeType livekit.NodeType) {
|
||||
prometheus.MustRegister(promConnections)
|
||||
prometheus.MustRegister(promForwardLatency)
|
||||
prometheus.MustRegister(promForwardJitter)
|
||||
|
||||
promPacketTotalIncomingInitial = promPacketTotal.WithLabelValues(string(Incoming), transmissionInitial)
|
||||
promPacketTotalIncomingRetransmit = promPacketTotal.WithLabelValues(string(Incoming), transmissionRetransmit)
|
||||
promPacketTotalOutgoingInitial = promPacketTotal.WithLabelValues(string(Outgoing), transmissionInitial)
|
||||
promPacketTotalOutgoingRetransmit = promPacketTotal.WithLabelValues(string(Outgoing), transmissionRetransmit)
|
||||
promPacketBytesIncomingInitial = promPacketBytes.WithLabelValues(string(Incoming), transmissionInitial)
|
||||
promPacketBytesIncomingRetransmit = promPacketBytes.WithLabelValues(string(Incoming), transmissionRetransmit)
|
||||
promPacketBytesOutgoingInitial = promPacketBytes.WithLabelValues(string(Outgoing), transmissionInitial)
|
||||
promPacketBytesOutgoingRetransmit = promPacketBytes.WithLabelValues(string(Outgoing), transmissionRetransmit)
|
||||
prometheus.MustRegister(promForwardLatencyHist)
|
||||
}
|
||||
|
||||
func IncrementPackets(direction Direction, count uint64, retransmit bool) {
|
||||
if direction == Incoming {
|
||||
if retransmit {
|
||||
promPacketTotalIncomingRetransmit.Add(float64(count))
|
||||
} else {
|
||||
promPacketTotalIncomingInitial.Add(float64(count))
|
||||
}
|
||||
func IncrementPackets(country string, direction Direction, count uint64, retransmit bool) {
|
||||
var transmission TransmissionType
|
||||
if retransmit {
|
||||
transmission = TransmissionRetransmit
|
||||
} else {
|
||||
if retransmit {
|
||||
promPacketTotalOutgoingRetransmit.Add(float64(count))
|
||||
} else {
|
||||
promPacketTotalOutgoingInitial.Add(float64(count))
|
||||
}
|
||||
transmission = TransmissionInitial
|
||||
}
|
||||
promPacketTotal.WithLabelValues(string(direction), string(transmission), country).Add(float64(count))
|
||||
|
||||
if direction == Incoming {
|
||||
packetsIn.Add(count)
|
||||
@@ -222,20 +227,14 @@ func IncrementPackets(direction Direction, count uint64, retransmit bool) {
|
||||
}
|
||||
}
|
||||
|
||||
func IncrementBytes(direction Direction, count uint64, retransmit bool) {
|
||||
if direction == Incoming {
|
||||
if retransmit {
|
||||
promPacketBytesIncomingRetransmit.Add(float64(count))
|
||||
} else {
|
||||
promPacketBytesIncomingInitial.Add(float64(count))
|
||||
}
|
||||
func IncrementBytes(country string, direction Direction, count uint64, retransmit bool) {
|
||||
var transmission TransmissionType
|
||||
if retransmit {
|
||||
transmission = TransmissionRetransmit
|
||||
} else {
|
||||
if retransmit {
|
||||
promPacketBytesOutgoingRetransmit.Add(float64(count))
|
||||
} else {
|
||||
promPacketBytesOutgoingInitial.Add(float64(count))
|
||||
}
|
||||
transmission = TransmissionInitial
|
||||
}
|
||||
promPacketBytes.WithLabelValues(string(direction), string(transmission), country).Add(float64(count))
|
||||
|
||||
if direction == Incoming {
|
||||
bytesIn.Add(count)
|
||||
@@ -247,46 +246,53 @@ func IncrementBytes(direction Direction, count uint64, retransmit bool) {
|
||||
}
|
||||
}
|
||||
|
||||
func IncrementRTCP(direction Direction, nack, pli, fir uint32) {
|
||||
func IncrementRTCP(country string, direction Direction, nack, pli, fir uint32) {
|
||||
if nack > 0 {
|
||||
promNackTotal.WithLabelValues(string(direction)).Add(float64(nack))
|
||||
promNackTotal.WithLabelValues(string(direction), country).Add(float64(nack))
|
||||
nackTotal.Add(uint64(nack))
|
||||
}
|
||||
if pli > 0 {
|
||||
promPliTotal.WithLabelValues(string(direction)).Add(float64(pli))
|
||||
promPliTotal.WithLabelValues(string(direction), country).Add(float64(pli))
|
||||
}
|
||||
if fir > 0 {
|
||||
promFirTotal.WithLabelValues(string(direction)).Add(float64(fir))
|
||||
promFirTotal.WithLabelValues(string(direction), country).Add(float64(fir))
|
||||
}
|
||||
}
|
||||
|
||||
func RecordPacketLoss(direction Direction, trackSource livekit.TrackSource, trackType livekit.TrackType, lost, total uint32) {
|
||||
func RecordPacketLoss(
|
||||
country string,
|
||||
direction Direction,
|
||||
trackSource livekit.TrackSource,
|
||||
trackType livekit.TrackType,
|
||||
lost uint32,
|
||||
total uint32,
|
||||
) {
|
||||
if total > 0 {
|
||||
promPacketLoss.WithLabelValues(string(direction), trackSource.String(), trackType.String()).Observe(float64(lost) / float64(total) * 100)
|
||||
promPacketLoss.WithLabelValues(string(direction), trackSource.String(), trackType.String(), country).Observe(float64(lost) / float64(total) * 100)
|
||||
}
|
||||
if lost > 0 {
|
||||
promPacketLossTotal.WithLabelValues(string(direction), trackSource.String(), trackType.String()).Add(float64(lost))
|
||||
promPacketLossTotal.WithLabelValues(string(direction), trackSource.String(), trackType.String(), country).Add(float64(lost))
|
||||
}
|
||||
}
|
||||
|
||||
func RecordPacketOutOfOrder(direction Direction, trackSource livekit.TrackSource, trackType livekit.TrackType, ooo, total uint32) {
|
||||
func RecordPacketOutOfOrder(country string, direction Direction, trackSource livekit.TrackSource, trackType livekit.TrackType, ooo, total uint32) {
|
||||
if total > 0 {
|
||||
promPacketOutOfOrder.WithLabelValues(string(direction), trackSource.String(), trackType.String()).Observe(float64(ooo) / float64(total) * 100)
|
||||
promPacketOutOfOrder.WithLabelValues(string(direction), trackSource.String(), trackType.String(), country).Observe(float64(ooo) / float64(total) * 100)
|
||||
}
|
||||
if ooo > 0 {
|
||||
promPacketOutOfOrderTotal.WithLabelValues(string(direction), trackSource.String(), trackType.String()).Add(float64(ooo))
|
||||
promPacketOutOfOrderTotal.WithLabelValues(string(direction), trackSource.String(), trackType.String(), country).Add(float64(ooo))
|
||||
}
|
||||
}
|
||||
|
||||
func RecordJitter(direction Direction, trackSource livekit.TrackSource, trackType livekit.TrackType, jitter uint32) {
|
||||
func RecordJitter(country string, direction Direction, trackSource livekit.TrackSource, trackType livekit.TrackType, jitter uint32) {
|
||||
if jitter > 0 {
|
||||
promJitter.WithLabelValues(string(direction), trackSource.String(), trackType.String()).Observe(float64(jitter))
|
||||
promJitter.WithLabelValues(string(direction), trackSource.String(), trackType.String(), country).Observe(float64(jitter))
|
||||
}
|
||||
}
|
||||
|
||||
func RecordRTT(direction Direction, trackSource livekit.TrackSource, trackType livekit.TrackType, rtt uint32) {
|
||||
func RecordRTT(country string, direction Direction, trackSource livekit.TrackSource, trackType livekit.TrackType, rtt uint32) {
|
||||
if rtt > 0 {
|
||||
promRTT.WithLabelValues(string(direction), trackSource.String(), trackType.String()).Observe(float64(rtt))
|
||||
promRTT.WithLabelValues(string(direction), trackSource.String(), trackType.String(), country).Observe(float64(rtt))
|
||||
}
|
||||
}
|
||||
|
||||
@@ -297,9 +303,17 @@ func IncrementParticipantJoin(join uint32) {
|
||||
}
|
||||
}
|
||||
|
||||
func IncrementParticipantJoinFail(join uint32) {
|
||||
if join > 0 {
|
||||
promParticipantJoin.WithLabelValues("signal_failed").Add(float64(join))
|
||||
func IncrementParticipantJoinFail(fail uint32) {
|
||||
if fail > 0 {
|
||||
participantSignalFailed.Add(uint64(fail))
|
||||
promParticipantJoin.WithLabelValues("signal_failed").Add(float64(fail))
|
||||
}
|
||||
}
|
||||
|
||||
func IncrementParticipantJoinValidationFail(validationFail uint32) {
|
||||
if validationFail > 0 {
|
||||
participantSignalValidationFailed.Add(uint64(validationFail))
|
||||
promParticipantJoin.WithLabelValues("signal_validation_failed").Add(float64(validationFail))
|
||||
}
|
||||
}
|
||||
|
||||
@@ -317,6 +331,20 @@ func IncrementParticipantRtcConnected(join uint32) {
|
||||
}
|
||||
}
|
||||
|
||||
func IncrementParticipantRtcActive(active uint32) {
|
||||
if active > 0 {
|
||||
participantRTCActive.Add(uint64(active))
|
||||
promParticipantJoin.WithLabelValues("rtc_active").Add(float64(active))
|
||||
}
|
||||
}
|
||||
|
||||
func IncrementParticipantRtcCanceled(numCancels uint64) {
|
||||
if numCancels > 0 {
|
||||
participantRTCCanceled.Add(numCancels)
|
||||
promParticipantJoin.WithLabelValues("rtc_canceled").Add(float64(numCancels))
|
||||
}
|
||||
}
|
||||
|
||||
func AddConnection(direction Direction) {
|
||||
promConnections.WithLabelValues(string(direction)).Add(1)
|
||||
}
|
||||
@@ -325,12 +353,16 @@ func SubConnection(direction Direction) {
|
||||
promConnections.WithLabelValues(string(direction)).Sub(1)
|
||||
}
|
||||
|
||||
func RecordForwardLatency(_, latencyAvg uint32) {
|
||||
forwardLatency.Store(latencyAvg)
|
||||
promForwardLatency.Set(float64(latencyAvg))
|
||||
func RecordForwardLatencySample(forwardLatency int64) {
|
||||
promForwardLatencyHist.Observe(float64(forwardLatency))
|
||||
}
|
||||
|
||||
func RecordForwardJitter(_, jitterAvg uint32) {
|
||||
forwardJitter.Store(jitterAvg)
|
||||
promForwardJitter.Set(float64(jitterAvg))
|
||||
func RecordForwardLatency(longTermLatencyAvg uint32) {
|
||||
forwardLatency.Store(longTermLatencyAvg)
|
||||
promForwardLatency.Set(float64(longTermLatencyAvg))
|
||||
}
|
||||
|
||||
func RecordForwardJitter(longTermJitterAvg uint32) {
|
||||
forwardJitter.Store(longTermJitterAvg)
|
||||
promForwardJitter.Set(float64(longTermJitterAvg))
|
||||
}
|
||||
|
||||
@@ -31,8 +31,10 @@ var (
|
||||
trackSubscribedCurrent atomic.Int32
|
||||
trackPublishAttempts atomic.Int32
|
||||
trackPublishSuccess atomic.Int32
|
||||
trackPublishCancels atomic.Int32
|
||||
trackSubscribeAttempts atomic.Int32
|
||||
trackSubscribeSuccess atomic.Int32
|
||||
trackSubscribeCancels atomic.Int32
|
||||
// count the number of failures that are due to user error (permissions, track doesn't exist), so we could compute
|
||||
// success rate by subtracting this from total attempts
|
||||
trackSubscribeUserError atomic.Int32
|
||||
@@ -162,30 +164,67 @@ func SubPublishedTrack(kind string) {
|
||||
trackPublishedCurrent.Dec()
|
||||
}
|
||||
|
||||
func AddPublishAttempt(kind string) {
|
||||
func RecordTrackPublishAttempt(kind string) {
|
||||
trackPublishAttempts.Inc()
|
||||
promTrackPublishCounter.WithLabelValues(kind, "attempt").Inc()
|
||||
}
|
||||
|
||||
func AddPublishSuccess(kind string) {
|
||||
func RecordTrackPublishSuccess(kind string) {
|
||||
trackPublishSuccess.Inc()
|
||||
promTrackPublishCounter.WithLabelValues(kind, "success").Inc()
|
||||
}
|
||||
|
||||
func RecordPublishTime(source livekit.TrackSource, trackType livekit.TrackType, d time.Duration, sdk livekit.ClientInfo_SDK, kind livekit.ParticipantInfo_Kind) {
|
||||
recordPubSubTime(true, source, trackType, d, sdk, kind, 1)
|
||||
func RecordTrackPublishCancels(kind string, numCancels int32) {
|
||||
trackPublishCancels.Add(numCancels)
|
||||
promTrackPublishCounter.WithLabelValues(kind, "cancel").Add(float64(numCancels))
|
||||
}
|
||||
|
||||
func RecordSubscribeTime(source livekit.TrackSource, trackType livekit.TrackType, d time.Duration, sdk livekit.ClientInfo_SDK, kind livekit.ParticipantInfo_Kind, count int) {
|
||||
recordPubSubTime(false, source, trackType, d, sdk, kind, count)
|
||||
func RecordPublishTime(
|
||||
country string,
|
||||
source livekit.TrackSource,
|
||||
trackType livekit.TrackType,
|
||||
d time.Duration,
|
||||
sdk livekit.ClientInfo_SDK,
|
||||
kind livekit.ParticipantInfo_Kind,
|
||||
) {
|
||||
recordPubSubTime(true, country, source, trackType, d, sdk, kind, 1)
|
||||
}
|
||||
|
||||
func recordPubSubTime(isPublish bool, source livekit.TrackSource, trackType livekit.TrackType, d time.Duration, sdk livekit.ClientInfo_SDK, kind livekit.ParticipantInfo_Kind, count int) {
|
||||
func RecordSubscribeTime(
|
||||
country string,
|
||||
source livekit.TrackSource,
|
||||
trackType livekit.TrackType,
|
||||
d time.Duration,
|
||||
sdk livekit.ClientInfo_SDK,
|
||||
kind livekit.ParticipantInfo_Kind,
|
||||
count int,
|
||||
) {
|
||||
recordPubSubTime(false, country, source, trackType, d, sdk, kind, count)
|
||||
}
|
||||
|
||||
func recordPubSubTime(
|
||||
isPublish bool,
|
||||
country string,
|
||||
source livekit.TrackSource,
|
||||
trackType livekit.TrackType,
|
||||
d time.Duration,
|
||||
sdk livekit.ClientInfo_SDK,
|
||||
kind livekit.ParticipantInfo_Kind,
|
||||
count int,
|
||||
) {
|
||||
direction := "subscribe"
|
||||
if isPublish {
|
||||
direction = "publish"
|
||||
}
|
||||
promPubSubTime.WithLabelValues(direction, source.String(), trackType.String(), sdk.String(), kind.String(), strconv.Itoa(count)).Observe(float64(d.Milliseconds()))
|
||||
promPubSubTime.WithLabelValues(
|
||||
direction,
|
||||
source.String(),
|
||||
trackType.String(),
|
||||
country,
|
||||
sdk.String(),
|
||||
kind.String(),
|
||||
strconv.Itoa(count),
|
||||
).Observe(float64(d.Milliseconds()))
|
||||
}
|
||||
|
||||
func RecordTrackSubscribeSuccess(kind string) {
|
||||
@@ -214,9 +253,15 @@ func RecordTrackSubscribeFailure(err error, isUserError bool) {
|
||||
|
||||
if isUserError {
|
||||
trackSubscribeUserError.Inc()
|
||||
trackSubscribeCancels.Inc()
|
||||
}
|
||||
}
|
||||
|
||||
func RecordTrackSubscribeCancels(numCancels int32) {
|
||||
trackSubscribeCancels.Add(numCancels)
|
||||
promTrackSubscribeCounter.WithLabelValues("cancel", "").Add(float64(numCancels))
|
||||
}
|
||||
|
||||
func RecordSessionStartTime(protocolVersion int, d time.Duration) {
|
||||
promSessionStartTime.WithLabelValues(strconv.Itoa(protocolVersion)).Observe(float64(d.Milliseconds()))
|
||||
}
|
||||
|
||||
@@ -1,205 +0,0 @@
|
||||
// Copyright 2023 LiveKit, Inc.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package telemetry
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/frostbyte73/core"
|
||||
"go.uber.org/atomic"
|
||||
|
||||
"github.com/livekit/protocol/livekit"
|
||||
"github.com/livekit/protocol/utils"
|
||||
)
|
||||
|
||||
type BytesTrackType string
|
||||
|
||||
const (
|
||||
BytesTrackTypeData BytesTrackType = "DT"
|
||||
BytesTrackTypeSignal BytesTrackType = "SG"
|
||||
)
|
||||
|
||||
// -------------------------------
|
||||
|
||||
type TrafficTotals struct {
|
||||
At time.Time
|
||||
SendBytes uint64
|
||||
SendMessages uint32
|
||||
RecvBytes uint64
|
||||
RecvMessages uint32
|
||||
}
|
||||
|
||||
// --------------------------------
|
||||
|
||||
// stats for signal and data channel
|
||||
type BytesTrackStats struct {
|
||||
trackID livekit.TrackID
|
||||
pID livekit.ParticipantID
|
||||
send, recv atomic.Uint64
|
||||
sendMessages, recvMessages atomic.Uint32
|
||||
totalSendBytes, totalRecvBytes atomic.Uint64
|
||||
totalSendMessages, totalRecvMessages atomic.Uint32
|
||||
telemetry TelemetryService
|
||||
done core.Fuse
|
||||
}
|
||||
|
||||
func NewBytesTrackStats(trackID livekit.TrackID, pID livekit.ParticipantID, telemetry TelemetryService) *BytesTrackStats {
|
||||
s := &BytesTrackStats{
|
||||
trackID: trackID,
|
||||
pID: pID,
|
||||
telemetry: telemetry,
|
||||
}
|
||||
go s.reporter()
|
||||
return s
|
||||
}
|
||||
|
||||
func (s *BytesTrackStats) AddBytes(bytes uint64, isSend bool) {
|
||||
if isSend {
|
||||
s.send.Add(bytes)
|
||||
s.sendMessages.Inc()
|
||||
s.totalSendBytes.Add(bytes)
|
||||
s.totalSendMessages.Inc()
|
||||
} else {
|
||||
s.recv.Add(bytes)
|
||||
s.recvMessages.Inc()
|
||||
s.totalRecvBytes.Add(bytes)
|
||||
s.totalRecvMessages.Inc()
|
||||
}
|
||||
}
|
||||
|
||||
func (s *BytesTrackStats) GetTrafficTotals() *TrafficTotals {
|
||||
return &TrafficTotals{
|
||||
At: time.Now(),
|
||||
SendBytes: s.totalSendBytes.Load(),
|
||||
SendMessages: s.totalSendMessages.Load(),
|
||||
RecvBytes: s.totalRecvBytes.Load(),
|
||||
RecvMessages: s.totalRecvMessages.Load(),
|
||||
}
|
||||
}
|
||||
|
||||
func (s *BytesTrackStats) Stop() {
|
||||
s.done.Break()
|
||||
}
|
||||
|
||||
func (s *BytesTrackStats) report() {
|
||||
if recv := s.recv.Swap(0); recv > 0 {
|
||||
s.telemetry.TrackStats(StatsKeyForData(livekit.StreamType_UPSTREAM, s.pID, s.trackID), &livekit.AnalyticsStat{
|
||||
Streams: []*livekit.AnalyticsStream{
|
||||
{
|
||||
PrimaryBytes: recv,
|
||||
PrimaryPackets: s.recvMessages.Swap(0),
|
||||
},
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
if send := s.send.Swap(0); send > 0 {
|
||||
s.telemetry.TrackStats(StatsKeyForData(livekit.StreamType_DOWNSTREAM, s.pID, s.trackID), &livekit.AnalyticsStat{
|
||||
Streams: []*livekit.AnalyticsStream{
|
||||
{
|
||||
PrimaryBytes: send,
|
||||
PrimaryPackets: s.sendMessages.Swap(0),
|
||||
},
|
||||
},
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func (s *BytesTrackStats) reporter() {
|
||||
ticker := time.NewTicker(telemetryNonMediaStatsUpdateInterval)
|
||||
defer func() {
|
||||
ticker.Stop()
|
||||
s.report()
|
||||
}()
|
||||
|
||||
for {
|
||||
select {
|
||||
case <-s.done.Watch():
|
||||
return
|
||||
case <-ticker.C:
|
||||
s.report()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// -----------------------------------------------------------------------
|
||||
|
||||
type BytesSignalStats struct {
|
||||
BytesTrackStats
|
||||
ctx context.Context
|
||||
|
||||
mu sync.Mutex
|
||||
ri *livekit.Room
|
||||
pi *livekit.ParticipantInfo
|
||||
}
|
||||
|
||||
func NewBytesSignalStats(ctx context.Context, telemetry TelemetryService) *BytesSignalStats {
|
||||
return &BytesSignalStats{
|
||||
BytesTrackStats: BytesTrackStats{
|
||||
telemetry: telemetry,
|
||||
},
|
||||
ctx: ctx,
|
||||
}
|
||||
}
|
||||
|
||||
func (s *BytesSignalStats) ResolveRoom(ri *livekit.Room) {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
if s.ri == nil && ri.GetSid() != "" {
|
||||
s.ri = &livekit.Room{
|
||||
Sid: ri.Sid,
|
||||
Name: ri.Name,
|
||||
}
|
||||
s.maybeStart()
|
||||
}
|
||||
}
|
||||
|
||||
func (s *BytesSignalStats) ResolveParticipant(pi *livekit.ParticipantInfo) {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
if s.pi == nil && pi != nil {
|
||||
s.pi = &livekit.ParticipantInfo{
|
||||
Sid: pi.Sid,
|
||||
Identity: pi.Identity,
|
||||
}
|
||||
s.maybeStart()
|
||||
}
|
||||
}
|
||||
|
||||
func (s *BytesSignalStats) maybeStart() {
|
||||
if s.ri == nil || s.pi == nil {
|
||||
return
|
||||
}
|
||||
|
||||
s.pID = livekit.ParticipantID(s.pi.Sid)
|
||||
s.trackID = BytesTrackIDForParticipantID(BytesTrackTypeSignal, s.pID)
|
||||
|
||||
s.telemetry.ParticipantJoined(s.ctx, s.ri, s.pi, nil, nil, false)
|
||||
go s.reporter()
|
||||
}
|
||||
|
||||
func (s *BytesSignalStats) reporter() {
|
||||
s.BytesTrackStats.reporter()
|
||||
s.telemetry.ParticipantLeft(s.ctx, s.ri, s.pi, false)
|
||||
}
|
||||
|
||||
// -----------------------------------------------------------------------
|
||||
|
||||
func BytesTrackIDForParticipantID(typ BytesTrackType, participantID livekit.ParticipantID) livekit.TrackID {
|
||||
return livekit.TrackID(fmt.Sprintf("%s_%s%s", utils.TrackPrefix, string(typ), participantID))
|
||||
}
|
||||
+28
-13
@@ -20,6 +20,7 @@ import (
|
||||
)
|
||||
|
||||
type StatsKey struct {
|
||||
country string
|
||||
streamType livekit.StreamType
|
||||
participantID livekit.ParticipantID
|
||||
trackID livekit.TrackID
|
||||
@@ -28,8 +29,16 @@ type StatsKey struct {
|
||||
track bool
|
||||
}
|
||||
|
||||
func StatsKeyForTrack(streamType livekit.StreamType, participantID livekit.ParticipantID, trackID livekit.TrackID, trackSource livekit.TrackSource, trackType livekit.TrackType) StatsKey {
|
||||
func StatsKeyForTrack(
|
||||
country string,
|
||||
streamType livekit.StreamType,
|
||||
participantID livekit.ParticipantID,
|
||||
trackID livekit.TrackID,
|
||||
trackSource livekit.TrackSource,
|
||||
trackType livekit.TrackType,
|
||||
) StatsKey {
|
||||
return StatsKey{
|
||||
country: country,
|
||||
streamType: streamType,
|
||||
participantID: participantID,
|
||||
trackID: trackID,
|
||||
@@ -39,15 +48,21 @@ func StatsKeyForTrack(streamType livekit.StreamType, participantID livekit.Parti
|
||||
}
|
||||
}
|
||||
|
||||
func StatsKeyForData(streamType livekit.StreamType, participantID livekit.ParticipantID, trackID livekit.TrackID) StatsKey {
|
||||
func StatsKeyForData(
|
||||
country string,
|
||||
streamType livekit.StreamType,
|
||||
participantID livekit.ParticipantID,
|
||||
trackID livekit.TrackID,
|
||||
) StatsKey {
|
||||
return StatsKey{
|
||||
country: country,
|
||||
streamType: streamType,
|
||||
participantID: participantID,
|
||||
trackID: trackID,
|
||||
}
|
||||
}
|
||||
|
||||
func (t *telemetryService) TrackStats(key StatsKey, stat *livekit.AnalyticsStat) {
|
||||
func (t *telemetryService) TrackStats(roomID livekit.RoomID, _roomName livekit.RoomName, key StatsKey, stat *livekit.AnalyticsStat) {
|
||||
t.enqueue(func() {
|
||||
direction := prometheus.Incoming
|
||||
if key.streamType == livekit.StreamType_DOWNSTREAM {
|
||||
@@ -76,23 +91,23 @@ func (t *telemetryService) TrackStats(key StatsKey, stat *livekit.AnalyticsStat)
|
||||
bytes += stream.RetransmitBytes
|
||||
}
|
||||
if key.track {
|
||||
prometheus.RecordPacketLoss(direction, key.trackSource, key.trackType, stream.PacketsLost, stream.PrimaryPackets+stream.PaddingPackets)
|
||||
prometheus.RecordPacketOutOfOrder(direction, key.trackSource, key.trackType, stream.PacketsOutOfOrder, stream.PrimaryPackets+stream.PaddingPackets)
|
||||
prometheus.RecordRTT(direction, key.trackSource, key.trackType, stream.Rtt)
|
||||
prometheus.RecordJitter(direction, key.trackSource, key.trackType, stream.Jitter)
|
||||
prometheus.RecordPacketLoss(key.country, direction, key.trackSource, key.trackType, stream.PacketsLost, stream.PrimaryPackets+stream.PaddingPackets)
|
||||
prometheus.RecordPacketOutOfOrder(key.country, direction, key.trackSource, key.trackType, stream.PacketsOutOfOrder, stream.PrimaryPackets+stream.PaddingPackets)
|
||||
prometheus.RecordRTT(key.country, direction, key.trackSource, key.trackType, stream.Rtt)
|
||||
prometheus.RecordJitter(key.country, direction, key.trackSource, key.trackType, stream.Jitter)
|
||||
}
|
||||
}
|
||||
prometheus.IncrementRTCP(direction, nacks, plis, firs)
|
||||
prometheus.IncrementPackets(direction, uint64(packets), false)
|
||||
prometheus.IncrementBytes(direction, bytes, false)
|
||||
prometheus.IncrementRTCP(key.country, direction, nacks, plis, firs)
|
||||
prometheus.IncrementPackets(key.country, direction, uint64(packets), false)
|
||||
prometheus.IncrementBytes(key.country, direction, bytes, false)
|
||||
if retransmitPackets != 0 {
|
||||
prometheus.IncrementPackets(direction, uint64(retransmitPackets), true)
|
||||
prometheus.IncrementPackets(key.country, direction, uint64(retransmitPackets), true)
|
||||
}
|
||||
if retransmitBytes != 0 {
|
||||
prometheus.IncrementBytes(direction, retransmitBytes, true)
|
||||
prometheus.IncrementBytes(key.country, direction, retransmitBytes, true)
|
||||
}
|
||||
|
||||
if worker, ok := t.getWorker(key.participantID); ok {
|
||||
if worker, ok := t.getWorker(roomID, key.participantID); ok {
|
||||
worker.OnTrackStat(key.trackID, key.streamType, stat)
|
||||
}
|
||||
})
|
||||
|
||||
+57
-45
@@ -52,12 +52,13 @@ func Test_ParticipantAndRoomDataAreSentWithAnalytics(t *testing.T) {
|
||||
partSID := livekit.ParticipantID("part1")
|
||||
clientInfo := &livekit.ClientInfo{Sdk: 2}
|
||||
participantInfo := &livekit.ParticipantInfo{Sid: string(partSID)}
|
||||
fixture.sut.ParticipantJoined(context.Background(), room, participantInfo, clientInfo, nil, true)
|
||||
guard := &telemetry.ReferenceGuard{}
|
||||
fixture.sut.ParticipantJoined(context.Background(), room, participantInfo, clientInfo, nil, true, guard)
|
||||
|
||||
// do
|
||||
packet := 33
|
||||
stat := &livekit.AnalyticsStat{Streams: []*livekit.AnalyticsStream{{PrimaryBytes: uint64(packet)}}}
|
||||
fixture.sut.TrackStats(telemetry.StatsKeyForData(livekit.StreamType_DOWNSTREAM, partSID, ""), stat)
|
||||
fixture.sut.TrackStats(livekit.RoomID(room.Sid), livekit.RoomName(room.Name), telemetry.StatsKeyForData("test", livekit.StreamType_DOWNSTREAM, partSID, ""), stat)
|
||||
|
||||
// flush
|
||||
fixture.flush()
|
||||
@@ -76,11 +77,12 @@ func Test_OnDownstreamPackets(t *testing.T) {
|
||||
fixture := createFixture()
|
||||
|
||||
// prepare
|
||||
room := &livekit.Room{}
|
||||
room := &livekit.Room{Sid: "RoomSid", Name: "RoomName"}
|
||||
partSID := livekit.ParticipantID("part1")
|
||||
clientInfo := &livekit.ClientInfo{Sdk: 2}
|
||||
participantInfo := &livekit.ParticipantInfo{Sid: string(partSID)}
|
||||
fixture.sut.ParticipantJoined(context.Background(), room, participantInfo, clientInfo, nil, true)
|
||||
guard := &telemetry.ReferenceGuard{}
|
||||
fixture.sut.ParticipantJoined(context.Background(), room, participantInfo, clientInfo, nil, true, guard)
|
||||
|
||||
// do
|
||||
packets := []int{33, 23}
|
||||
@@ -89,7 +91,7 @@ func Test_OnDownstreamPackets(t *testing.T) {
|
||||
trackID := livekit.TrackID("trackID")
|
||||
for i := range packets {
|
||||
stat := &livekit.AnalyticsStat{Streams: []*livekit.AnalyticsStream{{PrimaryBytes: uint64(packets[i]), PrimaryPackets: uint32(1)}}}
|
||||
fixture.sut.TrackStats(telemetry.StatsKeyForData(livekit.StreamType_DOWNSTREAM, partSID, trackID), stat)
|
||||
fixture.sut.TrackStats(livekit.RoomID(room.Sid), livekit.RoomName(room.Name), telemetry.StatsKeyForData("test", livekit.StreamType_DOWNSTREAM, partSID, trackID), stat)
|
||||
}
|
||||
|
||||
// flush
|
||||
@@ -109,22 +111,23 @@ func Test_OnDownstreamPackets_SeveralTracks(t *testing.T) {
|
||||
fixture := createFixture()
|
||||
|
||||
// prepare
|
||||
room := &livekit.Room{}
|
||||
room := &livekit.Room{Sid: "RoomSid", Name: "RoomName"}
|
||||
partSID := livekit.ParticipantID("part1")
|
||||
clientInfo := &livekit.ClientInfo{Sdk: 2}
|
||||
participantInfo := &livekit.ParticipantInfo{Sid: string(partSID)}
|
||||
fixture.sut.ParticipantJoined(context.Background(), room, participantInfo, clientInfo, nil, true)
|
||||
guard := &telemetry.ReferenceGuard{}
|
||||
fixture.sut.ParticipantJoined(context.Background(), room, participantInfo, clientInfo, nil, true, guard)
|
||||
|
||||
// do
|
||||
packet1 := 33
|
||||
trackID1 := livekit.TrackID("trackID1")
|
||||
stat1 := &livekit.AnalyticsStat{Streams: []*livekit.AnalyticsStream{{PrimaryBytes: uint64(packet1), PrimaryPackets: 1}}}
|
||||
fixture.sut.TrackStats(telemetry.StatsKeyForData(livekit.StreamType_DOWNSTREAM, partSID, trackID1), stat1)
|
||||
fixture.sut.TrackStats(livekit.RoomID(room.Sid), livekit.RoomName(room.Name), telemetry.StatsKeyForData("test", livekit.StreamType_DOWNSTREAM, partSID, trackID1), stat1)
|
||||
|
||||
packet2 := 23
|
||||
trackID2 := livekit.TrackID("trackID2")
|
||||
stat2 := &livekit.AnalyticsStat{Streams: []*livekit.AnalyticsStream{{PrimaryBytes: uint64(packet2), PrimaryPackets: 1}}}
|
||||
fixture.sut.TrackStats(telemetry.StatsKeyForData(livekit.StreamType_DOWNSTREAM, partSID, trackID2), stat2)
|
||||
fixture.sut.TrackStats(livekit.RoomID(room.Sid), livekit.RoomName(room.Name), telemetry.StatsKeyForData("test", livekit.StreamType_DOWNSTREAM, partSID, trackID2), stat2)
|
||||
|
||||
// flush
|
||||
fixture.flush()
|
||||
@@ -155,10 +158,11 @@ func Test_OnDownStreamStat(t *testing.T) {
|
||||
fixture := createFixture()
|
||||
|
||||
// prepare
|
||||
room := &livekit.Room{}
|
||||
room := &livekit.Room{Sid: "RoomSid", Name: "RoomName"}
|
||||
partSID := livekit.ParticipantID("part1")
|
||||
participantInfo := &livekit.ParticipantInfo{Sid: string(partSID)}
|
||||
fixture.sut.ParticipantJoined(context.Background(), room, participantInfo, nil, nil, true)
|
||||
guard := &telemetry.ReferenceGuard{}
|
||||
fixture.sut.ParticipantJoined(context.Background(), room, participantInfo, nil, nil, true, guard)
|
||||
|
||||
// do
|
||||
stat1 := &livekit.AnalyticsStat{
|
||||
@@ -175,7 +179,7 @@ func Test_OnDownStreamStat(t *testing.T) {
|
||||
},
|
||||
}
|
||||
trackID := livekit.TrackID("trackID1")
|
||||
fixture.sut.TrackStats(telemetry.StatsKeyForData(livekit.StreamType_DOWNSTREAM, partSID, trackID), stat1)
|
||||
fixture.sut.TrackStats(livekit.RoomID(room.Sid), livekit.RoomName(room.Name), telemetry.StatsKeyForData("test", livekit.StreamType_DOWNSTREAM, partSID, trackID), stat1)
|
||||
|
||||
stat2 := &livekit.AnalyticsStat{
|
||||
Streams: []*livekit.AnalyticsStream{
|
||||
@@ -191,7 +195,7 @@ func Test_OnDownStreamStat(t *testing.T) {
|
||||
},
|
||||
},
|
||||
}
|
||||
fixture.sut.TrackStats(telemetry.StatsKeyForData(livekit.StreamType_DOWNSTREAM, partSID, trackID), stat2)
|
||||
fixture.sut.TrackStats(livekit.RoomID(room.Sid), livekit.RoomName(room.Name), telemetry.StatsKeyForData("test", livekit.StreamType_DOWNSTREAM, partSID, trackID), stat2)
|
||||
|
||||
// flush
|
||||
fixture.flush()
|
||||
@@ -214,10 +218,11 @@ func Test_PacketLostDiffShouldBeSentToTelemetry(t *testing.T) {
|
||||
fixture := createFixture()
|
||||
|
||||
// prepare
|
||||
room := &livekit.Room{}
|
||||
room := &livekit.Room{Sid: "RoomSid", Name: "RoomName"}
|
||||
partSID := livekit.ParticipantID("part1")
|
||||
participantInfo := &livekit.ParticipantInfo{Sid: string(partSID)}
|
||||
fixture.sut.ParticipantJoined(context.Background(), room, participantInfo, nil, nil, true)
|
||||
guard := &telemetry.ReferenceGuard{}
|
||||
fixture.sut.ParticipantJoined(context.Background(), room, participantInfo, nil, nil, true, guard)
|
||||
|
||||
// do
|
||||
trackID := livekit.TrackID("trackID1")
|
||||
@@ -230,7 +235,7 @@ func Test_PacketLostDiffShouldBeSentToTelemetry(t *testing.T) {
|
||||
},
|
||||
},
|
||||
}
|
||||
fixture.sut.TrackStats(telemetry.StatsKeyForData(livekit.StreamType_DOWNSTREAM, partSID, trackID), stat1) // there should be bytes reported so that stats are sent
|
||||
fixture.sut.TrackStats(livekit.RoomID(room.Sid), livekit.RoomName(room.Name), telemetry.StatsKeyForData("test", livekit.StreamType_DOWNSTREAM, partSID, trackID), stat1) // there should be bytes reported so that stats are sent
|
||||
|
||||
// flush
|
||||
fixture.flush()
|
||||
@@ -244,7 +249,7 @@ func Test_PacketLostDiffShouldBeSentToTelemetry(t *testing.T) {
|
||||
},
|
||||
},
|
||||
}
|
||||
fixture.sut.TrackStats(telemetry.StatsKeyForData(livekit.StreamType_DOWNSTREAM, partSID, trackID), stat2)
|
||||
fixture.sut.TrackStats(livekit.RoomID(room.Sid), livekit.RoomName(room.Name), telemetry.StatsKeyForData("test", livekit.StreamType_DOWNSTREAM, partSID, trackID), stat2)
|
||||
|
||||
// flush
|
||||
fixture.flush()
|
||||
@@ -266,10 +271,11 @@ func Test_OnDownStreamRTCP_SeveralTracks(t *testing.T) {
|
||||
fixture := createFixture()
|
||||
|
||||
// prepare
|
||||
room := &livekit.Room{}
|
||||
room := &livekit.Room{Sid: "RoomSid", Name: "RoomName"}
|
||||
partSID := livekit.ParticipantID("part1")
|
||||
participantInfo := &livekit.ParticipantInfo{Sid: string(partSID)}
|
||||
fixture.sut.ParticipantJoined(context.Background(), room, participantInfo, nil, nil, true)
|
||||
guard := &telemetry.ReferenceGuard{}
|
||||
fixture.sut.ParticipantJoined(context.Background(), room, participantInfo, nil, nil, true, guard)
|
||||
|
||||
// do
|
||||
trackID1 := livekit.TrackID("trackID1")
|
||||
@@ -282,7 +288,7 @@ func Test_OnDownStreamRTCP_SeveralTracks(t *testing.T) {
|
||||
},
|
||||
},
|
||||
}
|
||||
fixture.sut.TrackStats(telemetry.StatsKeyForData(livekit.StreamType_DOWNSTREAM, partSID, trackID1), stat1) // there should be bytes reported so that stats are sent
|
||||
fixture.sut.TrackStats(livekit.RoomID(room.Sid), livekit.RoomName(room.Name), telemetry.StatsKeyForData("test", livekit.StreamType_DOWNSTREAM, partSID, trackID1), stat1) // there should be bytes reported so that stats are sent
|
||||
|
||||
stat2 := &livekit.AnalyticsStat{
|
||||
Streams: []*livekit.AnalyticsStream{
|
||||
@@ -293,7 +299,7 @@ func Test_OnDownStreamRTCP_SeveralTracks(t *testing.T) {
|
||||
},
|
||||
},
|
||||
}
|
||||
fixture.sut.TrackStats(telemetry.StatsKeyForData(livekit.StreamType_DOWNSTREAM, partSID, trackID1), stat2)
|
||||
fixture.sut.TrackStats(livekit.RoomID(room.Sid), livekit.RoomName(room.Name), telemetry.StatsKeyForData("test", livekit.StreamType_DOWNSTREAM, partSID, trackID1), stat2)
|
||||
|
||||
stat3 := &livekit.AnalyticsStat{
|
||||
Streams: []*livekit.AnalyticsStream{
|
||||
@@ -304,7 +310,7 @@ func Test_OnDownStreamRTCP_SeveralTracks(t *testing.T) {
|
||||
},
|
||||
},
|
||||
}
|
||||
fixture.sut.TrackStats(telemetry.StatsKeyForData(livekit.StreamType_DOWNSTREAM, partSID, trackID2), stat3)
|
||||
fixture.sut.TrackStats(livekit.RoomID(room.Sid), livekit.RoomName(room.Name), telemetry.StatsKeyForData("test", livekit.StreamType_DOWNSTREAM, partSID, trackID2), stat3)
|
||||
|
||||
// flush
|
||||
fixture.flush()
|
||||
@@ -335,10 +341,11 @@ func Test_OnUpstreamStat(t *testing.T) {
|
||||
fixture := createFixture()
|
||||
|
||||
// prepare
|
||||
room := &livekit.Room{}
|
||||
room := &livekit.Room{Sid: "RoomSid", Name: "RoomName"}
|
||||
partSID := livekit.ParticipantID("part1")
|
||||
participantInfo := &livekit.ParticipantInfo{Sid: string(partSID)}
|
||||
fixture.sut.ParticipantJoined(context.Background(), room, participantInfo, nil, nil, true)
|
||||
guard := &telemetry.ReferenceGuard{}
|
||||
fixture.sut.ParticipantJoined(context.Background(), room, participantInfo, nil, nil, true, guard)
|
||||
|
||||
// do
|
||||
stat1 := &livekit.AnalyticsStat{
|
||||
@@ -357,7 +364,7 @@ func Test_OnUpstreamStat(t *testing.T) {
|
||||
}
|
||||
trackID := livekit.TrackID("trackID")
|
||||
|
||||
fixture.sut.TrackStats(telemetry.StatsKeyForData(livekit.StreamType_UPSTREAM, partSID, trackID), stat1)
|
||||
fixture.sut.TrackStats(livekit.RoomID(room.Sid), livekit.RoomName(room.Name), telemetry.StatsKeyForData("test", livekit.StreamType_UPSTREAM, partSID, trackID), stat1)
|
||||
|
||||
stat2 := &livekit.AnalyticsStat{
|
||||
Streams: []*livekit.AnalyticsStream{
|
||||
@@ -373,7 +380,7 @@ func Test_OnUpstreamStat(t *testing.T) {
|
||||
},
|
||||
},
|
||||
}
|
||||
fixture.sut.TrackStats(telemetry.StatsKeyForData(livekit.StreamType_UPSTREAM, partSID, trackID), stat2)
|
||||
fixture.sut.TrackStats(livekit.RoomID(room.Sid), livekit.RoomName(room.Name), telemetry.StatsKeyForData("test", livekit.StreamType_UPSTREAM, partSID, trackID), stat2)
|
||||
|
||||
// flush
|
||||
fixture.flush()
|
||||
@@ -396,11 +403,12 @@ func Test_OnUpstreamRTCP_SeveralTracks(t *testing.T) {
|
||||
fixture := createFixture()
|
||||
|
||||
// prepare
|
||||
room := &livekit.Room{}
|
||||
room := &livekit.Room{Sid: "RoomSid", Name: "RoomName"}
|
||||
partSID := livekit.ParticipantID("part1")
|
||||
identity := livekit.ParticipantIdentity("part1Identity")
|
||||
participantInfo := &livekit.ParticipantInfo{Sid: string(partSID), Identity: string(identity)}
|
||||
fixture.sut.ParticipantJoined(context.Background(), room, participantInfo, nil, nil, true)
|
||||
guard := &telemetry.ReferenceGuard{}
|
||||
fixture.sut.ParticipantJoined(context.Background(), room, participantInfo, nil, nil, true, guard)
|
||||
|
||||
// there should be bytes reported so that stats are sent
|
||||
totalBytes := 1
|
||||
@@ -416,8 +424,8 @@ func Test_OnUpstreamRTCP_SeveralTracks(t *testing.T) {
|
||||
},
|
||||
},
|
||||
}
|
||||
fixture.sut.TrackStats(telemetry.StatsKeyForData(livekit.StreamType_UPSTREAM, partSID, trackID1), stat1)
|
||||
fixture.sut.TrackStats(telemetry.StatsKeyForData(livekit.StreamType_UPSTREAM, partSID, trackID2), stat1) // using same buffer is not correct but for test it is fine
|
||||
fixture.sut.TrackStats(livekit.RoomID(room.Sid), livekit.RoomName(room.Name), telemetry.StatsKeyForData("test", livekit.StreamType_UPSTREAM, partSID, trackID1), stat1)
|
||||
fixture.sut.TrackStats(livekit.RoomID(room.Sid), livekit.RoomName(room.Name), telemetry.StatsKeyForData("test", livekit.StreamType_UPSTREAM, partSID, trackID2), stat1) // using same buffer is not correct but for test it is fine
|
||||
|
||||
// do
|
||||
totalBytes++
|
||||
@@ -431,7 +439,7 @@ func Test_OnUpstreamRTCP_SeveralTracks(t *testing.T) {
|
||||
},
|
||||
},
|
||||
}
|
||||
fixture.sut.TrackStats(telemetry.StatsKeyForData(livekit.StreamType_UPSTREAM, partSID, trackID1), stat2)
|
||||
fixture.sut.TrackStats(livekit.RoomID(room.Sid), livekit.RoomName(room.Name), telemetry.StatsKeyForData("test", livekit.StreamType_UPSTREAM, partSID, trackID1), stat2)
|
||||
|
||||
stat3 := &livekit.AnalyticsStat{
|
||||
Streams: []*livekit.AnalyticsStream{
|
||||
@@ -442,7 +450,7 @@ func Test_OnUpstreamRTCP_SeveralTracks(t *testing.T) {
|
||||
},
|
||||
},
|
||||
}
|
||||
fixture.sut.TrackStats(telemetry.StatsKeyForData(livekit.StreamType_UPSTREAM, partSID, trackID2), stat3)
|
||||
fixture.sut.TrackStats(livekit.RoomID(room.Sid), livekit.RoomName(room.Name), telemetry.StatsKeyForData("test", livekit.StreamType_UPSTREAM, partSID, trackID2), stat3)
|
||||
|
||||
// flush
|
||||
fixture.flush()
|
||||
@@ -471,7 +479,7 @@ func Test_OnUpstreamRTCP_SeveralTracks(t *testing.T) {
|
||||
require.True(t, found2)
|
||||
|
||||
// remove 1 track - track stats were flushed above, so no more calls to SendStats
|
||||
fixture.sut.TrackUnpublished(context.Background(), partSID, identity, &livekit.TrackInfo{Sid: string(trackID2)}, true)
|
||||
fixture.sut.TrackUnpublished(context.Background(), livekit.RoomID(room.Sid), livekit.RoomName(room.Name), partSID, identity, &livekit.TrackInfo{Sid: string(trackID2)}, true)
|
||||
|
||||
// flush
|
||||
fixture.flush()
|
||||
@@ -486,10 +494,11 @@ func Test_AnalyticsSentWhenParticipantLeaves(t *testing.T) {
|
||||
room := &livekit.Room{}
|
||||
partSID := "part1"
|
||||
participantInfo := &livekit.ParticipantInfo{Sid: partSID}
|
||||
fixture.sut.ParticipantJoined(context.Background(), room, participantInfo, nil, nil, true)
|
||||
guard := &telemetry.ReferenceGuard{}
|
||||
fixture.sut.ParticipantJoined(context.Background(), room, participantInfo, nil, nil, true, guard)
|
||||
|
||||
// do
|
||||
fixture.sut.ParticipantLeft(context.Background(), room, participantInfo, true)
|
||||
fixture.sut.ParticipantLeft(context.Background(), room, participantInfo, true, guard)
|
||||
|
||||
// should not be called if there are no track stats
|
||||
time.Sleep(time.Millisecond * 500)
|
||||
@@ -500,10 +509,11 @@ func Test_AddUpTrack(t *testing.T) {
|
||||
fixture := createFixture()
|
||||
|
||||
// prepare
|
||||
room := &livekit.Room{}
|
||||
room := &livekit.Room{Sid: "RoomSid", Name: "RoomName"}
|
||||
partSID := livekit.ParticipantID("part1")
|
||||
participantInfo := &livekit.ParticipantInfo{Sid: string(partSID)}
|
||||
fixture.sut.ParticipantJoined(context.Background(), room, participantInfo, nil, nil, true)
|
||||
guard := &telemetry.ReferenceGuard{}
|
||||
fixture.sut.ParticipantJoined(context.Background(), room, participantInfo, nil, nil, true, guard)
|
||||
|
||||
// do
|
||||
var totalBytes uint64 = 3
|
||||
@@ -518,7 +528,7 @@ func Test_AddUpTrack(t *testing.T) {
|
||||
},
|
||||
}
|
||||
trackID := livekit.TrackID("trackID")
|
||||
fixture.sut.TrackStats(telemetry.StatsKeyForData(livekit.StreamType_UPSTREAM, partSID, trackID), stat)
|
||||
fixture.sut.TrackStats(livekit.RoomID(room.Sid), livekit.RoomName(room.Name), telemetry.StatsKeyForData("test", livekit.StreamType_UPSTREAM, partSID, trackID), stat)
|
||||
|
||||
// flush
|
||||
fixture.flush()
|
||||
@@ -537,10 +547,11 @@ func Test_AddUpTrack_SeveralBuffers_Simulcast(t *testing.T) {
|
||||
fixture := createFixture()
|
||||
|
||||
// prepare
|
||||
room := &livekit.Room{}
|
||||
room := &livekit.Room{Sid: "RoomSid", Name: "RoomName"}
|
||||
partSID := livekit.ParticipantID("part1")
|
||||
participantInfo := &livekit.ParticipantInfo{Sid: string(partSID)}
|
||||
fixture.sut.ParticipantJoined(context.Background(), room, participantInfo, nil, nil, true)
|
||||
guard := &telemetry.ReferenceGuard{}
|
||||
fixture.sut.ParticipantJoined(context.Background(), room, participantInfo, nil, nil, true, guard)
|
||||
|
||||
// do
|
||||
trackID := livekit.TrackID("trackID")
|
||||
@@ -556,7 +567,7 @@ func Test_AddUpTrack_SeveralBuffers_Simulcast(t *testing.T) {
|
||||
},
|
||||
},
|
||||
}
|
||||
fixture.sut.TrackStats(telemetry.StatsKeyForData(livekit.StreamType_UPSTREAM, partSID, trackID), stat1)
|
||||
fixture.sut.TrackStats(livekit.RoomID(room.Sid), livekit.RoomName(room.Name), telemetry.StatsKeyForData("test", livekit.StreamType_UPSTREAM, partSID, trackID), stat1)
|
||||
|
||||
// flush
|
||||
fixture.flush()
|
||||
@@ -576,10 +587,11 @@ func Test_BothDownstreamAndUpstreamStatsAreSentTogether(t *testing.T) {
|
||||
fixture := createFixture()
|
||||
|
||||
// prepare
|
||||
room := &livekit.Room{}
|
||||
room := &livekit.Room{Sid: "RoomSid", Name: "RoomName"}
|
||||
partSID := livekit.ParticipantID("part1")
|
||||
participantInfo := &livekit.ParticipantInfo{Sid: string(partSID)}
|
||||
fixture.sut.ParticipantJoined(context.Background(), room, participantInfo, nil, nil, true)
|
||||
guard := &telemetry.ReferenceGuard{}
|
||||
fixture.sut.ParticipantJoined(context.Background(), room, participantInfo, nil, nil, true, guard)
|
||||
|
||||
// do
|
||||
// upstream bytes
|
||||
@@ -591,7 +603,7 @@ func Test_BothDownstreamAndUpstreamStatsAreSentTogether(t *testing.T) {
|
||||
},
|
||||
},
|
||||
}
|
||||
fixture.sut.TrackStats(telemetry.StatsKeyForData(livekit.StreamType_UPSTREAM, partSID, "trackID"), stat1)
|
||||
fixture.sut.TrackStats(livekit.RoomID(room.Sid), livekit.RoomName(room.Name), telemetry.StatsKeyForData("test", livekit.StreamType_UPSTREAM, partSID, "trackID"), stat1)
|
||||
// downstream bytes
|
||||
stat2 := &livekit.AnalyticsStat{
|
||||
Streams: []*livekit.AnalyticsStream{
|
||||
@@ -601,7 +613,7 @@ func Test_BothDownstreamAndUpstreamStatsAreSentTogether(t *testing.T) {
|
||||
},
|
||||
},
|
||||
}
|
||||
fixture.sut.TrackStats(telemetry.StatsKeyForData(livekit.StreamType_DOWNSTREAM, partSID, "trackID1"), stat2)
|
||||
fixture.sut.TrackStats(livekit.RoomID(room.Sid), livekit.RoomName(room.Name), telemetry.StatsKeyForData("test", livekit.StreamType_DOWNSTREAM, partSID, "trackID1"), stat2)
|
||||
|
||||
// flush
|
||||
fixture.flush()
|
||||
|
||||
+23
-12
@@ -17,7 +17,7 @@ package telemetry
|
||||
import (
|
||||
"net"
|
||||
|
||||
"github.com/pion/turn/v4"
|
||||
"github.com/pion/turn/v5"
|
||||
|
||||
"github.com/livekit/livekit-server/pkg/telemetry/prometheus"
|
||||
)
|
||||
@@ -52,7 +52,8 @@ func NewConn(c net.Conn, direction prometheus.Direction) *Conn {
|
||||
func (c *Conn) Read(b []byte) (n int, err error) {
|
||||
n, err = c.Conn.Read(b)
|
||||
if n > 0 {
|
||||
prometheus.IncrementBytes(prometheus.Incoming, uint64(n), false)
|
||||
prometheus.IncrementBytes("", prometheus.Incoming, uint64(n), false)
|
||||
prometheus.IncrementPackets("", prometheus.Incoming, 1, false)
|
||||
}
|
||||
return
|
||||
}
|
||||
@@ -60,7 +61,8 @@ func (c *Conn) Read(b []byte) (n int, err error) {
|
||||
func (c *Conn) Write(b []byte) (n int, err error) {
|
||||
n, err = c.Conn.Write(b)
|
||||
if n > 0 {
|
||||
prometheus.IncrementBytes(prometheus.Outgoing, uint64(n), false)
|
||||
prometheus.IncrementBytes("", prometheus.Outgoing, uint64(n), false)
|
||||
prometheus.IncrementPackets("", prometheus.Outgoing, 1, false)
|
||||
}
|
||||
return
|
||||
}
|
||||
@@ -83,8 +85,8 @@ func NewPacketConn(c net.PacketConn, direction prometheus.Direction) *PacketConn
|
||||
func (c *PacketConn) ReadFrom(p []byte) (n int, addr net.Addr, err error) {
|
||||
n, addr, err = c.PacketConn.ReadFrom(p)
|
||||
if n > 0 {
|
||||
prometheus.IncrementBytes(prometheus.Incoming, uint64(n), false)
|
||||
prometheus.IncrementPackets(prometheus.Incoming, 1, false)
|
||||
prometheus.IncrementBytes("", prometheus.Incoming, uint64(n), false)
|
||||
prometheus.IncrementPackets("", prometheus.Incoming, 1, false)
|
||||
}
|
||||
return
|
||||
}
|
||||
@@ -92,8 +94,8 @@ func (c *PacketConn) ReadFrom(p []byte) (n int, addr net.Addr, err error) {
|
||||
func (c *PacketConn) WriteTo(p []byte, addr net.Addr) (n int, err error) {
|
||||
n, err = c.PacketConn.WriteTo(p, addr)
|
||||
if n > 0 {
|
||||
prometheus.IncrementBytes(prometheus.Outgoing, uint64(n), false)
|
||||
prometheus.IncrementPackets(prometheus.Outgoing, 1, false)
|
||||
prometheus.IncrementBytes("", prometheus.Outgoing, uint64(n), false)
|
||||
prometheus.IncrementPackets("", prometheus.Outgoing, 1, false)
|
||||
}
|
||||
return
|
||||
}
|
||||
@@ -111,8 +113,8 @@ func NewRelayAddressGenerator(g turn.RelayAddressGenerator) *RelayAddressGenerat
|
||||
return &RelayAddressGenerator{RelayAddressGenerator: g}
|
||||
}
|
||||
|
||||
func (g *RelayAddressGenerator) AllocatePacketConn(network string, requestedPort int) (net.PacketConn, net.Addr, error) {
|
||||
conn, addr, err := g.RelayAddressGenerator.AllocatePacketConn(network, requestedPort)
|
||||
func (g *RelayAddressGenerator) AllocatePacketConn(c turn.AllocateListenerConfig) (net.PacketConn, net.Addr, error) {
|
||||
conn, addr, err := g.RelayAddressGenerator.AllocatePacketConn(c)
|
||||
if err != nil {
|
||||
return nil, addr, err
|
||||
}
|
||||
@@ -120,11 +122,20 @@ func (g *RelayAddressGenerator) AllocatePacketConn(network string, requestedPort
|
||||
return NewPacketConn(conn, prometheus.Outgoing), addr, err
|
||||
}
|
||||
|
||||
func (g *RelayAddressGenerator) AllocateConn(network string, requestedPort int) (net.Conn, net.Addr, error) {
|
||||
conn, addr, err := g.RelayAddressGenerator.AllocateConn(network, requestedPort)
|
||||
func (g *RelayAddressGenerator) AllocateConn(c turn.AllocateConnConfig) (net.Conn, error) {
|
||||
conn, err := g.RelayAddressGenerator.AllocateConn(c)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return NewConn(conn, prometheus.Outgoing), err
|
||||
}
|
||||
|
||||
func (g *RelayAddressGenerator) AllocateListener(c turn.AllocateListenerConfig) (net.Listener, net.Addr, error) {
|
||||
l, addr, err := g.RelayAddressGenerator.AllocateListener(c)
|
||||
if err != nil {
|
||||
return nil, addr, err
|
||||
}
|
||||
|
||||
return NewConn(conn, prometheus.Outgoing), addr, err
|
||||
return NewListener(l), addr, err
|
||||
}
|
||||
|
||||
@@ -19,6 +19,7 @@ import (
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"go.uber.org/zap/zapcore"
|
||||
"google.golang.org/protobuf/types/known/timestamppb"
|
||||
|
||||
"github.com/livekit/livekit-server/pkg/utils"
|
||||
@@ -27,6 +28,37 @@ import (
|
||||
protoutils "github.com/livekit/protocol/utils"
|
||||
)
|
||||
|
||||
type ReferenceGuard struct {
|
||||
activated, released bool
|
||||
}
|
||||
|
||||
type ReferenceCount struct {
|
||||
count int
|
||||
}
|
||||
|
||||
func (s *ReferenceCount) Activate(guard *ReferenceGuard) {
|
||||
if guard != nil && !guard.activated {
|
||||
guard.activated = true
|
||||
s.count++
|
||||
}
|
||||
}
|
||||
|
||||
func (s *ReferenceCount) Release(guard *ReferenceGuard) bool {
|
||||
if guard == nil || !guard.activated || guard.released {
|
||||
return false
|
||||
}
|
||||
guard.released = true
|
||||
s.count--
|
||||
return s.count == 0
|
||||
}
|
||||
|
||||
func (s ReferenceCount) MarshalLogObject(e zapcore.ObjectEncoder) error {
|
||||
e.AddInt("count", s.count)
|
||||
return nil
|
||||
}
|
||||
|
||||
// ----------------------------------------
|
||||
|
||||
// StatsWorker handles participant stats
|
||||
type StatsWorker struct {
|
||||
next *StatsWorker
|
||||
@@ -42,6 +74,7 @@ type StatsWorker struct {
|
||||
lock sync.RWMutex
|
||||
outgoingPerTrack map[livekit.TrackID][]*livekit.AnalyticsStat
|
||||
incomingPerTrack map[livekit.TrackID][]*livekit.AnalyticsStat
|
||||
refCount ReferenceCount
|
||||
closedAt time.Time
|
||||
}
|
||||
|
||||
@@ -52,6 +85,7 @@ func newStatsWorker(
|
||||
roomName livekit.RoomName,
|
||||
participantID livekit.ParticipantID,
|
||||
identity livekit.ParticipantIdentity,
|
||||
guard *ReferenceGuard,
|
||||
) *StatsWorker {
|
||||
s := &StatsWorker{
|
||||
ctx: ctx,
|
||||
@@ -63,6 +97,7 @@ func newStatsWorker(
|
||||
outgoingPerTrack: make(map[livekit.TrackID][]*livekit.AnalyticsStat),
|
||||
incomingPerTrack: make(map[livekit.TrackID][]*livekit.AnalyticsStat),
|
||||
}
|
||||
s.refCount.Activate(guard)
|
||||
return s
|
||||
}
|
||||
|
||||
@@ -93,7 +128,7 @@ func (s *StatsWorker) IsConnected() bool {
|
||||
return s.isConnected
|
||||
}
|
||||
|
||||
func (s *StatsWorker) Flush(now time.Time) bool {
|
||||
func (s *StatsWorker) Flush(now time.Time, closeWait time.Duration) bool {
|
||||
ts := timestamppb.New(now)
|
||||
|
||||
s.lock.Lock()
|
||||
@@ -105,7 +140,7 @@ func (s *StatsWorker) Flush(now time.Time) bool {
|
||||
outgoingPerTrack := s.outgoingPerTrack
|
||||
s.outgoingPerTrack = make(map[livekit.TrackID][]*livekit.AnalyticsStat)
|
||||
|
||||
closed := !s.closedAt.IsZero() && now.Sub(s.closedAt) > workerCleanupWait
|
||||
closed := !s.closedAt.IsZero() && now.Sub(s.closedAt) > closeWait
|
||||
s.lock.Unlock()
|
||||
|
||||
stats = s.collectStats(ts, livekit.StreamType_UPSTREAM, incomingPerTrack, stats)
|
||||
@@ -117,10 +152,14 @@ func (s *StatsWorker) Flush(now time.Time) bool {
|
||||
return closed
|
||||
}
|
||||
|
||||
func (s *StatsWorker) Close() bool {
|
||||
func (s *StatsWorker) Close(guard *ReferenceGuard) bool {
|
||||
s.lock.Lock()
|
||||
defer s.lock.Unlock()
|
||||
|
||||
if !s.refCount.Release(guard) {
|
||||
return false
|
||||
}
|
||||
|
||||
ok := s.closedAt.IsZero()
|
||||
if ok {
|
||||
s.closedAt = time.Now()
|
||||
@@ -128,10 +167,14 @@ func (s *StatsWorker) Close() bool {
|
||||
return ok
|
||||
}
|
||||
|
||||
func (s *StatsWorker) Closed() bool {
|
||||
func (s *StatsWorker) Closed(guard *ReferenceGuard) bool {
|
||||
s.lock.Lock()
|
||||
defer s.lock.Unlock()
|
||||
return !s.closedAt.IsZero()
|
||||
if s.closedAt.IsZero() {
|
||||
s.refCount.Activate(guard)
|
||||
return false
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
func (s *StatsWorker) collectStats(
|
||||
@@ -157,6 +200,20 @@ func (s *StatsWorker) collectStats(
|
||||
return stats
|
||||
}
|
||||
|
||||
func (s *StatsWorker) MarshalLogObject(e zapcore.ObjectEncoder) error {
|
||||
s.lock.RLock()
|
||||
defer s.lock.RUnlock()
|
||||
|
||||
e.AddString("room", string(s.roomName))
|
||||
e.AddString("roomID", string(s.roomID))
|
||||
e.AddString("participant", string(s.participantIdentity))
|
||||
e.AddString("participantID", string(s.participantID))
|
||||
e.AddBool("isConnected", s.isConnected)
|
||||
e.AddTime("closedAt", s.closedAt)
|
||||
e.AddObject("refCount", s.refCount)
|
||||
return nil
|
||||
}
|
||||
|
||||
// -------------------------------------------------------------------------
|
||||
|
||||
// create a single stream and single video layer post aggregation
|
||||
@@ -253,8 +310,9 @@ func coalesce(stats []*livekit.AnalyticsStat) *livekit.AnalyticsStat {
|
||||
|
||||
stat := &livekit.AnalyticsStat{
|
||||
MinScore: minScore,
|
||||
MedianScore: utils.MedianFloat32(scores),
|
||||
MedianScore: utils.Median(scores),
|
||||
Streams: []*livekit.AnalyticsStream{coalescedStream},
|
||||
Mime: stats[len(stats)-1].Mime, // use the latest Mime
|
||||
}
|
||||
numScores := len(scores)
|
||||
if numScores > 0 {
|
||||
@@ -263,6 +321,41 @@ func coalesce(stats []*livekit.AnalyticsStat) *livekit.AnalyticsStat {
|
||||
return stat
|
||||
}
|
||||
|
||||
type CondensedStat struct {
|
||||
StartTime time.Time
|
||||
EndTime time.Time
|
||||
Bytes uint64
|
||||
Packets uint32
|
||||
PacketsLost uint32
|
||||
Frames uint32
|
||||
}
|
||||
|
||||
func CondenseStat(stat *livekit.AnalyticsStat) (ps CondensedStat, ok bool) {
|
||||
if ok = isValid(stat); !ok {
|
||||
return
|
||||
}
|
||||
|
||||
for _, stream := range stat.Streams {
|
||||
startTime := stream.StartTime.AsTime()
|
||||
endTime := stream.EndTime.AsTime()
|
||||
if ps.StartTime.IsZero() || startTime.Before(ps.StartTime) {
|
||||
ps.StartTime = startTime
|
||||
}
|
||||
if endTime.After(ps.EndTime) {
|
||||
ps.EndTime = endTime
|
||||
}
|
||||
|
||||
ps.Bytes += stream.PrimaryBytes
|
||||
ps.Packets += stream.PrimaryPackets
|
||||
ps.PacketsLost += stream.PacketsLost
|
||||
if stream.Frames > ps.Frames {
|
||||
ps.Frames = stream.Frames
|
||||
}
|
||||
}
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
func isValid(stat *livekit.AnalyticsStat) bool {
|
||||
for _, analyticsStream := range stat.Streams {
|
||||
if int32(analyticsStream.PrimaryPackets) < 0 ||
|
||||
|
||||
@@ -0,0 +1,19 @@
|
||||
package telemetry
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestStatsWorker(t *testing.T) {
|
||||
t.Run("reference counted close works", func(t *testing.T) {
|
||||
var g0, g1 ReferenceGuard
|
||||
w := newStatsWorker(t.Context(), nil, "", "", "", "", &g0)
|
||||
require.False(t, w.Closed(&g1))
|
||||
require.False(t, w.Close(&g0))
|
||||
require.False(t, w.Closed(&g1))
|
||||
require.True(t, w.Close(&g1))
|
||||
require.True(t, w.Closed(&g1))
|
||||
})
|
||||
}
|
||||
@@ -7,9 +7,21 @@ import (
|
||||
|
||||
"github.com/livekit/livekit-server/pkg/telemetry"
|
||||
"github.com/livekit/protocol/livekit"
|
||||
"github.com/livekit/protocol/observability/roomobs"
|
||||
)
|
||||
|
||||
type FakeAnalyticsService struct {
|
||||
RoomProjectReporterStub func(context.Context) roomobs.ProjectReporter
|
||||
roomProjectReporterMutex sync.RWMutex
|
||||
roomProjectReporterArgsForCall []struct {
|
||||
arg1 context.Context
|
||||
}
|
||||
roomProjectReporterReturns struct {
|
||||
result1 roomobs.ProjectReporter
|
||||
}
|
||||
roomProjectReporterReturnsOnCall map[int]struct {
|
||||
result1 roomobs.ProjectReporter
|
||||
}
|
||||
SendEventStub func(context.Context, *livekit.AnalyticsEvent)
|
||||
sendEventMutex sync.RWMutex
|
||||
sendEventArgsForCall []struct {
|
||||
@@ -32,6 +44,67 @@ type FakeAnalyticsService struct {
|
||||
invocationsMutex sync.RWMutex
|
||||
}
|
||||
|
||||
func (fake *FakeAnalyticsService) RoomProjectReporter(arg1 context.Context) roomobs.ProjectReporter {
|
||||
fake.roomProjectReporterMutex.Lock()
|
||||
ret, specificReturn := fake.roomProjectReporterReturnsOnCall[len(fake.roomProjectReporterArgsForCall)]
|
||||
fake.roomProjectReporterArgsForCall = append(fake.roomProjectReporterArgsForCall, struct {
|
||||
arg1 context.Context
|
||||
}{arg1})
|
||||
stub := fake.RoomProjectReporterStub
|
||||
fakeReturns := fake.roomProjectReporterReturns
|
||||
fake.recordInvocation("RoomProjectReporter", []interface{}{arg1})
|
||||
fake.roomProjectReporterMutex.Unlock()
|
||||
if stub != nil {
|
||||
return stub(arg1)
|
||||
}
|
||||
if specificReturn {
|
||||
return ret.result1
|
||||
}
|
||||
return fakeReturns.result1
|
||||
}
|
||||
|
||||
func (fake *FakeAnalyticsService) RoomProjectReporterCallCount() int {
|
||||
fake.roomProjectReporterMutex.RLock()
|
||||
defer fake.roomProjectReporterMutex.RUnlock()
|
||||
return len(fake.roomProjectReporterArgsForCall)
|
||||
}
|
||||
|
||||
func (fake *FakeAnalyticsService) RoomProjectReporterCalls(stub func(context.Context) roomobs.ProjectReporter) {
|
||||
fake.roomProjectReporterMutex.Lock()
|
||||
defer fake.roomProjectReporterMutex.Unlock()
|
||||
fake.RoomProjectReporterStub = stub
|
||||
}
|
||||
|
||||
func (fake *FakeAnalyticsService) RoomProjectReporterArgsForCall(i int) context.Context {
|
||||
fake.roomProjectReporterMutex.RLock()
|
||||
defer fake.roomProjectReporterMutex.RUnlock()
|
||||
argsForCall := fake.roomProjectReporterArgsForCall[i]
|
||||
return argsForCall.arg1
|
||||
}
|
||||
|
||||
func (fake *FakeAnalyticsService) RoomProjectReporterReturns(result1 roomobs.ProjectReporter) {
|
||||
fake.roomProjectReporterMutex.Lock()
|
||||
defer fake.roomProjectReporterMutex.Unlock()
|
||||
fake.RoomProjectReporterStub = nil
|
||||
fake.roomProjectReporterReturns = struct {
|
||||
result1 roomobs.ProjectReporter
|
||||
}{result1}
|
||||
}
|
||||
|
||||
func (fake *FakeAnalyticsService) RoomProjectReporterReturnsOnCall(i int, result1 roomobs.ProjectReporter) {
|
||||
fake.roomProjectReporterMutex.Lock()
|
||||
defer fake.roomProjectReporterMutex.Unlock()
|
||||
fake.RoomProjectReporterStub = nil
|
||||
if fake.roomProjectReporterReturnsOnCall == nil {
|
||||
fake.roomProjectReporterReturnsOnCall = make(map[int]struct {
|
||||
result1 roomobs.ProjectReporter
|
||||
})
|
||||
}
|
||||
fake.roomProjectReporterReturnsOnCall[i] = struct {
|
||||
result1 roomobs.ProjectReporter
|
||||
}{result1}
|
||||
}
|
||||
|
||||
func (fake *FakeAnalyticsService) SendEvent(arg1 context.Context, arg2 *livekit.AnalyticsEvent) {
|
||||
fake.sendEventMutex.Lock()
|
||||
fake.sendEventArgsForCall = append(fake.sendEventArgsForCall, struct {
|
||||
@@ -139,12 +212,6 @@ func (fake *FakeAnalyticsService) SendStatsArgsForCall(i int) (context.Context,
|
||||
func (fake *FakeAnalyticsService) Invocations() map[string][][]interface{} {
|
||||
fake.invocationsMutex.RLock()
|
||||
defer fake.invocationsMutex.RUnlock()
|
||||
fake.sendEventMutex.RLock()
|
||||
defer fake.sendEventMutex.RUnlock()
|
||||
fake.sendNodeRoomStatesMutex.RLock()
|
||||
defer fake.sendNodeRoomStatesMutex.RUnlock()
|
||||
fake.sendStatsMutex.RLock()
|
||||
defer fake.sendStatsMutex.RUnlock()
|
||||
copiedInvocations := map[string][][]interface{}{}
|
||||
for key, value := range fake.invocations {
|
||||
copiedInvocations[key] = value
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -19,8 +19,8 @@ import (
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/livekit/livekit-server/pkg/sfu/mime"
|
||||
"github.com/livekit/livekit-server/pkg/utils"
|
||||
"github.com/livekit/protocol/codecs/mime"
|
||||
"github.com/livekit/protocol/livekit"
|
||||
"github.com/livekit/protocol/logger"
|
||||
"github.com/livekit/protocol/webhook"
|
||||
@@ -31,70 +31,145 @@ import (
|
||||
//counterfeiter:generate . TelemetryService
|
||||
type TelemetryService interface {
|
||||
// TrackStats is called periodically for each track in both directions (published/subscribed)
|
||||
TrackStats(key StatsKey, stat *livekit.AnalyticsStat)
|
||||
TrackStats(roomID livekit.RoomID, roomName livekit.RoomName, key StatsKey, stat *livekit.AnalyticsStat)
|
||||
|
||||
// events
|
||||
RoomStarted(ctx context.Context, room *livekit.Room)
|
||||
RoomEnded(ctx context.Context, room *livekit.Room)
|
||||
|
||||
// ParticipantJoined - a participant establishes signal connection to a room
|
||||
ParticipantJoined(ctx context.Context, room *livekit.Room, participant *livekit.ParticipantInfo, clientInfo *livekit.ClientInfo, clientMeta *livekit.AnalyticsClientMeta, shouldSendEvent bool)
|
||||
ParticipantJoined(ctx context.Context, room *livekit.Room, participant *livekit.ParticipantInfo, clientInfo *livekit.ClientInfo, clientMeta *livekit.AnalyticsClientMeta, shouldSendEvent bool, guard *ReferenceGuard)
|
||||
// ParticipantActive - a participant establishes media connection
|
||||
ParticipantActive(ctx context.Context, room *livekit.Room, participant *livekit.ParticipantInfo, clientMeta *livekit.AnalyticsClientMeta, isMigration bool)
|
||||
ParticipantActive(ctx context.Context, room *livekit.Room, participant *livekit.ParticipantInfo, clientMeta *livekit.AnalyticsClientMeta, isMigration bool, guard *ReferenceGuard)
|
||||
// ParticipantResumed - there has been an ICE restart or connection resume attempt, and we've received their signal connection
|
||||
ParticipantResumed(ctx context.Context, room *livekit.Room, participant *livekit.ParticipantInfo, nodeID livekit.NodeID, reason livekit.ReconnectReason)
|
||||
// ParticipantLeft - the participant leaves the room, only sent if ParticipantActive has been called before
|
||||
ParticipantLeft(ctx context.Context, room *livekit.Room, participant *livekit.ParticipantInfo, shouldSendEvent bool)
|
||||
ParticipantLeft(ctx context.Context, room *livekit.Room, participant *livekit.ParticipantInfo, shouldSendEvent bool, guard *ReferenceGuard)
|
||||
// TrackPublishRequested - a publication attempt has been received
|
||||
TrackPublishRequested(ctx context.Context, participantID livekit.ParticipantID, identity livekit.ParticipantIdentity, track *livekit.TrackInfo)
|
||||
TrackPublishRequested(ctx context.Context, roomID livekit.RoomID, roomName livekit.RoomName, participantID livekit.ParticipantID, identity livekit.ParticipantIdentity, track *livekit.TrackInfo)
|
||||
// TrackPublished - a publication attempt has been successful
|
||||
TrackPublished(ctx context.Context, participantID livekit.ParticipantID, identity livekit.ParticipantIdentity, track *livekit.TrackInfo)
|
||||
TrackPublished(ctx context.Context, roomID livekit.RoomID, roomName livekit.RoomName, participantID livekit.ParticipantID, identity livekit.ParticipantIdentity, track *livekit.TrackInfo, shouldSendEvent bool)
|
||||
// TrackUnpublished - a participant unpublished a track
|
||||
TrackUnpublished(ctx context.Context, participantID livekit.ParticipantID, identity livekit.ParticipantIdentity, track *livekit.TrackInfo, shouldSendEvent bool)
|
||||
TrackUnpublished(ctx context.Context, roomID livekit.RoomID, roomName livekit.RoomName, participantID livekit.ParticipantID, identity livekit.ParticipantIdentity, track *livekit.TrackInfo, shouldSendEvent bool)
|
||||
// TrackSubscribeRequested - a participant requested to subscribe to a track
|
||||
TrackSubscribeRequested(ctx context.Context, participantID livekit.ParticipantID, track *livekit.TrackInfo)
|
||||
TrackSubscribeRequested(ctx context.Context, roomID livekit.RoomID, roomName livekit.RoomName, participantID livekit.ParticipantID, track *livekit.TrackInfo)
|
||||
// TrackSubscribed - a participant subscribed to a track successfully
|
||||
TrackSubscribed(ctx context.Context, participantID livekit.ParticipantID, track *livekit.TrackInfo, publisher *livekit.ParticipantInfo, shouldSendEvent bool)
|
||||
TrackSubscribed(ctx context.Context, roomID livekit.RoomID, roomName livekit.RoomName, participantID livekit.ParticipantID, track *livekit.TrackInfo, publisher *livekit.ParticipantInfo, shouldSendEvent bool)
|
||||
// TrackUnsubscribed - a participant unsubscribed from a track successfully
|
||||
TrackUnsubscribed(ctx context.Context, participantID livekit.ParticipantID, track *livekit.TrackInfo, shouldSendEvent bool)
|
||||
TrackUnsubscribed(ctx context.Context, roomID livekit.RoomID, roomName livekit.RoomName, participantID livekit.ParticipantID, track *livekit.TrackInfo, shouldSendEvent bool)
|
||||
// TrackSubscribeFailed - failure to subscribe to a track
|
||||
TrackSubscribeFailed(ctx context.Context, participantID livekit.ParticipantID, trackID livekit.TrackID, err error, isUserError bool)
|
||||
TrackSubscribeFailed(ctx context.Context, roomID livekit.RoomID, roomName livekit.RoomName, participantID livekit.ParticipantID, trackID livekit.TrackID, err error, isUserError bool)
|
||||
// TrackMuted - the publisher has muted the Track
|
||||
TrackMuted(ctx context.Context, participantID livekit.ParticipantID, track *livekit.TrackInfo)
|
||||
TrackMuted(ctx context.Context, roomID livekit.RoomID, roomName livekit.RoomName, participantID livekit.ParticipantID, track *livekit.TrackInfo)
|
||||
// TrackUnmuted - the publisher has muted the Track
|
||||
TrackUnmuted(ctx context.Context, participantID livekit.ParticipantID, track *livekit.TrackInfo)
|
||||
TrackUnmuted(ctx context.Context, roomID livekit.RoomID, roomName livekit.RoomName, participantID livekit.ParticipantID, track *livekit.TrackInfo)
|
||||
// TrackPublishedUpdate - track metadata has been updated
|
||||
TrackPublishedUpdate(ctx context.Context, participantID livekit.ParticipantID, track *livekit.TrackInfo)
|
||||
TrackPublishedUpdate(ctx context.Context, roomID livekit.RoomID, roomName livekit.RoomName, participantID livekit.ParticipantID, track *livekit.TrackInfo)
|
||||
// TrackMaxSubscribedVideoQuality - publisher is notified of the max quality subscribers desire
|
||||
TrackMaxSubscribedVideoQuality(ctx context.Context, participantID livekit.ParticipantID, track *livekit.TrackInfo, mime mime.MimeType, maxQuality livekit.VideoQuality)
|
||||
TrackPublishRTPStats(ctx context.Context, participantID livekit.ParticipantID, trackID livekit.TrackID, mimeType mime.MimeType, layer int, stats *livekit.RTPStats)
|
||||
TrackSubscribeRTPStats(ctx context.Context, participantID livekit.ParticipantID, trackID livekit.TrackID, mimeType mime.MimeType, stats *livekit.RTPStats)
|
||||
TrackMaxSubscribedVideoQuality(ctx context.Context, roomID livekit.RoomID, roomName livekit.RoomName, participantID livekit.ParticipantID, track *livekit.TrackInfo, mime mime.MimeType, maxQuality livekit.VideoQuality)
|
||||
TrackPublishRTPStats(ctx context.Context, roomID livekit.RoomID, roomName livekit.RoomName, participantID livekit.ParticipantID, trackID livekit.TrackID, mimeType mime.MimeType, layer int, stats *livekit.RTPStats)
|
||||
TrackSubscribeRTPStats(ctx context.Context, roomID livekit.RoomID, roomName livekit.RoomName, participantID livekit.ParticipantID, trackID livekit.TrackID, mimeType mime.MimeType, stats *livekit.RTPStats)
|
||||
|
||||
EgressStarted(ctx context.Context, info *livekit.EgressInfo)
|
||||
EgressUpdated(ctx context.Context, info *livekit.EgressInfo)
|
||||
EgressEnded(ctx context.Context, info *livekit.EgressInfo)
|
||||
|
||||
IngressCreated(ctx context.Context, info *livekit.IngressInfo)
|
||||
IngressDeleted(ctx context.Context, info *livekit.IngressInfo)
|
||||
IngressStarted(ctx context.Context, info *livekit.IngressInfo)
|
||||
IngressUpdated(ctx context.Context, info *livekit.IngressInfo)
|
||||
IngressEnded(ctx context.Context, info *livekit.IngressInfo)
|
||||
|
||||
LocalRoomState(ctx context.Context, info *livekit.AnalyticsNodeRooms)
|
||||
|
||||
Report(ctx context.Context, reportInfo *livekit.ReportInfo)
|
||||
|
||||
APICall(ctx context.Context, apiCallInfo *livekit.APICallInfo)
|
||||
|
||||
Webhook(ctx context.Context, webhookInfo *livekit.WebhookInfo)
|
||||
|
||||
// helpers
|
||||
AnalyticsService
|
||||
NotifyEvent(ctx context.Context, event *livekit.WebhookEvent)
|
||||
NotifyEgressEvent(ctx context.Context, event string, info *livekit.EgressInfo)
|
||||
FlushStats()
|
||||
}
|
||||
|
||||
// -----------------------------
|
||||
|
||||
var _ TelemetryService = (*NullTelemetryService)(nil)
|
||||
|
||||
type NullTelemetryService struct {
|
||||
NullAnalyticService
|
||||
}
|
||||
|
||||
func (n NullTelemetryService) TrackStats(roomID livekit.RoomID, roomName livekit.RoomName, key StatsKey, stat *livekit.AnalyticsStat) {
|
||||
}
|
||||
func (n NullTelemetryService) RoomStarted(ctx context.Context, room *livekit.Room) {}
|
||||
func (n NullTelemetryService) RoomEnded(ctx context.Context, room *livekit.Room) {}
|
||||
func (n NullTelemetryService) ParticipantJoined(ctx context.Context, room *livekit.Room, participant *livekit.ParticipantInfo, clientInfo *livekit.ClientInfo, clientMeta *livekit.AnalyticsClientMeta, shouldSendEvent bool, guard *ReferenceGuard) {
|
||||
}
|
||||
func (n NullTelemetryService) ParticipantActive(ctx context.Context, room *livekit.Room, participant *livekit.ParticipantInfo, clientMeta *livekit.AnalyticsClientMeta, isMigration bool, guard *ReferenceGuard) {
|
||||
}
|
||||
func (n NullTelemetryService) ParticipantResumed(ctx context.Context, room *livekit.Room, participant *livekit.ParticipantInfo, nodeID livekit.NodeID, reason livekit.ReconnectReason) {
|
||||
}
|
||||
func (n NullTelemetryService) ParticipantLeft(ctx context.Context, room *livekit.Room, participant *livekit.ParticipantInfo, shouldSendEvent bool, guard *ReferenceGuard) {
|
||||
}
|
||||
func (n NullTelemetryService) TrackPublishRequested(ctx context.Context, roomID livekit.RoomID, roomName livekit.RoomName, participantID livekit.ParticipantID, identity livekit.ParticipantIdentity, track *livekit.TrackInfo) {
|
||||
}
|
||||
func (n NullTelemetryService) TrackPublished(ctx context.Context, roomID livekit.RoomID, roomName livekit.RoomName, participantID livekit.ParticipantID, identity livekit.ParticipantIdentity, track *livekit.TrackInfo, shouldSendEvent bool) {
|
||||
}
|
||||
func (n NullTelemetryService) TrackUnpublished(ctx context.Context, roomID livekit.RoomID, roomName livekit.RoomName, participantID livekit.ParticipantID, identity livekit.ParticipantIdentity, track *livekit.TrackInfo, shouldSendEvent bool) {
|
||||
}
|
||||
func (n NullTelemetryService) TrackSubscribeRequested(ctx context.Context, roomID livekit.RoomID, roomName livekit.RoomName, participantID livekit.ParticipantID, track *livekit.TrackInfo) {
|
||||
}
|
||||
func (n NullTelemetryService) TrackSubscribed(ctx context.Context, roomID livekit.RoomID, roomName livekit.RoomName, participantID livekit.ParticipantID, track *livekit.TrackInfo, publisher *livekit.ParticipantInfo, shouldSendEvent bool) {
|
||||
}
|
||||
func (n NullTelemetryService) TrackUnsubscribed(ctx context.Context, roomID livekit.RoomID, roomName livekit.RoomName, participantID livekit.ParticipantID, track *livekit.TrackInfo, shouldSendEvent bool) {
|
||||
}
|
||||
func (n NullTelemetryService) TrackSubscribeFailed(ctx context.Context, roomID livekit.RoomID, roomName livekit.RoomName, participantID livekit.ParticipantID, trackID livekit.TrackID, err error, isUserError bool) {
|
||||
}
|
||||
func (n NullTelemetryService) TrackMuted(ctx context.Context, roomID livekit.RoomID, roomName livekit.RoomName, participantID livekit.ParticipantID, track *livekit.TrackInfo) {
|
||||
}
|
||||
func (n NullTelemetryService) TrackUnmuted(ctx context.Context, roomID livekit.RoomID, roomName livekit.RoomName, participantID livekit.ParticipantID, track *livekit.TrackInfo) {
|
||||
}
|
||||
func (n NullTelemetryService) TrackPublishedUpdate(ctx context.Context, roomID livekit.RoomID, roomName livekit.RoomName, participantID livekit.ParticipantID, track *livekit.TrackInfo) {
|
||||
}
|
||||
func (n NullTelemetryService) TrackMaxSubscribedVideoQuality(ctx context.Context, roomID livekit.RoomID, roomName livekit.RoomName, participantID livekit.ParticipantID, track *livekit.TrackInfo, mime mime.MimeType, maxQuality livekit.VideoQuality) {
|
||||
}
|
||||
func (n NullTelemetryService) TrackPublishRTPStats(ctx context.Context, roomID livekit.RoomID, roomName livekit.RoomName, participantID livekit.ParticipantID, trackID livekit.TrackID, mimeType mime.MimeType, layer int, stats *livekit.RTPStats) {
|
||||
}
|
||||
func (n NullTelemetryService) TrackSubscribeRTPStats(ctx context.Context, roomID livekit.RoomID, roomName livekit.RoomName, participantID livekit.ParticipantID, trackID livekit.TrackID, mimeType mime.MimeType, stats *livekit.RTPStats) {
|
||||
}
|
||||
func (n NullTelemetryService) EgressStarted(ctx context.Context, info *livekit.EgressInfo) {}
|
||||
func (n NullTelemetryService) EgressUpdated(ctx context.Context, info *livekit.EgressInfo) {}
|
||||
func (n NullTelemetryService) EgressEnded(ctx context.Context, info *livekit.EgressInfo) {}
|
||||
func (n NullTelemetryService) IngressCreated(ctx context.Context, info *livekit.IngressInfo) {}
|
||||
func (n NullTelemetryService) IngressDeleted(ctx context.Context, info *livekit.IngressInfo) {}
|
||||
func (n NullTelemetryService) IngressStarted(ctx context.Context, info *livekit.IngressInfo) {}
|
||||
func (n NullTelemetryService) IngressUpdated(ctx context.Context, info *livekit.IngressInfo) {}
|
||||
func (n NullTelemetryService) IngressEnded(ctx context.Context, info *livekit.IngressInfo) {}
|
||||
func (n NullTelemetryService) LocalRoomState(ctx context.Context, info *livekit.AnalyticsNodeRooms) {}
|
||||
func (n NullTelemetryService) Report(ctx context.Context, reportInfo *livekit.ReportInfo) {}
|
||||
func (n NullTelemetryService) APICall(ctx context.Context, apiCallInfo *livekit.APICallInfo) {}
|
||||
func (n NullTelemetryService) Webhook(ctx context.Context, webhookInfo *livekit.WebhookInfo) {}
|
||||
func (n NullTelemetryService) NotifyEgressEvent(ctx context.Context, event string, info *livekit.EgressInfo) {
|
||||
}
|
||||
func (n NullTelemetryService) FlushStats() {}
|
||||
|
||||
// -----------------------------
|
||||
|
||||
const (
|
||||
workerCleanupWait = 3 * time.Minute
|
||||
jobsQueueMinSize = 2048
|
||||
|
||||
telemetryStatsUpdateInterval = time.Second * 30
|
||||
telemetryNonMediaStatsUpdateInterval = time.Second * 30
|
||||
telemetryStatsUpdateInterval = time.Second * 30
|
||||
)
|
||||
|
||||
type statsWorkerKey struct {
|
||||
roomID livekit.RoomID
|
||||
participantID livekit.ParticipantID
|
||||
}
|
||||
|
||||
type telemetryService struct {
|
||||
AnalyticsService
|
||||
|
||||
@@ -102,7 +177,7 @@ type telemetryService struct {
|
||||
jobsQueue *utils.OpsQueue
|
||||
|
||||
workersMu sync.RWMutex
|
||||
workers map[livekit.ParticipantID]*StatsWorker
|
||||
workers map[statsWorkerKey]*StatsWorker
|
||||
workerList *StatsWorker
|
||||
|
||||
flushMu sync.Mutex
|
||||
@@ -118,12 +193,7 @@ func NewTelemetryService(notifier webhook.QueuedNotifier, analytics AnalyticsSer
|
||||
FlushOnStop: true,
|
||||
Logger: logger.GetLogger(),
|
||||
}),
|
||||
workers: make(map[livekit.ParticipantID]*StatsWorker),
|
||||
}
|
||||
if t.notifier != nil {
|
||||
t.notifier.RegisterProcessedHook(func(ctx context.Context, whi *livekit.WebhookInfo) {
|
||||
t.Webhook(ctx, whi)
|
||||
})
|
||||
workers: make(map[statsWorkerKey]*StatsWorker),
|
||||
}
|
||||
|
||||
t.jobsQueue.Start()
|
||||
@@ -144,7 +214,7 @@ func (t *telemetryService) FlushStats() {
|
||||
var prev, reap *StatsWorker
|
||||
for worker != nil {
|
||||
next := worker.next
|
||||
if closed := worker.Flush(now); closed {
|
||||
if closed := worker.Flush(now, workerCleanupWait); closed {
|
||||
if prev == nil {
|
||||
// this worker was at the head of the list
|
||||
t.workersMu.Lock()
|
||||
@@ -172,8 +242,9 @@ func (t *telemetryService) FlushStats() {
|
||||
if reap != nil {
|
||||
t.workersMu.Lock()
|
||||
for reap != nil {
|
||||
if reap == t.workers[reap.participantID] {
|
||||
delete(t.workers, reap.participantID)
|
||||
key := statsWorkerKey{reap.roomID, reap.participantID}
|
||||
if reap == t.workers[key] {
|
||||
delete(t.workers, key)
|
||||
}
|
||||
reap = reap.next
|
||||
}
|
||||
@@ -191,11 +262,11 @@ func (t *telemetryService) enqueue(op func()) {
|
||||
t.jobsQueue.Enqueue(op)
|
||||
}
|
||||
|
||||
func (t *telemetryService) getWorker(participantID livekit.ParticipantID) (worker *StatsWorker, ok bool) {
|
||||
func (t *telemetryService) getWorker(roomID livekit.RoomID, participantID livekit.ParticipantID) (worker *StatsWorker, ok bool) {
|
||||
t.workersMu.RLock()
|
||||
defer t.workersMu.RUnlock()
|
||||
|
||||
worker, ok = t.workers[participantID]
|
||||
worker, ok = t.workers[statsWorkerKey{roomID, participantID}]
|
||||
return
|
||||
}
|
||||
|
||||
@@ -205,12 +276,14 @@ func (t *telemetryService) getOrCreateWorker(
|
||||
roomName livekit.RoomName,
|
||||
participantID livekit.ParticipantID,
|
||||
participantIdentity livekit.ParticipantIdentity,
|
||||
guard *ReferenceGuard,
|
||||
) (*StatsWorker, bool) {
|
||||
t.workersMu.Lock()
|
||||
defer t.workersMu.Unlock()
|
||||
|
||||
worker, ok := t.workers[participantID]
|
||||
if ok && !worker.Closed() {
|
||||
key := statsWorkerKey{roomID, participantID}
|
||||
worker, ok := t.workers[key]
|
||||
if ok && !worker.Closed(guard) {
|
||||
return worker, true
|
||||
}
|
||||
|
||||
@@ -226,12 +299,13 @@ func (t *telemetryService) getOrCreateWorker(
|
||||
roomName,
|
||||
participantID,
|
||||
participantIdentity,
|
||||
guard,
|
||||
)
|
||||
if existingIsConnected {
|
||||
worker.SetConnected()
|
||||
}
|
||||
|
||||
t.workers[participantID] = worker
|
||||
t.workers[key] = worker
|
||||
|
||||
worker.next = t.workerList
|
||||
t.workerList = worker
|
||||
|
||||
Reference in New Issue
Block a user